diff options
| -rw-r--r-- | tensorflow/python/keras/engine/base_layer.py | 69 | ||||
| -rw-r--r-- | tensorflow/python/keras/engine/network.py | 20 | ||||
| -rw-r--r-- | tensorflow/python/keras/engine/topology_test.py | 172 | ||||
| -rw-r--r-- | tensorflow/python/layers/base.py | 10 | ||||
| -rw-r--r-- | tensorflow/python/layers/base_test.py | 62 |
5 files changed, 11 insertions, 322 deletions
diff --git a/tensorflow/python/keras/engine/base_layer.py b/tensorflow/python/keras/engine/base_layer.py index b05bc96e28..e8cdda30a2 100644 --- a/tensorflow/python/keras/engine/base_layer.py +++ b/tensorflow/python/keras/engine/base_layer.py @@ -41,7 +41,6 @@ from tensorflow.python.keras.utils.generic_utils import to_snake_case # pylint: from tensorflow.python.keras.utils.tf_utils import is_tensor_or_tensor_list # pylint: disable=unused-import from tensorflow.python.ops import array_ops from tensorflow.python.ops import init_ops -from tensorflow.python.ops import math_ops from tensorflow.python.ops import variable_scope as vs from tensorflow.python.ops import variables as tf_variables from tensorflow.python.training.checkpointable import base as checkpointable @@ -89,11 +88,6 @@ class Layer(checkpointable.CheckpointableBase): once. Should actually perform the logic of applying the layer to the input tensors (which should be passed in as the first argument). - A note on a layer's `dtype` property: - A layer's dtype can be specified via the constructor `dtype` argument, and - defaults to the dtype of the first input when the layer is called. The dtype - cannot be changed once set. - All floating point tensor inputs and arguments are casted to the layer's dtype, before the body of the layer computation happens. For models with layers of different dtypes, this helps getting rid of the explicit casts @@ -106,15 +100,13 @@ class Layer(checkpointable.CheckpointableBase): Arguments: trainable: Boolean, whether the layer's variables should be trainable. name: String name of the layer. - dtype: Default dtype of the layer's weights and computations (default of - `None` means use the type of the first input). If not None, inputs will be - casted to this dtype. + dtype: Default dtype of the layer's weights (default of `None` means use the + type of the first input). Read-only properties: name: The name of the layer (string). - dtype: Default dtype of the layer's weights and computations. (default of - `None` means use the type of the first input). If not None, inputs will be - casted to this dtype. + dtype: Default dtype of the layer's weights (default of `None` means use the + type of the first input). trainable_variables: List of trainable variables. non_trainable_variables: List of non-trainable variables. variables: List of all variables of this layer, trainable and @@ -683,12 +675,6 @@ class Layer(checkpointable.CheckpointableBase): kwargs['mask'] = previous_mask input_shapes = None - # Inputs are only casted if a dtype is pased in the constructor, or if a - # layer's __call__() has been previously invoked. At present, only floating - # point tensor inputs are affected. - # TODO(b/77478433): Perhaps we should only cast inputs if a dtype was passed - # to the constructor, not when the layer has previously been called. - inputs_should_be_cast = (self.dtype is not None) with ops.name_scope(self._name_scope()): if not self.built: @@ -723,12 +709,7 @@ class Layer(checkpointable.CheckpointableBase): self._assert_input_compatibility(inputs) if not in_deferred_mode: - if inputs_should_be_cast: - cast_inputs, cast_args, cast_kwargs = self._cast_inputs_and_args( - inputs, *args, **kwargs) - else: - cast_inputs, cast_args, cast_kwargs = inputs, args, kwargs - outputs = self.call(cast_inputs, *cast_args, **cast_kwargs) + outputs = self.call(inputs, *args, **kwargs) if outputs is None: raise ValueError('A layer\'s `call` method should return a Tensor ' 'or a list of Tensors, not None (layer: ' + @@ -743,9 +724,6 @@ class Layer(checkpointable.CheckpointableBase): output_shapes = nest.flatten(output_shapes) outputs = [ # TODO(fchollet): name the deferred tensors? - # TODO(b/77478433): Compute the proper dtype here, by adding a - # compute_output_dtype method. Currently keras Models do not - # properly compute the output dtype. DeferredTensor(shape=shape, dtype=self._dtype) for shape in output_shapes ] @@ -804,43 +782,6 @@ class Layer(checkpointable.CheckpointableBase): """ return self.__call__(inputs, *args, **kwargs) - def _cast_fn(self, x): - """If x is a tensor, casts to this layer's dtype.""" - # TODO(b/77478433): Cast tensor-like things like SparseTensors, Variables, - # ResourceVariables, etc. - if (isinstance(x, ops.Tensor) and x.dtype.is_floating and - dtypes.as_dtype(self.dtype).is_floating): - return math_ops.cast(x, self.dtype) - else: - return x - - def _cast_inputs_and_args(self, inputs, *args, **kwargs): - """Casts the inputs, args, and kwargs of a layer to the layer's dtype. - - This is intended to be potentially overridden by subclasses. By default, - inputs, args, and kwargs are automatically casted to the layer's dtype. - Overriding this method allows only some of the parameters to be treated - differently. - - Currently, this only casts floating point tensors to floating point dtypes, - but more types may be casted in the future. - - Does not modify inputs, args, or kwargs. - - Args: - inputs: The inputs to self.__call__. - *args: The args to self.__call__. - **kwargs: The kwargs to self.__call__. - - Returns: - A tuple (new_inputs, new_args, new_kwargs), where tensors in inputs, - args, and kwargs have been casted to self.dtype. - """ - new_inputs = nest.map_structure(self._cast_fn, inputs) - new_args = nest.map_structure(self._cast_fn, args) - new_kwargs = nest.map_structure(self._cast_fn, kwargs) - return new_inputs, new_args, new_kwargs - def _set_learning_phase_metadata(self, inputs, outputs): # Update learning phase info. To work with subclassed models, # this should be done even if Keras metadata is absent. diff --git a/tensorflow/python/keras/engine/network.py b/tensorflow/python/keras/engine/network.py index 1c9135982e..427efaaf11 100644 --- a/tensorflow/python/keras/engine/network.py +++ b/tensorflow/python/keras/engine/network.py @@ -887,16 +887,8 @@ class Network(base_layer.Layer): if 'training' in tf_inspect.getargspec(layer.call).args: kwargs.setdefault('training', training) - if layer.dtype is not None: - cast_computed_tensors, cast_args, cast_kwargs = ( - layer._cast_inputs_and_args(computed_tensor, **kwargs)) - else: - cast_computed_tensors = [computed_tensor] - cast_args = () - cast_kwargs = kwargs - output_tensors = nest.flatten( - layer.call(cast_computed_tensors, *cast_args, **cast_kwargs)) + layer.call(computed_tensor, **kwargs)) if hasattr(layer, 'compute_mask'): output_masks = layer.compute_mask(computed_tensor, computed_mask) @@ -916,16 +908,8 @@ class Network(base_layer.Layer): if 'training' in tf_inspect.getargspec(layer.call).args: kwargs.setdefault('training', training) - if layer.dtype is not None: - cast_computed_tensors, cast_args, cast_kwargs = ( - layer._cast_inputs_and_args(computed_tensors, **kwargs)) - else: - cast_computed_tensors = computed_tensors - cast_args = () - cast_kwargs = kwargs - output_tensors = nest.flatten( - layer.call(cast_computed_tensors, *cast_args, **cast_kwargs)) + layer.call(computed_tensors, **kwargs)) if hasattr(layer, 'compute_mask'): output_masks = layer.compute_mask(computed_tensors, diff --git a/tensorflow/python/keras/engine/topology_test.py b/tensorflow/python/keras/engine/topology_test.py index d28c30cb7d..183e26e8bf 100644 --- a/tensorflow/python/keras/engine/topology_test.py +++ b/tensorflow/python/keras/engine/topology_test.py @@ -18,8 +18,6 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -import collections - import numpy as np from tensorflow.python import keras @@ -912,176 +910,6 @@ class TopologyConstructionTest(test.TestCase): assert out.shape == (4, 3, 2, 1) self.assertAllClose(out, x * 0.2 + x * 0.3, atol=1e-4) - @test_util.run_in_graph_and_eager_modes() - def test_casting_args(self): - # args of type B will be casted, as we cast elements of namedtuples - B = collections.namedtuple('B', ['x', 'y', 'z']) # pylint: disable=invalid-name - - # args of type C will not be casted, as we do not look at object - # attributes for tensors to cast - class C(object): - - def __init__(self, w): - self.w = w - - inp = array_ops.ones((1,), name='input', dtype='float64') - a = array_ops.ones((1,), name='a', dtype='float64') - b = B(array_ops.ones((1,), name='a', dtype='float64'), None, - np.ones((1,), 'float64')) # Numpy tensors should not be casted - c = C(array_ops.ones((1,), name='a', dtype='float64')) - - # Test inputs are automatically casted. - class MyLayer(keras.layers.Layer): - - def call(self, inputs, a, b, c): - self.a = a - self.b = b - self.c = c - return inputs - - def compute_output_shape(self, input_shape): - return input_shape - - layer = MyLayer(dtype='float16') - out = layer(inp, a=a, b=b, c=c) - self.assertEqual(out.dtype, dtypes.float16) - self.assertEqual(layer.a.dtype, dtypes.float16) - self.assertEqual(layer.b.x.dtype, dtypes.float16) - self.assertEqual(layer.b.y, None) - self.assertEqual(layer.b.z.dtype, np.float64) - self.assertEqual(layer.c.w.dtype, dtypes.float64) - - # Test overriding _cast_inputs_and_args - class MyLayerOverrideCastInputs(MyLayer): - - def _cast_inputs_and_args(self, inputs, a, b, c): - new_inputs = self._cast_fn(inputs) - new_a = a - new_b = b - new_c = C(self._cast_fn(c.w)) - return new_inputs, (new_a, new_b, new_c), {} - - layer = MyLayerOverrideCastInputs(dtype='float16') - out = layer(inp, a=a, b=b, c=c) - self.assertEqual(out.dtype, dtypes.float16) - self.assertEqual(layer.a.dtype, dtypes.float64) - self.assertEqual(layer.b.x.dtype, dtypes.float64) - self.assertEqual(layer.b.y, None) - self.assertEqual(layer.b.z.dtype, np.float64) - self.assertEqual(layer.c.w.dtype, dtypes.float16) - - @test_util.run_in_graph_and_eager_modes() - def test_do_not_cast_ints(self): - class MyLayer(keras.layers.Layer): - - def build(self, input_shape): - self.v = self.add_variable('v', (), 'int32') - super(MyLayer, self).build(input_shape) - - def call(self, inputs): - return inputs + self.v - - def compute_output_shape(self, input_shape): - return input_shape - - a = array_ops.ones((10, 32), dtype='int32') - layer = MyLayer(dtype='float32') - b = layer(a) - self.assertEqual(layer.v.dtype.base_dtype, dtypes.int32) - self.assertEqual(b.dtype, dtypes.int32) - - @test_util.run_in_graph_and_eager_modes() - def test_casting_when_dtype_not_passed_to_constructor(self): - class MyLayer(keras.layers.Layer): - - def call(self, a): - self.a = a - return a - - def compute_output_shape(self, input_shape): - return input_shape - - # Do not cast inputs for the first __call__ if a dtype is not passed to the - # constructor. - a = array_ops.ones((10, 32), dtype='float64') - layer = MyLayer() - self.assertEqual(layer.dtype, None) - b = layer(a) - self.assertEqual(layer.dtype, 'float64') - self.assertEqual(layer.a.dtype, dtypes.float64) - self.assertEqual(b.dtype, dtypes.float64) - - # For a subsequent __call__, the layer's dtype has been set so inputs should - # be casted to the dtype of the input to the first __call__. - a = array_ops.ones((10, 32), dtype='float32') - b = layer(a) - self.assertEqual(layer.dtype, 'float64') - self.assertEqual(layer.a.dtype, dtypes.float64) - self.assertEqual(b.dtype, dtypes.float64) - - @test_util.run_in_graph_and_eager_modes() - def test_casting_with_build_before_call(self): - a = keras.Input(shape=(32,), name='input_a', dtype='float32') - dense_layer = keras.layers.Dense(16, dtype='float16') - dense_layer.build((32,)) - b = dense_layer(a) - - self.assertEqual(dense_layer.dtype, 'float16') - self.assertEqual(dense_layer.input, a) - self.assertEqual(dense_layer.output, b) - self.assertEqual(a.dtype, dtypes.float32) - self.assertEqual(dense_layer.kernel.dtype.base_dtype, dtypes.float16) - self.assertEqual(dense_layer.bias.dtype.base_dtype, dtypes.float16) - self.assertEqual(b.dtype, dtypes.float16) - - @test_util.run_in_graph_and_eager_modes() - def test_casting_in_network(self): - - class SingleInputLayer(keras.layers.Layer): - - def call(self, a): - self.a = a - return a - - def compute_output_shape(self, input_shape): - return input_shape - - class MultiInputLayer(keras.layers.Layer): - - def call(self, inputs): - a, b = inputs - self.a = a - self.b = b - return a + b - - def compute_output_shape(self, input_shapes): - return input_shapes[0] - - default_layer = SingleInputLayer() - fp32_layer = SingleInputLayer(dtype='float32') - fp16_layer = MultiInputLayer(dtype='float16') - - input_t = keras.layers.Input((32,), dtype='float64') - o1 = default_layer(input_t) - o2 = fp32_layer(o1) - # fp16_layer has inputs of different dtypes. - output_t = fp16_layer((o1, o2)) - network = keras.engine.Network(input_t, output_t) - - x = array_ops.ones((32,), dtype='float16') - y = network(x) - self.assertEqual(default_layer.dtype, dtypes.float64) - self.assertEqual(default_layer.a.dtype, dtypes.float64) - - self.assertEqual(fp32_layer.dtype, dtypes.float32) - self.assertEqual(fp32_layer.a.dtype, dtypes.float32) - - self.assertEqual(fp16_layer.dtype, dtypes.float16) - self.assertEqual(fp16_layer.a.dtype, dtypes.float16) - self.assertEqual(fp16_layer.b.dtype, dtypes.float16) - - self.assertEqual(y.dtype, dtypes.float16) - class DeferredModeTest(test.TestCase): diff --git a/tensorflow/python/layers/base.py b/tensorflow/python/layers/base.py index abbe9d0c56..b8969a41ab 100644 --- a/tensorflow/python/layers/base.py +++ b/tensorflow/python/layers/base.py @@ -43,15 +43,13 @@ class Layer(base_layer.Layer): Arguments: trainable: Boolean, whether the layer's variables should be trainable. name: String name of the layer. - dtype: Default dtype of the layer's weights and computations (default of - `None` means use the type of the first input). If not None, inputs will be - casted to this dtype. + dtype: Default dtype of the layer's weights (default of `None` means use the + type of the first input). Read-only properties: name: The name of the layer (string). - dtype: Default dtype of the layer's weights and computations. (default of - `None` means use the type of the first input). If not None, inputs will be - casted to this dtype. + dtype: Default dtype of the layer's weights (default of `None` means use the + type of the first input). trainable_variables: List of trainable variables. non_trainable_variables: List of non-trainable variables. variables: List of all variables of this layer, trainable and diff --git a/tensorflow/python/layers/base_test.py b/tensorflow/python/layers/base_test.py index ad44328aab..fcacc8d603 100644 --- a/tensorflow/python/layers/base_test.py +++ b/tensorflow/python/layers/base_test.py @@ -25,8 +25,6 @@ from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.framework import test_util -from tensorflow.python.keras import backend -from tensorflow.python.keras.engine import base_layer as keras_base_layer from tensorflow.python.layers import base as base_layers from tensorflow.python.layers import core as core_layers from tensorflow.python.ops import array_ops @@ -591,65 +589,5 @@ class BaseLayerTest(test.TestCase): ValueError, 'Input graph and Layer graph are not the same'): layer.apply(constant_op.constant([[1.]])) - @test_util.run_in_graph_and_eager_modes() - def testOnlyCastInputsWhenDtypeSpecified(self): - - class MyKerasLayer(keras_base_layer.Layer): - - def call(self, inputs): - self.x = inputs[0] - self.y = inputs[1] - return self.x + 1, self.y + 2 - - # Inherit from both the Keras Layer and base_layers.Layer to ensure we - # still get the base_layers.Layer behavior when directly inheriting from - # the Keras Layer. - class MyTFLayer(MyKerasLayer, base_layers.Layer): - pass - - # Test inputs are casted. - input1 = array_ops.constant(1.0, dtype=dtypes.float64) - input2 = array_ops.constant(1.0, dtype=dtypes.float32) - layer = MyTFLayer(dtype=dtypes.float16) - output1, output2 = layer([input1, input2]) - self.assertEqual(output1.dtype, dtypes.float16) - self.assertEqual(output2.dtype, dtypes.float16) - - # Test inputs are not casted. - input1 = array_ops.constant(1.0, dtype=dtypes.float64) - input2 = array_ops.constant(1.0, dtype=dtypes.float32) - layer = MyTFLayer() - output1, output2 = layer([input1, input2]) - self.assertEqual(output1.dtype, dtypes.float64) - self.assertEqual(output2.dtype, dtypes.float32) - - @test_util.run_in_graph_and_eager_modes() - def testVariablesDefaultToFloat32(self): - - class MyKerasLayer(keras_base_layer.Layer): - - def build(self, input_shape): - self.x = self.add_weight('x', ()) - - def call(self, inputs): - return inputs + self.x - - # Inherit from both the Keras Layer and base_layers.Layer to ensure we - # still get the base_layers.Layer behavior when directly inheriting from - # the Keras Layer. - class MyTFLayer(MyKerasLayer, base_layers.Layer): - pass - - try: - # The behavior of Keras Layers is to default to floatx. Ensure that this - # behavior is overridden to instead default to float32. - backend.set_floatx('float16') - layer = MyTFLayer() - layer.build(()) - self.assertEqual(layer.dtype, None) - self.assertEqual(layer.x.dtype.base_dtype, dtypes.float32) - finally: - backend.set_floatx('float32') - if __name__ == '__main__': test.main() |