diff options
Diffstat (limited to 'tensorflow/python/data/kernel_tests')
7 files changed, 386 insertions, 11 deletions
diff --git a/tensorflow/python/data/kernel_tests/BUILD b/tensorflow/python/data/kernel_tests/BUILD index 3bde62fa1d..23c98247bf 100644 --- a/tensorflow/python/data/kernel_tests/BUILD +++ b/tensorflow/python/data/kernel_tests/BUILD @@ -318,7 +318,7 @@ tf_py_test( ], ) -tf_py_test( +cuda_py_test( name = "iterator_ops_test", size = "small", srcs = ["iterator_ops_test.py"], @@ -329,6 +329,8 @@ tf_py_test( "//tensorflow/python/data/ops:dataset_ops", "//tensorflow/python/data/ops:iterator_ops", "//tensorflow/python/data/util:sparse", + "//tensorflow/python/eager:context", + "//tensorflow/python/training/checkpointable:util", "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", "//tensorflow/python:constant_op", @@ -349,6 +351,9 @@ tf_py_test( "//tensorflow/python:sparse_tensor", "//tensorflow/python:tensor_shape", "//tensorflow/python:training", + "//tensorflow/python/compat:compat", + "//tensorflow/python:util", + "//tensorflow/python:variables", ], grpc_enabled = True, ) @@ -380,3 +385,22 @@ tf_py_test( "no_windows", ], ) + +cuda_py_test( + name = "optional_ops_test", + size = "small", + srcs = ["optional_ops_test.py"], + additional_deps = [ + "//tensorflow/python/data/ops:dataset_ops", + "//tensorflow/python/data/ops:iterator_ops", + "//tensorflow/python/data/ops:optional_ops", + "//tensorflow/python:client_testlib", + "//tensorflow/python:array_ops", + "//tensorflow/python:constant_op", + "//tensorflow/python:dtypes", + "//tensorflow/python:errors", + "//tensorflow/python:framework_ops", + "//tensorflow/python:framework_test_lib", + "//tensorflow/python:tensor_shape", + ], +) diff --git a/tensorflow/python/data/kernel_tests/batch_dataset_op_test.py b/tensorflow/python/data/kernel_tests/batch_dataset_op_test.py index c3d42b49af..89de55dd4f 100644 --- a/tensorflow/python/data/kernel_tests/batch_dataset_op_test.py +++ b/tensorflow/python/data/kernel_tests/batch_dataset_op_test.py @@ -278,7 +278,7 @@ class PaddedBatchDatasetTest(test.TestCase, parameterized.TestCase): result = sess.run(get_next) padded_len = padded_shapes[0] if padded_len is None or padded_len == -1: - padded_len = np.max(result) + padded_len = np.max(result) if result.size > 0 else 0 self.assertEqual((batch_size, padded_len), result.shape) for j in range(batch_size): seq_len = seq_lens[(i * batch_size) + j] @@ -288,7 +288,7 @@ class PaddedBatchDatasetTest(test.TestCase, parameterized.TestCase): if not drop_remainder and len(seq_lens) % batch_size > 0: result = sess.run(get_next) - padded_len = np.max(result) + padded_len = np.max(result) if result.size > 0 else 0 self.assertEqual((len(seq_lens) % batch_size, padded_len), result.shape) for j in range(len(seq_lens) % batch_size): diff --git a/tensorflow/python/data/kernel_tests/cache_dataset_op_test.py b/tensorflow/python/data/kernel_tests/cache_dataset_op_test.py index 25269dc810..4f7fd3566e 100644 --- a/tensorflow/python/data/kernel_tests/cache_dataset_op_test.py +++ b/tensorflow/python/data/kernel_tests/cache_dataset_op_test.py @@ -34,7 +34,7 @@ from tensorflow.python.ops import variables from tensorflow.python.platform import test -class FilesystemCacheDatasetTest(test.TestCase): +class FileCacheDatasetTest(test.TestCase): def setUp(self): self.tmp_dir = tempfile.mkdtemp() diff --git a/tensorflow/python/data/kernel_tests/iterator_ops_test.py b/tensorflow/python/data/kernel_tests/iterator_ops_test.py index 820c167b6b..352424514e 100644 --- a/tensorflow/python/data/kernel_tests/iterator_ops_test.py +++ b/tensorflow/python/data/kernel_tests/iterator_ops_test.py @@ -17,6 +17,7 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +import functools import os import warnings @@ -25,6 +26,7 @@ import numpy as np from tensorflow.core.protobuf import cluster_pb2 from tensorflow.core.protobuf import config_pb2 from tensorflow.python.client import session +from tensorflow.python.compat import compat as forward_compat from tensorflow.python.data.ops import dataset_ops from tensorflow.python.data.ops import iterator_ops from tensorflow.python.data.ops import readers @@ -45,7 +47,9 @@ from tensorflow.python.ops import parsing_ops from tensorflow.python.ops import script_ops from tensorflow.python.ops import variables from tensorflow.python.platform import test +from tensorflow.python.training import checkpoint_management from tensorflow.python.training import server_lib +from tensorflow.python.training.checkpointable import util as checkpointable_utils from tensorflow.python.util import compat @@ -415,6 +419,69 @@ class IteratorTest(test.TestCase): sess.run( next_element, feed_dict={handle_placeholder: iterator_4_handle}) + def testIteratorStringHandleFuture(self): + with forward_compat.forward_compatibility_horizon(2018, 8, 4): + dataset_3 = dataset_ops.Dataset.from_tensor_slices([1, 2, 3]) + dataset_4 = dataset_ops.Dataset.from_tensor_slices([10, 20, 30, 40]) + + iterator_3 = dataset_3.make_one_shot_iterator() + iterator_4 = dataset_4.make_one_shot_iterator() + + handle_placeholder = array_ops.placeholder(dtypes.string, shape=[]) + feedable_iterator = iterator_ops.Iterator.from_string_handle( + handle_placeholder, dataset_3.output_types, dataset_3.output_shapes) + next_element = feedable_iterator.get_next() + + self.assertEqual(dataset_3.output_types, feedable_iterator.output_types) + self.assertEqual(dataset_4.output_types, feedable_iterator.output_types) + self.assertEqual([], feedable_iterator.output_shapes) + + with self.test_session() as sess: + iterator_3_handle = sess.run(iterator_3.string_handle()) + iterator_4_handle = sess.run(iterator_4.string_handle()) + + self.assertEqual( + 10, + sess.run( + next_element, + feed_dict={handle_placeholder: iterator_4_handle})) + self.assertEqual( + 1, + sess.run( + next_element, + feed_dict={handle_placeholder: iterator_3_handle})) + self.assertEqual( + 20, + sess.run( + next_element, + feed_dict={handle_placeholder: iterator_4_handle})) + self.assertEqual( + 2, + sess.run( + next_element, + feed_dict={handle_placeholder: iterator_3_handle})) + self.assertEqual( + 30, + sess.run( + next_element, + feed_dict={handle_placeholder: iterator_4_handle})) + self.assertEqual( + 3, + sess.run( + next_element, + feed_dict={handle_placeholder: iterator_3_handle})) + self.assertEqual( + 40, + sess.run( + next_element, + feed_dict={handle_placeholder: iterator_4_handle})) + with self.assertRaises(errors.OutOfRangeError): + sess.run( + next_element, feed_dict={handle_placeholder: iterator_3_handle}) + with self.assertRaises(errors.OutOfRangeError): + sess.run( + next_element, feed_dict={handle_placeholder: iterator_4_handle}) + def testIteratorStringHandleReuseTensorObject(self): dataset = dataset_ops.Dataset.from_tensor_slices([1, 2, 3]) one_shot_iterator = dataset.make_one_shot_iterator() @@ -724,5 +791,98 @@ class IteratorTest(test.TestCase): val += 1 +class IteratorCheckpointingTest(test.TestCase): + + @test_util.run_in_graph_and_eager_modes + def testSaveRestoreOneShotIterator(self): + checkpoint_directory = self.get_temp_dir() + checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt") + dataset = dataset_ops.Dataset.from_tensor_slices([1, 2, 3, 4, 5, 6]).map( + math_ops.square).batch(2) + iterator = dataset.make_one_shot_iterator() + get_next = iterator.get_next if context.executing_eagerly( + ) else functools.partial(self.evaluate, iterator.get_next()) + checkpoint = checkpointable_utils.Checkpoint(iterator=iterator) + with self.test_session() as sess: + self.assertAllEqual([1, 4], get_next()) + save_path = checkpoint.save(checkpoint_prefix) + self.assertAllEqual([9, 16], get_next()) + self.assertAllEqual([25, 36], get_next()) + checkpoint.restore(save_path).run_restore_ops(sess) + self.assertAllEqual([9, 16], get_next()) + self.assertAllEqual([25, 36], get_next()) + with self.assertRaises(errors.OutOfRangeError): + get_next() + + @test_util.run_in_graph_and_eager_modes + def testSaveRestoreMultipleIterator(self): + checkpoint_directory = self.get_temp_dir() + checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt") + dataset = dataset_ops.Dataset.from_tensor_slices( + [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]) + dataset = dataset.map(math_ops.square).batch(2) + iterator_1 = dataset.make_one_shot_iterator() + get_next_1 = iterator_1.get_next if context.executing_eagerly( + ) else functools.partial(self.evaluate, iterator_1.get_next()) + iterator_2 = dataset.make_one_shot_iterator() + get_next_2 = iterator_2.get_next if context.executing_eagerly( + ) else functools.partial(self.evaluate, iterator_2.get_next()) + dataset_2 = dataset_ops.Dataset.range(10) + iterator_3 = dataset_2.make_one_shot_iterator() + get_next_3 = iterator_3.get_next if context.executing_eagerly( + ) else functools.partial(self.evaluate, iterator_3.get_next()) + checkpoint = checkpointable_utils.Checkpoint( + iterator_1=iterator_1, iterator_2=iterator_2, iterator_3=iterator_3) + with self.test_session() as sess: + self.assertAllEqual([1, 4], get_next_1()) + self.assertAllEqual(0, get_next_3()) + self.assertAllEqual(1, get_next_3()) + self.assertAllEqual(2, get_next_3()) + save_path = checkpoint.save(checkpoint_prefix) + self.assertAllEqual([1, 4], get_next_2()) + self.assertAllEqual([9, 16], get_next_2()) + self.assertAllEqual(3, get_next_3()) + checkpoint.restore(save_path).run_restore_ops(sess) + self.assertAllEqual([9, 16], get_next_1()) + self.assertAllEqual([1, 4], get_next_2()) + self.assertAllEqual(3, get_next_3()) + + @test_util.run_in_graph_and_eager_modes + def testRestoreExhaustedIterator(self): + checkpoint_directory = self.get_temp_dir() + checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt") + dataset = dataset_ops.Dataset.range(3) + iterator = dataset.make_one_shot_iterator() + get_next = iterator.get_next if context.executing_eagerly( + ) else functools.partial(self.evaluate, iterator.get_next()) + checkpoint = checkpointable_utils.Checkpoint(iterator=iterator) + with self.test_session() as sess: + self.assertAllEqual(0, get_next()) + self.assertAllEqual(1, get_next()) + save_path = checkpoint.save(checkpoint_prefix) + self.assertAllEqual(2, get_next()) + checkpoint.restore(save_path).run_restore_ops(sess) + self.assertAllEqual(2, get_next()) + save_path = checkpoint.save(checkpoint_prefix) + checkpoint.restore(save_path).run_restore_ops(sess) + with self.assertRaises(errors.OutOfRangeError): + get_next() + + def testRestoreInReconstructedIteratorInitializable(self): + checkpoint_directory = self.get_temp_dir() + checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt") + dataset = dataset_ops.Dataset.range(10) + iterator = dataset.make_initializable_iterator() + get_next = iterator.get_next() + checkpoint = checkpointable_utils.Checkpoint(iterator=iterator) + for i in range(5): + with self.test_session() as sess: + checkpoint.restore(checkpoint_management.latest_checkpoint( + checkpoint_directory)).initialize_or_restore(sess) + for j in range(2): + self.assertEqual(i * 2 + j, sess.run(get_next)) + checkpoint.save(file_prefix=checkpoint_prefix) + + if __name__ == "__main__": test.main() diff --git a/tensorflow/python/data/kernel_tests/list_files_dataset_op_test.py b/tensorflow/python/data/kernel_tests/list_files_dataset_op_test.py index f7d7d085c9..579096f880 100644 --- a/tensorflow/python/data/kernel_tests/list_files_dataset_op_test.py +++ b/tensorflow/python/data/kernel_tests/list_files_dataset_op_test.py @@ -123,13 +123,11 @@ class ListFilesDatasetOpTest(test.TestCase): with self.test_session() as sess: itr = dataset.make_initializable_iterator() - next_element = itr.get_next() - sess.run( - itr.initializer, - feed_dict={filename_placeholder: path.join(self.tmp_dir, '*')}) - - with self.assertRaises(errors.OutOfRangeError): - sess.run(next_element) + with self.assertRaisesRegexp( + errors.InvalidArgumentError, 'No files matched pattern: '): + sess.run( + itr.initializer, + feed_dict={filename_placeholder: path.join(self.tmp_dir, '*')}) def testSimpleDirectoryInitializer(self): filenames = ['a', 'b', 'c'] diff --git a/tensorflow/python/data/kernel_tests/map_dataset_op_test.py b/tensorflow/python/data/kernel_tests/map_dataset_op_test.py index 0ecd821e9e..637bde9ae4 100644 --- a/tensorflow/python/data/kernel_tests/map_dataset_op_test.py +++ b/tensorflow/python/data/kernel_tests/map_dataset_op_test.py @@ -666,6 +666,13 @@ class MapDatasetTest(test.TestCase): "currently support nested datasets as outputs."): _ = dataset.map(dataset_ops.Dataset.from_tensor_slices) + def testReturnValueError(self): + dataset = dataset_ops.Dataset.from_tensors([1.0, 2.0, 3.0]) + with self.assertRaisesRegexp( + TypeError, r"Unsupported return value from function passed to " + r"Dataset.map\(\): None."): + _ = dataset.map(lambda x: None) + class MapDatasetBenchmark(test.Benchmark): diff --git a/tensorflow/python/data/kernel_tests/optional_ops_test.py b/tensorflow/python/data/kernel_tests/optional_ops_test.py new file mode 100644 index 0000000000..a32527af8d --- /dev/null +++ b/tensorflow/python/data/kernel_tests/optional_ops_test.py @@ -0,0 +1,186 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for the Optional data type wrapper.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np + +from tensorflow.python.data.ops import dataset_ops +from tensorflow.python.data.ops import iterator_ops +from tensorflow.python.data.ops import optional_ops +from tensorflow.python.framework import constant_op +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import errors +from tensorflow.python.framework import ops +from tensorflow.python.framework import sparse_tensor +from tensorflow.python.framework import tensor_shape +from tensorflow.python.framework import test_util +from tensorflow.python.ops import array_ops +from tensorflow.python.platform import test + + +class OptionalTest(test.TestCase): + + @test_util.run_in_graph_and_eager_modes + def testFromValue(self): + opt = optional_ops.Optional.from_value(constant_op.constant(37.0)) + self.assertEqual(dtypes.float32, opt.output_types) + self.assertEqual([], opt.output_shapes) + self.assertEqual(ops.Tensor, opt.output_classes) + self.assertTrue(self.evaluate(opt.has_value())) + self.assertEqual(37.0, self.evaluate(opt.get_value())) + + @test_util.run_in_graph_and_eager_modes + def testFromStructuredValue(self): + opt = optional_ops.Optional.from_value({ + "a": constant_op.constant(37.0), + "b": (constant_op.constant(["Foo"]), constant_op.constant("Bar")) + }) + self.assertEqual({ + "a": dtypes.float32, + "b": (dtypes.string, dtypes.string) + }, opt.output_types) + self.assertEqual({"a": [], "b": ([1], [])}, opt.output_shapes) + self.assertEqual({ + "a": ops.Tensor, + "b": (ops.Tensor, ops.Tensor) + }, opt.output_classes) + self.assertTrue(self.evaluate(opt.has_value())) + self.assertEqual({ + "a": 37.0, + "b": ([b"Foo"], b"Bar") + }, self.evaluate(opt.get_value())) + + @test_util.run_in_graph_and_eager_modes + def testFromSparseTensor(self): + st_0 = sparse_tensor.SparseTensorValue( + indices=np.array([[0]]), + values=np.array([0], dtype=np.int64), + dense_shape=np.array([1])) + st_1 = sparse_tensor.SparseTensorValue( + indices=np.array([[0, 0], [1, 1]]), + values=np.array([-1., 1.], dtype=np.float32), + dense_shape=np.array([2, 2])) + opt = optional_ops.Optional.from_value((st_0, st_1)) + self.assertEqual((dtypes.int64, dtypes.float32), opt.output_types) + self.assertEqual(([1], [2, 2]), opt.output_shapes) + self.assertEqual((sparse_tensor.SparseTensor, sparse_tensor.SparseTensor), + opt.output_classes) + + @test_util.run_in_graph_and_eager_modes + def testFromNone(self): + opt = optional_ops.Optional.none_from_structure(tensor_shape.scalar(), + dtypes.float32, ops.Tensor) + self.assertEqual(dtypes.float32, opt.output_types) + self.assertEqual([], opt.output_shapes) + self.assertEqual(ops.Tensor, opt.output_classes) + self.assertFalse(self.evaluate(opt.has_value())) + with self.assertRaises(errors.InvalidArgumentError): + self.evaluate(opt.get_value()) + + def testStructureMismatchError(self): + tuple_output_shapes = (tensor_shape.scalar(), tensor_shape.scalar()) + tuple_output_types = (dtypes.float32, dtypes.float32) + tuple_output_classes = (ops.Tensor, ops.Tensor) + + dict_output_shapes = { + "a": tensor_shape.scalar(), + "b": tensor_shape.scalar() + } + dict_output_types = {"a": dtypes.float32, "b": dtypes.float32} + dict_output_classes = {"a": ops.Tensor, "b": ops.Tensor} + + with self.assertRaises(TypeError): + optional_ops.Optional.none_from_structure( + tuple_output_shapes, tuple_output_types, dict_output_classes) + + with self.assertRaises(TypeError): + optional_ops.Optional.none_from_structure( + tuple_output_shapes, dict_output_types, tuple_output_classes) + + with self.assertRaises(TypeError): + optional_ops.Optional.none_from_structure( + dict_output_shapes, tuple_output_types, tuple_output_classes) + + @test_util.run_in_graph_and_eager_modes + def testCopyToGPU(self): + if not test_util.is_gpu_available(): + self.skipTest("No GPU available") + + with ops.device("/cpu:0"): + optional_with_value = optional_ops.Optional.from_value( + (constant_op.constant(37.0), constant_op.constant("Foo"), + constant_op.constant(42))) + optional_none = optional_ops.Optional.none_from_structure( + tensor_shape.scalar(), dtypes.float32, ops.Tensor) + + with ops.device("/gpu:0"): + gpu_optional_with_value = optional_ops._OptionalImpl( + array_ops.identity(optional_with_value._variant_tensor), + optional_with_value.output_shapes, optional_with_value.output_types, + optional_with_value.output_classes) + gpu_optional_none = optional_ops._OptionalImpl( + array_ops.identity(optional_none._variant_tensor), + optional_none.output_shapes, optional_none.output_types, + optional_none.output_classes) + + gpu_optional_with_value_has_value = gpu_optional_with_value.has_value() + gpu_optional_with_value_values = gpu_optional_with_value.get_value() + + gpu_optional_none_has_value = gpu_optional_none.has_value() + + self.assertTrue(self.evaluate(gpu_optional_with_value_has_value)) + self.assertEqual((37.0, b"Foo", 42), + self.evaluate(gpu_optional_with_value_values)) + self.assertFalse(self.evaluate(gpu_optional_none_has_value)) + + def testIteratorGetNextAsOptional(self): + ds = dataset_ops.Dataset.range(3) + iterator = ds.make_initializable_iterator() + next_elem = iterator_ops.get_next_as_optional(iterator) + self.assertTrue(isinstance(next_elem, optional_ops.Optional)) + self.assertEqual(ds.output_types, next_elem.output_types) + self.assertEqual(ds.output_shapes, next_elem.output_shapes) + self.assertEqual(ds.output_classes, next_elem.output_classes) + elem_has_value_t = next_elem.has_value() + elem_value_t = next_elem.get_value() + with self.test_session() as sess: + # Before initializing the iterator, evaluating the optional fails with + # a FailedPreconditionError. + with self.assertRaises(errors.FailedPreconditionError): + sess.run(elem_has_value_t) + with self.assertRaises(errors.FailedPreconditionError): + sess.run(elem_value_t) + + # For each element of the dataset, assert that the optional evaluates to + # the expected value. + sess.run(iterator.initializer) + for i in range(3): + elem_has_value, elem_value = sess.run([elem_has_value_t, elem_value_t]) + self.assertTrue(elem_has_value) + self.assertEqual(i, elem_value) + + # After exhausting the iterator, `next_elem.has_value()` will evaluate to + # false, and attempting to get the value will fail. + for _ in range(2): + self.assertFalse(sess.run(elem_has_value_t)) + with self.assertRaises(errors.InvalidArgumentError): + sess.run(elem_value_t) + + +if __name__ == "__main__": + test.main() |