aboutsummaryrefslogtreecommitdiffhomepage
path: root/tensorflow/contrib/data
diff options
context:
space:
mode:
authorGravatar Derek Murray <mrry@google.com>2018-09-04 14:01:13 -0700
committerGravatar TensorFlower Gardener <gardener@tensorflow.org>2018-09-04 14:17:53 -0700
commit06e8109af2e5ae5bc149e25fc64fbf66d6c8b817 (patch)
treeb13b214063b3a4ba8f15c26e9170c9f44c49b854 /tensorflow/contrib/data
parent8ef276fd2181fb71c2e232f60aa45ee96cb5905b (diff)
[tf.data] Add internal optimizations for executing simple functions in `MapDataset`.
PiperOrigin-RevId: 211520001
Diffstat (limited to 'tensorflow/contrib/data')
-rw-r--r--tensorflow/contrib/data/python/ops/interleave_ops.py17
-rw-r--r--tensorflow/contrib/data/python/ops/readers.py6
2 files changed, 16 insertions, 7 deletions
diff --git a/tensorflow/contrib/data/python/ops/interleave_ops.py b/tensorflow/contrib/data/python/ops/interleave_ops.py
index 38c0a09c33..92d4251a86 100644
--- a/tensorflow/contrib/data/python/ops/interleave_ops.py
+++ b/tensorflow/contrib/data/python/ops/interleave_ops.py
@@ -220,6 +220,7 @@ def sample_from_datasets(datasets, weights=None, seed=None):
if weights is None:
# Select inputs with uniform probability.
logits = [[1.0] * num_datasets]
+
else:
# Use the given `weights` as the probability of choosing the respective
# input.
@@ -245,8 +246,11 @@ def sample_from_datasets(datasets, weights=None, seed=None):
return array_ops.squeeze(
stateless.stateless_multinomial(logits, 1, seed=seed), axis=[0, 1])
- selector_input = random_ops.RandomDataset(seed).batch(2).map(
- select_dataset_constant_logits)
+ selector_input = dataset_ops.MapDataset(
+ random_ops.RandomDataset(seed).batch(2),
+ select_dataset_constant_logits,
+ use_inter_op_parallelism=False)
+
else:
# Use each element of the given `weights` dataset as the probability of
# choosing the respective input.
@@ -259,9 +263,12 @@ def sample_from_datasets(datasets, weights=None, seed=None):
return array_ops.squeeze(
stateless.stateless_multinomial(logits, 1, seed=seed), axis=[0, 1])
- selector_input = dataset_ops.Dataset.zip(
- (logits_ds, random_ops.RandomDataset(seed).batch(2)
- )).map(select_dataset_varying_logits)
+ logits_and_seeds = dataset_ops.Dataset.zip(
+ (logits_ds, random_ops.RandomDataset(seed).batch(2)))
+ selector_input = dataset_ops.MapDataset(
+ logits_and_seeds,
+ select_dataset_varying_logits,
+ use_inter_op_parallelism=False)
return _DirectedInterleaveDataset(selector_input, datasets)
diff --git a/tensorflow/contrib/data/python/ops/readers.py b/tensorflow/contrib/data/python/ops/readers.py
index 7f09ba71dc..4c466781f7 100644
--- a/tensorflow/contrib/data/python/ops/readers.py
+++ b/tensorflow/contrib/data/python/ops/readers.py
@@ -499,7 +499,8 @@ def make_csv_dataset(
# indefinitely, and all batches will be full-sized.
dataset = dataset.batch(batch_size=batch_size,
drop_remainder=num_epochs is None)
- dataset = dataset.map(map_fn)
+ dataset = dataset_ops.MapDataset(
+ dataset, map_fn, use_inter_op_parallelism=False)
dataset = dataset.prefetch(prefetch_buffer_size)
return dataset
@@ -778,7 +779,8 @@ def make_batched_features_dataset(file_pattern,
# Extract values if the `Example` tensors are stored as key-value tuples.
if dataset.output_types == (dtypes.string, dtypes.string):
- dataset = dataset.map(lambda _, v: v)
+ dataset = dataset_ops.MapDataset(
+ dataset, lambda _, v: v, use_inter_op_parallelism=False)
# Apply dataset repeat and shuffle transformations.
dataset = _maybe_shuffle_and_repeat(