diff options
61 files changed, 774 insertions, 220 deletions
diff --git a/CODEOWNERS b/CODEOWNERS new file mode 100644 index 0000000000..2f216e4161 --- /dev/null +++ b/CODEOWNERS @@ -0,0 +1,50 @@ +# Where component owners are known, add them here. + +tensorflow/tensorboard/* @jart @dandelionmane +tensorflow/tools/docs/* @markdaoust +tensorflow/java/* @asimshankar + +# contrib + +# NEED OWNER: tensorflow/contrib/avro/* +tensorflow/contrib/batching/* @alextp @chrisolston +tensorflow/contrib/bayesflow/* @ebrevdo @rsepassi @jvdillon +tensorflow/contrib/cmake/* @mrry @benoitsteiner +tensorflow/contrib/copy_graph/* @tucker @poxvoculi +tensorflow/contrib/crf/* @kentonl +tensorflow/contrib/data/* @mrry +tensorflow/contrib/distributions/* @jvdillon @langmore @rsepassi +tensorflow/contrib/factorization/* @agarwal-ashish @xavigonzalvo +tensorflow/contrib/ffmpeg/* @fredbertsch +# NEED OWNERT: tensorflow/contrib/framework/* +tensorflow/contrib/graph_editor/* @purpledog +# NEED OWNER: tensorflow/contrib/grid_rnn/* +tensorflow/contrib/hvx/* @satok16 +tensorflow/contrib/imperative/* @keveman +tensorflow/contrib/integrate/* @shoyer +tensorflow/contrib/kernel_methods/* @petrosmol +tensorflow/contrib/ios_examples/* @petewarden +tensorflow/contrib/labeled_tensor/* @shoyer +tensorflow/contrib/layers/* @fchollet @martinwicke +tensorflow/contrib/learn/* @martinwicke @ispirmustafa @alextp +tensorflow/contrib/linalg/* @langmore +tensorflow/contrib/linear_optimizer/* @petrosmol @andreasst @katsiapis +tensorflow/contrib/lookup/* @ysuematsu @andreasst +tensorflow/contrib/losses/* @alextp @ispirmustafa +tensorflow/contrib/makefile/* @petewarden @satok16 @wolffg +tensorflow/contrib/metrics/* @alextp @honkentuber @ispirmustafa +tensorflow/contrib/nccl/* @cwhipkey @zheng-xq +tensorflow/contrib/opt/* @strategist333 +tensorflow/contrib/pi_examples/* @maciekcc +tensorflow/contrib/quantization/* @petewarden @cwhipkey @keveman +tensorflow/contrib/rnn/* @ebrevdo +tensorflow/contrib/saved_model/* @nfiedel @sukritiramesh +tensorflow/contrib/seq2seq/* @lukaszkaiser +tensorflow/contrib/session_bundle/* @nfiedel @sukritiramesh +tensorflow/contrib/slim/* @sguada @thenbasilmanran +tensorflow/contrib/stateless/* @girving +tensorflow/contrib/tensor_forest/* @gilberthendry @thomascolthurst +tensorflow/contrib/testing/* @dandelionmane +tensorflow/contrib/timeseries/* @allenlavoie +tensorflow/contrib/training/* @joel-shor @ebrevdo +tensorflow/contrib/util/* @sherrym @@ -62,7 +62,7 @@ $ python ## For more information -* [TensorFlow website](https://tensorflow.org) +* [TensorFlow website](https://www.tensorflow.org) * [TensorFlow whitepaper](http://download.tensorflow.org/paper/whitepaper2015.pdf) * [TensorFlow Model Zoo](https://github.com/tensorflow/models) * [TensorFlow MOOC on Udacity](https://www.udacity.com/course/deep-learning--ud730) diff --git a/RELEASE.md b/RELEASE.md index 0cd4eef5d6..33fe88cd54 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -65,37 +65,6 @@ integration into apps. See https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/android/README.md for more details. -* RNNCells' variable names have been renamed for consistency with Keras layers. - Specifically, the previous variable names "weights" and "biases" have - been changed to "kernel" and "bias", respectively. - This may cause backward incompatibility with regard to your old - checkpoints containing such RNN cells, in which case you can use the tool - [checkpoint_convert script](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/rnn/python/tools/checkpoint_convert.py) - to convert the variable names in your old checkpoints. -* Many of the RNN functions and classes that were in the `tf.nn` namespace - before the 1.0 release and which were moved to `tf.contrib.rnn` have now - been moved back to the core namespace. This includes - `RNNCell`, `LSTMCell`, `GRUCell`, and a number of other cells. These - now reside in `tf.nn.rnn_cell` (with aliases in `tf.contrib.rnn` for backwards - compatibility). The original `tf.nn.rnn` function is now `tf.nn.static_rnn`, - and the bidirectional static and state saving static rnn functions are also - now back in the `tf.nn` namespace. - - Notable exceptions are the `EmbeddingWrapper`, `InputProjectionWrapper` and - `OutputProjectionWrapper`, which will slowly be moved to deprecation - in `tf.contrib.rnn`. These are inefficient wrappers that should often - be replaced by calling `embedding_lookup` or `layers.dense` as pre- or post- - processing of the rnn. For RNN decoding, this functionality has been replaced - with an alternative API in `tf.contrib.seq2seq`. -* Intel MKL Integration (https://software.intel.com/en-us/articles/tensorflow-optimizations-on-modern-intel-architecture). Intel developed a number of - optimized deep learning primitives: In addition to matrix multiplication and - convolution, these building blocks include: - Direct batched convolution - Pooling: maximum, minimum, average - Normalization: LRN, batch normalization - Activation: rectified linear unit (ReLU) - Data manipulation: multi-dimensional transposition (conversion), split, - concat, sum and scale. ## Deprecations diff --git a/tensorflow/cc/BUILD b/tensorflow/cc/BUILD index b461a475c1..c65170dfe8 100644 --- a/tensorflow/cc/BUILD +++ b/tensorflow/cc/BUILD @@ -248,6 +248,7 @@ cc_library( ":gradients", "//tensorflow/core:lib_proto_parsing", ], + alwayslink = 1, ) tf_cc_test( @@ -275,6 +276,7 @@ cc_library( ":cc_ops", ":grad_op_registry", ], + alwayslink = 1, ) tf_cc_test( @@ -302,6 +304,7 @@ cc_library( ":cc_ops_internal", ":grad_op_registry", ], + alwayslink = 1, ) tf_cc_test( diff --git a/tensorflow/compiler/tests/tensor_array_ops_test.py b/tensorflow/compiler/tests/tensor_array_ops_test.py index b3067be51d..f277314352 100644 --- a/tensorflow/compiler/tests/tensor_array_ops_test.py +++ b/tensorflow/compiler/tests/tensor_array_ops_test.py @@ -139,7 +139,7 @@ class TensorArrayTest(xla_test.XLATestCase): ta = tensor_array_ops.TensorArray( dtype=tf_dtype, tensor_array_name="foo", size=3) - # Unpack a matrix into vectors + # Unpack a matrix into vectors. w1 = ta.unstack(convert([[1.0, 1.1], [2.0, 2.1], [3.0, 3.1]])) r0 = w1.read(0) r1 = w1.read(1) @@ -180,7 +180,7 @@ class TensorArrayTest(xla_test.XLATestCase): convert = _make_converter(tf_dtype) - # Split an empty vector + # Split an empty vector. lengths = constant_op.constant([0, 0, 0]) w0 = ta.split(convert([]), lengths=lengths) r0 = w0.read(0) @@ -192,7 +192,7 @@ class TensorArrayTest(xla_test.XLATestCase): self.assertAllEqual(convert([]), d1) self.assertAllEqual(convert([]), d2) - # Split a vector + # Split a vector. ta = tensor_array_ops.TensorArray( dtype=tf_dtype, tensor_array_name="foo", size=3) lengths = constant_op.constant([1, 1, 1]) @@ -206,7 +206,7 @@ class TensorArrayTest(xla_test.XLATestCase): self.assertAllEqual(convert([2.0]), d1) self.assertAllEqual(convert([3.0]), d2) - # Split a matrix + # Split a matrix. ta = tensor_array_ops.TensorArray( dtype=tf_dtype, tensor_array_name="foo", size=3) lengths = constant_op.constant([1, 1, 1]) @@ -319,27 +319,31 @@ class TensorArrayTest(xla_test.XLATestCase): ta = tensor_array_ops.TensorArray( dtype=dtypes.float32, tensor_array_name="foo", size=3) - # Test writing the wrong datatype + # Test writing the wrong datatype. with self.assertRaisesOpError( "TensorArray dtype is float but op has dtype int32"): ta.write(-1, np.int32(7)).flow.eval() def testTensorArrayReadWrongIndexOrDataTypeFails(self): - with self.test_session(), self.test_scope(): - ta = tensor_array_ops.TensorArray( - dtype=dtypes.float32, tensor_array_name="foo", size=3) - - w0 = ta.write(0, [[4.0, 5.0]]) - - # Test reading wrong datatype - r0_bad = gen_data_flow_ops._tensor_array_read_v3( - handle=w0.handle, index=0, dtype=dtypes.float64, flow_in=w0.flow) - with self.assertRaisesOpError( - "TensorArray dtype is float but op has dtype double."): - r0_bad.eval() - - # Test reading from a different index than the one we wrote to - w0.read(1) + # Find two different floating point types, create an array of + # the first type, but try to read the other type. + if len(self.float_types) > 1: + dtype1 = self.float_types[0] + dtype2 = self.float_types[1] + with self.test_session(), self.test_scope(): + ta = tensor_array_ops.TensorArray( + dtype=dtype1, tensor_array_name="foo", size=3) + + w0 = ta.write(0, [[4.0, 5.0]]) + + # Test reading wrong datatype. + r0_bad = gen_data_flow_ops._tensor_array_read_v3( + handle=w0.handle, index=0, dtype=dtype2, flow_in=w0.flow) + with self.assertRaisesOpError("TensorArray dtype is "): + r0_bad.eval() + + # Test reading from a different index than the one we wrote to + w0.read(1) def testTensorArraySplitIncompatibleShapesFails(self): with self.test_session(), self.test_scope(): @@ -487,7 +491,7 @@ class TensorArrayTest(xla_test.XLATestCase): r0 = w1.read(0) s0 = w1.concat() - # Test gradient accumulation between read(0), pack(), and concat() + # Test gradient accumulation between read(0), pack(), and concat(). with ops.control_dependencies([p0, r0, s0]): grad_r = gradients_impl.gradients( ys=[p0, r0, s0], @@ -536,7 +540,7 @@ class TensorArrayTest(xla_test.XLATestCase): r0_1 = w.read(0) r1 = w.read(1) - # Test combined gradients + aggregation of read(0) + # Test combined gradients + aggregation of read(0). grad = gradients_impl.gradients( ys=[r0, r0_1, r1], xs=[value], @@ -744,7 +748,7 @@ class TensorArrayTest(xla_test.XLATestCase): grad_b_t, = session.run([grad_b]) self.assertAllEqual(grad_b_t, g0) - # Test gradients calculated jointly + # Test gradients calculated jointly. joint_grad_a_t, joint_grad_b_t = session.run([grad_a, grad_b]) self.assertAllEqual(joint_grad_a_t, g0) self.assertAllEqual(joint_grad_b_t, g0) @@ -877,7 +881,7 @@ class TensorArrayTest(xla_test.XLATestCase): x = constant_op.constant([2.0, 3.0]) w = ta.unstack(x) r0 = w.read(0) - # calculate (dr0/dx0, dr0/dx1). since r0 = x0, gradients are (1, 0). + # Calculate (dr0/dx0, dr0/dx1). since r0 = x0, gradients are (1, 0). grad_r0 = gradients_impl.gradients(ys=[r0], xs=[x], grad_ys=[1.0]) grad_r0_vals = session.run(grad_r0)[0] self.assertAllEqual(grad_r0_vals, [1.0, 0.0]) @@ -927,7 +931,7 @@ class TensorArrayTest(xla_test.XLATestCase): r0 = w.read(1) r1 = w.read(8) - # Test combined gradients + aggregation of read(0) + # Test combined gradients + aggregation of read(0). grad = gradients_impl.gradients( ys=[r0, r1], xs=[value], grad_ys=[[2.0, 3.0], [4.0, 5.0]]) read_vals, grad_vals = session.run([[r0, r1], grad]) @@ -951,7 +955,7 @@ class TensorArrayTest(xla_test.XLATestCase): w = ta.unstack(values) g = w.gather(indices) - # Test combined gradients + aggregation of read(0) + # Test combined gradients + aggregation of read(0). grad = gradients_impl.gradients( ys=[g], xs=[values], grad_ys=[[[2.0, 3.0], [4.0, 5.0]]]) g_vals, grad_vals = session.run([[g], grad]) diff --git a/tensorflow/compiler/xla/util.cc b/tensorflow/compiler/xla/util.cc index d467178cb5..b58670ecc5 100644 --- a/tensorflow/compiler/xla/util.cc +++ b/tensorflow/compiler/xla/util.cc @@ -16,6 +16,7 @@ limitations under the License. #include "tensorflow/compiler/xla/util.h" #include <stdarg.h> +#include <numeric> #include "tensorflow/compiler/xla/legacy_flags/util_flags.h" #include "tensorflow/compiler/xla/types.h" diff --git a/tensorflow/contrib/copy_graph/python/util/copy_elements.py b/tensorflow/contrib/copy_graph/python/util/copy_elements.py index 3f124be771..8c2528f548 100644 --- a/tensorflow/contrib/copy_graph/python/util/copy_elements.py +++ b/tensorflow/contrib/copy_graph/python/util/copy_elements.py @@ -100,7 +100,9 @@ def copy_variable_to_graph(org_instance, to_graph, scope=""): def copy_op_to_graph(org_instance, to_graph, variables, scope=""): - """Given an `Operation` 'org_instance` from one `Graph`, + """Returns a copy of an operation from another Graph under a specified scope. + + Given an `Operation` `org_instance` from one `Graph`, initializes and returns a copy of it from another `Graph`, under the specified scope (default `""`). diff --git a/tensorflow/contrib/cudnn_rnn/python/ops/cudnn_rnn_ops.py b/tensorflow/contrib/cudnn_rnn/python/ops/cudnn_rnn_ops.py index 81ebc821a3..9307fbabac 100644 --- a/tensorflow/contrib/cudnn_rnn/python/ops/cudnn_rnn_ops.py +++ b/tensorflow/contrib/cudnn_rnn/python/ops/cudnn_rnn_ops.py @@ -197,7 +197,7 @@ class RNNParamsSaveable(saver.BaseSaverBuilder.SaveableObject): prefix = "multi_rnn_cell/cell_%d/cudnn_compatible_lstm_cell" % i w_names.append(prefix + "/kernel") # Three transformed bias tensors each layer: - # the 1st is for CudnnCompatbleLSTM(Block)Cell restore; the latter two + # the 1st is for CudnnCompatibleLSTM(Block)Cell restore; the latter two # sum up to the 1st, and are used for cuDNN restore. b_names.append(prefix + "/bias") b_names.extend([prefix + "/bias_cudnn_%d" % j for j in range(2)]) diff --git a/tensorflow/contrib/factorization/kernels/wals_solver_ops.cc b/tensorflow/contrib/factorization/kernels/wals_solver_ops.cc index cd1f23bba2..bb9b835889 100644 --- a/tensorflow/contrib/factorization/kernels/wals_solver_ops.cc +++ b/tensorflow/contrib/factorization/kernels/wals_solver_ops.cc @@ -213,7 +213,7 @@ class WALSComputePartialLhsAndRhsOp : public OpKernel { CHECK_LE(shard.second, perm.size()); CHECK_LE(shard.first, shard.second); const int64 input_index = get_input_index(perm[shard.first]); - // Acccumulate the rhs and lhs terms in the normal equations + // Accumulate the rhs and lhs terms in the normal equations // for the non-zero elements in the row or column of the sparse matrix // corresponding to input_index. int num_batched = 0; diff --git a/tensorflow/contrib/framework/python/ops/ops.py b/tensorflow/contrib/framework/python/ops/ops.py index 4fccc2ceac..ac451a974d 100644 --- a/tensorflow/contrib/framework/python/ops/ops.py +++ b/tensorflow/contrib/framework/python/ops/ops.py @@ -68,6 +68,6 @@ def get_name_scope(): would print the string `scope1/scope2`. Returns: - A string represnting the current name scope. + A string representing the current name scope. """ return ops.get_default_graph().get_name_scope() diff --git a/tensorflow/contrib/labeled_tensor/python/ops/core.py b/tensorflow/contrib/labeled_tensor/python/ops/core.py index 04bf26a5dd..fc1ea83449 100644 --- a/tensorflow/contrib/labeled_tensor/python/ops/core.py +++ b/tensorflow/contrib/labeled_tensor/python/ops/core.py @@ -199,7 +199,7 @@ class Axes(collections.Mapping): """Axis names and indices for a tensor. It is an ordered mapping, with keys given by axis name and values given - by Axis objets. Duplicate axis names are not allowed. + by Axis objects. Duplicate axis names are not allowed. """ @tc.accepts(object, tc.List(AxisLike)) diff --git a/tensorflow/contrib/layers/python/layers/initializers.py b/tensorflow/contrib/layers/python/layers/initializers.py index 271b3c01ff..b12a882d9a 100644 --- a/tensorflow/contrib/layers/python/layers/initializers.py +++ b/tensorflow/contrib/layers/python/layers/initializers.py @@ -42,7 +42,7 @@ def xavier_initializer(uniform=True, seed=None, dtype=dtypes.float32): This initializer is designed to keep the scale of the gradients roughly the same in all layers. In uniform distribution this ends up being the range: `x = sqrt(6. / (in + out)); [-x, x]` and for normal distribution a standard - deviation of `sqrt(3. / (in + out))` is used. + deviation of `sqrt(2. / (in + out))` is used. Args: uniform: Whether to use uniform or normal distributed random initialization. diff --git a/tensorflow/contrib/learn/python/learn/utils/gc.py b/tensorflow/contrib/learn/python/learn/utils/gc.py index 5af9e8b9e2..b890129064 100644 --- a/tensorflow/contrib/learn/python/learn/utils/gc.py +++ b/tensorflow/contrib/learn/python/learn/utils/gc.py @@ -46,14 +46,14 @@ For example, path_list = gc.get_paths("/tmp", parser) # contains all ten Paths every_fifth = gc.mod_export_version(5) - print every_fifth(path_list) # shows ["/tmp/0", "/tmp/5"] + print(every_fifth(path_list)) # shows ["/tmp/0", "/tmp/5"] largest_three = gc.largest_export_versions(3) - print largest_three(all_paths) # shows ["/tmp/7", "/tmp/8", "/tmp/9"] + print(largest_three(all_paths)) # shows ["/tmp/7", "/tmp/8", "/tmp/9"] both = gc.union(every_fifth, largest_three) - print both(all_paths) # shows ["/tmp/0", "/tmp/5", - # "/tmp/7", "/tmp/8", "/tmp/9"] + print(both(all_paths)) # shows ["/tmp/0", "/tmp/5", + # "/tmp/7", "/tmp/8", "/tmp/9"] # delete everything not in 'both' to_delete = gc.negation(both) for p in to_delete(all_paths): diff --git a/tensorflow/contrib/lookup/lookup_ops.py b/tensorflow/contrib/lookup/lookup_ops.py index f53f38f3cf..66caa6a2e5 100644 --- a/tensorflow/contrib/lookup/lookup_ops.py +++ b/tensorflow/contrib/lookup/lookup_ops.py @@ -300,7 +300,7 @@ class MutableHashTable(LookupInterface): default_value=-1) table.insert(keys, values) out = table.lookup(query_keys) - print out.eval() + print(out.eval()) ``` """ @@ -502,7 +502,7 @@ class MutableDenseHashTable(LookupInterface): empty_key=0) table.insert(keys, values) out = table.lookup(query_keys) - print out.eval() + print(out.eval()) ``` """ diff --git a/tensorflow/contrib/makefile/README.md b/tensorflow/contrib/makefile/README.md index 9ba5c035a2..0306ecb214 100644 --- a/tensorflow/contrib/makefile/README.md +++ b/tensorflow/contrib/makefile/README.md @@ -130,7 +130,7 @@ For more details, see the [benchmark documentation](../../tools/benchmark). ## iOS _Note: To use this library in an iOS application, see related instructions in -the [iOS examples](../ios_examples/) directory._ +the [iOS examples](../../examples/ios/) directory._ Install XCode 7.3 or more recent. If you have not already, you will need to install the command-line tools using `xcode-select`: @@ -176,7 +176,7 @@ benchmark program. Although successfully compiling the benchmark program is a sign of success, the program is not a complete iOS app. To see TensorFlow running on iOS, the example Xcode project in -[tensorflow/contrib/ios_examples](../ios_examples) shows how to use the static +[tensorflow/examples/ios](../../examples/ios/) shows how to use the static library in a simple app. ### Building by hand @@ -214,7 +214,7 @@ benchmark program. Although successfully compiling the benchmark program is a sign of success, the program is not a complete iOS app. To see TensorFlow running on iOS, the example Xcode project in -[tensorflow/contrib/ios_examples](../ios_examples) shows how to use the static +[tensorflow/examples/ios](../../examples/ios/) shows how to use the static library in a simple app. #### Universal binaries diff --git a/tensorflow/contrib/mpi/mpi_rendezvous_mgr.cc b/tensorflow/contrib/mpi/mpi_rendezvous_mgr.cc index e97e8d0163..1a2563d20f 100644 --- a/tensorflow/contrib/mpi/mpi_rendezvous_mgr.cc +++ b/tensorflow/contrib/mpi/mpi_rendezvous_mgr.cc @@ -44,7 +44,8 @@ MPIRendezvousMgr::MPIRendezvousMgr(const WorkerEnv* env) // extract worker-name auto parsed = env->local_devices[0]->parsed_name(); - const std::string task_id = strings::StrCat(parsed.job, ":", parsed.replica); + const std::string task_id = + strings::StrCat(parsed.job, ":", parsed.replica, ":", parsed.task); mpiutils_ = new MPIUtils(task_id); background_thread_ = @@ -66,8 +67,8 @@ void MPIRemoteRendezvous::RecvFromRemoteAsync( VLOG(2) << "MPI User requested " << parsed.FullKey() << " @ step: " << step_id_; - std::string src_task = - strings::StrCat(parsed.src.job, ":", parsed.src.replica); + std::string src_task = strings::StrCat( + parsed.src.job, ":", parsed.src.replica, ":", parsed.src.task); const int dst = mpiutils_->GetSourceID(src_task); Device* dst_device; @@ -138,11 +139,7 @@ void MPIRemoteRendezvous::RecvFromRemoteAsync( std::move(request_call), rendezvous_call); } -MPIRemoteRendezvous::~MPIRemoteRendezvous() { - MPIRendezvousMgr* mgr = - reinterpret_cast<MPIRendezvousMgr*>(this->rendezvous_mgr_); - mgr->RemoveStepID(step_id_); -} +MPIRemoteRendezvous::~MPIRemoteRendezvous() {} /* * Add the request for one of our Tensors by a remote process diff --git a/tensorflow/contrib/mpi/mpi_rendezvous_mgr.h b/tensorflow/contrib/mpi/mpi_rendezvous_mgr.h index 50fc380496..24e784df3e 100644 --- a/tensorflow/contrib/mpi/mpi_rendezvous_mgr.h +++ b/tensorflow/contrib/mpi/mpi_rendezvous_mgr.h @@ -147,15 +147,8 @@ class MPIRendezvousMgr : public BaseRendezvousMgr { MPIRequestTensorCall* rCall) { mutex_lock l(mrq_); request_queue_.push(RequestQueueEntry(key, std::move(request_call))); - recv_tensor_map_[step_id][key] = - std::shared_ptr<MPIRequestTensorCall>(rCall); - } - - void RemoveStepID(const int64 step_id) { - mutex_lock l(mrq_); - CHECK(recv_tensor_map_[step_id].size() == 0) << "Removing unfinished step"; - recv_tensor_map_.erase(step_id); - // TODO(jbedorf) Should we verify that the step_id is clear before remove? + const std::string key_id = strings::StrCat(key, "_", step_id); + recv_tensor_map_[key_id] = std::shared_ptr<MPIRequestTensorCall>(rCall); } protected: @@ -181,9 +174,8 @@ class MPIRendezvousMgr : public BaseRendezvousMgr { std::queue<SendQueueEntry> send_queue_ GUARDED_BY(msq_); std::queue<RequestQueueEntry> request_queue_ GUARDED_BY(mrq_); - std::map<int64, std::unordered_map<std::string, - std::shared_ptr<MPIRequestTensorCall>>> - recv_tensor_map_ GUARDED_BY(mrq_); + std::map<std::string, std::shared_ptr<MPIRequestTensorCall>> recv_tensor_map_ + GUARDED_BY(mrq_); void AddRequest(RecvTensorRequest, const int); void MPIBackgroundThread(); @@ -196,22 +188,19 @@ class MPIRendezvousMgr : public BaseRendezvousMgr { void GetRecvCall(const int64 step_id, const std::string& key, std::shared_ptr<MPIRequestTensorCall>* call) { mutex_lock l(mrq_); - if (recv_tensor_map_.find(step_id) == recv_tensor_map_.end()) { - LOG(FATAL) << "Step not found in recv_tensor_map_, step: " << step_id - << " key: " << key << std::endl; - } - if (recv_tensor_map_[step_id].find(key) != - recv_tensor_map_[step_id].end()) { - *call = recv_tensor_map_[step_id][key]; - } else { - LOG(FATAL) << "Key not found in recv_tensor_map_, step: " << step_id + + const std::string key_id = strings::StrCat(key, "_", step_id); + if (recv_tensor_map_.find(key_id) == recv_tensor_map_.end()) { + LOG(FATAL) << "Key/step not found in recv_tensor_map_, step: " << step_id << " key: " << key << std::endl; } + *call = recv_tensor_map_[key_id]; } void RemoveRecvCall(const int64 step_id, const std::string& key) { mutex_lock l(mrq_); - recv_tensor_map_[step_id].erase(key); + const std::string key_id = strings::StrCat(key, "_", step_id); + recv_tensor_map_.erase(key_id); } bool GetRequest(RequestQueueEntry* req) { diff --git a/tensorflow/contrib/session_bundle/gc.py b/tensorflow/contrib/session_bundle/gc.py index fb9e0743a1..249c23c88f 100644 --- a/tensorflow/contrib/session_bundle/gc.py +++ b/tensorflow/contrib/session_bundle/gc.py @@ -46,14 +46,14 @@ For example, path_list = gc.get_paths("/tmp", parser) # contains all ten Paths every_fifth = gc.mod_export_version(5) - print every_fifth(path_list) # shows ["/tmp/0", "/tmp/5"] + print(every_fifth(path_list)) # shows ["/tmp/0", "/tmp/5"] largest_three = gc.largest_export_versions(3) - print largest_three(all_paths) # shows ["/tmp/7", "/tmp/8", "/tmp/9"] + print(largest_three(all_paths)) # shows ["/tmp/7", "/tmp/8", "/tmp/9"] both = gc.union(every_fifth, largest_three) - print both(all_paths) # shows ["/tmp/0", "/tmp/5", - # "/tmp/7", "/tmp/8", "/tmp/9"] + print(both(all_paths)) # shows ["/tmp/0", "/tmp/5", + # "/tmp/7", "/tmp/8", "/tmp/9"] # delete everything not in 'both' to_delete = gc.negation(both) for p in to_delete(all_paths): diff --git a/tensorflow/contrib/timeseries/python/timeseries/BUILD b/tensorflow/contrib/timeseries/python/timeseries/BUILD index e854910f85..d2aa73a178 100644 --- a/tensorflow/contrib/timeseries/python/timeseries/BUILD +++ b/tensorflow/contrib/timeseries/python/timeseries/BUILD @@ -82,6 +82,9 @@ py_test( "estimators_test.py", ], srcs_version = "PY2AND3", + tags = [ + "no_pip_gpu", # b/63391119 + ], deps = [ ":estimators", ":feature_keys", @@ -122,6 +125,9 @@ py_test( "model_utils_test.py", ], srcs_version = "PY2AND3", + tags = [ + "no_pip_gpu", # b/63391119 + ], deps = [ ":model_utils", ":state_management", @@ -153,6 +159,9 @@ py_test( "state_management_test.py", ], srcs_version = "PY2AND3", + tags = [ + "no_pip_gpu", # b/63391119 + ], deps = [ ":input_pipeline", ":model", @@ -203,6 +212,9 @@ py_test( "input_pipeline_test.py", ], srcs_version = "PY2AND3", + tags = [ + "no_pip_gpu", # b/63391119 + ], deps = [ ":input_pipeline", ":test_utils", @@ -313,6 +325,9 @@ py_test( "math_utils_test.py", ], srcs_version = "PY2AND3", + tags = [ + "no_pip_gpu", # b/63391119 + ], deps = [ ":math_utils", "//tensorflow/python:array_ops", diff --git a/tensorflow/contrib/timeseries/python/timeseries/state_space_models/periodic.py b/tensorflow/contrib/timeseries/python/timeseries/state_space_models/periodic.py index d0770b86fe..e70db93ea1 100644 --- a/tensorflow/contrib/timeseries/python/timeseries/state_space_models/periodic.py +++ b/tensorflow/contrib/timeseries/python/timeseries/state_space_models/periodic.py @@ -50,7 +50,7 @@ class CycleStateSpaceModel(state_space_model.StateSpaceModel): def get_noise_transform(self): # transition_power_noise_accumulator makes assumptions about this - # transformation. If the noise transform is modified or overriden, + # transformation. If the noise transform is modified or overridden, # transition_power_noise_accumulator must be modified as well (or discarded, # as it is simply an optimization). return array_ops.pad( diff --git a/tensorflow/core/kernels/decode_wav_op_test.cc b/tensorflow/core/kernels/decode_wav_op_test.cc index c282d53a5a..fc323a5e04 100644 --- a/tensorflow/core/kernels/decode_wav_op_test.cc +++ b/tensorflow/core/kernels/decode_wav_op_test.cc @@ -23,6 +23,7 @@ limitations under the License. #include "tensorflow/cc/ops/audio_ops.h" #include "tensorflow/cc/ops/const_op.h" #include "tensorflow/cc/ops/math_ops.h" +#include "tensorflow/core/framework/shape_inference_testutil.h" #include "tensorflow/core/framework/tensor_testutil.h" #include "tensorflow/core/framework/types.h" #include "tensorflow/core/framework/types.pb.h" @@ -83,4 +84,41 @@ TEST(DecodeWavOpTest, DecodeWavTest) { EXPECT_EQ(14099, sample_rate); } +TEST(DecodeWavOpTest, DecodeWav_ShapeFn) { + ShapeInferenceTestOp op("DecodeWav"); + INFER_ERROR("Shape must be rank 0 but is rank 1", op, "[1]"); + + // audio shape is unknown when desired_{samples,channels} are default. + TF_ASSERT_OK(NodeDefBuilder("test", "DecodeWav") + .Input({"a", 0, DT_STRING}) + .Finalize(&op.node_def)); + INFER_OK(op, "[]", "[?,?];[]"); + + TF_ASSERT_OK(NodeDefBuilder("test", "DecodeWav") + .Input({"a", 0, DT_STRING}) + .Attr("desired_samples", 42) + .Finalize(&op.node_def)); + INFER_OK(op, "[]", "[42,?];[]"); + + // Negative sample value is rejected. + TF_ASSERT_OK(NodeDefBuilder("test", "DecodeWav") + .Input({"a", 0, DT_STRING}) + .Attr("desired_samples", -2) + .Finalize(&op.node_def)); + INFER_ERROR("samples must be non-negative, got -2", op, "[]"); + + TF_ASSERT_OK(NodeDefBuilder("test", "DecodeWav") + .Input({"a", 0, DT_STRING}) + .Attr("desired_channels", 2) + .Finalize(&op.node_def)); + INFER_OK(op, "[]", "[?,2];[]"); + + // Negative channel value is rejected. + TF_ASSERT_OK(NodeDefBuilder("test", "DecodeWav") + .Input({"a", 0, DT_STRING}) + .Attr("desired_channels", -2) + .Finalize(&op.node_def)); + INFER_ERROR("channels must be non-negative, got -2", op, "[]"); +} + } // namespace tensorflow diff --git a/tensorflow/core/lib/io/snappy/snappy_buffers_test.cc b/tensorflow/core/lib/io/snappy/snappy_buffers_test.cc index a27ab73bf7..e0918c70a7 100644 --- a/tensorflow/core/lib/io/snappy/snappy_buffers_test.cc +++ b/tensorflow/core/lib/io/snappy/snappy_buffers_test.cc @@ -117,7 +117,7 @@ Status TestMultipleWrites(size_t compress_input_buf_size, io::SnappyInputBuffer in(file_reader.get(), uncompress_input_buf_size, uncompress_output_buf_size); - // Run the test twice, reseting the stream after the first attempt. + // Run the test twice, resetting the stream after the first attempt. for (int attempt = 0; attempt < 2; ++attempt) { string actual_result; for (int i = 0; i < num_writes; i++) { diff --git a/tensorflow/core/lib/wav/wav_io.cc b/tensorflow/core/lib/wav/wav_io.cc index 028ff26ffb..79918690db 100644 --- a/tensorflow/core/lib/wav/wav_io.cc +++ b/tensorflow/core/lib/wav/wav_io.cc @@ -147,7 +147,7 @@ Status EncodeAudioAsS16LEWav(const float* audio, size_t sample_rate, return errors::InvalidArgument("num_frames must be positive."); } - const size_t bytes_per_second = sample_rate * kBytesPerSample; + const size_t bytes_per_second = sample_rate * kBytesPerSample * num_channels; const size_t num_samples = num_frames * num_channels; const size_t data_size = num_samples * kBytesPerSample; const size_t file_size = kHeaderSize + num_samples * kBytesPerSample; @@ -242,8 +242,7 @@ Status DecodeLin16WaveAsFloatVector(const string& wav_string, "Bad bytes per sample in WAV header: Expected ", expected_bytes_per_sample, " but got ", bytes_per_sample); } - const uint32 expected_bytes_per_second = - (bytes_per_sample * (*sample_rate)) / *channel_count; + const uint32 expected_bytes_per_second = bytes_per_sample * *sample_rate; if (bytes_per_second != expected_bytes_per_second) { return errors::InvalidArgument( "Bad bytes per second in WAV header: Expected ", diff --git a/tensorflow/core/lib/wav/wav_io_test.cc b/tensorflow/core/lib/wav/wav_io_test.cc index e54b9445ab..40ddd94abe 100644 --- a/tensorflow/core/lib/wav/wav_io_test.cc +++ b/tensorflow/core/lib/wav/wav_io_test.cc @@ -97,5 +97,63 @@ TEST(WavIO, EncodeThenDecode) { } } +TEST(WavIO, BasicMono) { + std::vector<uint8> wav_data = { + 'R', 'I', 'F', 'F', // ChunkID + 44, 0, 0, 0, // ChunkSize: 36 + SubChunk2Size + 'W', 'A', 'V', 'E', // Format + 'f', 'm', 't', ' ', // Subchunk1ID + 16, 0, 0, 0, // Subchunk1Size + 1, 0, // AudioFormat: 1=PCM + 1, 0, // NumChannels + 0x44, 0xac, 0, 0, // SampleRate: 44100 + 0x88, 0x58, 0x1, 0, // BytesPerSecond: SampleRate * NumChannels * + // BitsPerSample/8 + 2, 0, // BytesPerSample: NumChannels * BitsPerSample/8 + 16, 0, // BitsPerSample + 'd', 'a', 't', 'a', // Subchunk2ID + 8, 0, 0, 0, // Subchunk2Size: NumSamples * NumChannels * + // BitsPerSample/8 + 0, 0, // Sample 1: 0 + 0xff, 0x7f, // Sample 2: 32767 (saturated) + 0, 0, // Sample 3: 0 + 0x00, 0x80, // Sample 4: -32768 (saturated) + }; + string expected(wav_data.begin(), wav_data.end()); + float audio[] = {0.0f, 1.0f, 0.0f, -1.0f}; + string result; + TF_EXPECT_OK(EncodeAudioAsS16LEWav(audio, 44100, 1, 4, &result)); + EXPECT_EQ(expected, result); +} + +TEST(WavIO, BasicStereo) { + std::vector<uint8> wav_data = { + 'R', 'I', 'F', 'F', // ChunkID + 44, 0, 0, 0, // ChunkSize: 36 + SubChunk2Size + 'W', 'A', 'V', 'E', // Format + 'f', 'm', 't', ' ', // Subchunk1ID + 16, 0, 0, 0, // Subchunk1Size + 1, 0, // AudioFormat: 1=PCM + 2, 0, // NumChannels + 0x44, 0xac, 0, 0, // SampleRate: 44100 + 0x10, 0xb1, 0x2, 0, // BytesPerSecond: SampleRate * NumChannels * + // BitsPerSample/8 + 4, 0, // BytesPerSample: NumChannels * BitsPerSample/8 + 16, 0, // BitsPerSample + 'd', 'a', 't', 'a', // Subchunk2ID + 8, 0, 0, 0, // Subchunk2Size: NumSamples * NumChannels * + // BitsPerSample/8 + 0, 0, // Sample 1: 0 + 0xff, 0x7f, // Sample 2: 32767 (saturated) + 0, 0, // Sample 3: 0 + 0x00, 0x80, // Sample 4: -32768 (saturated) + }; + string expected(wav_data.begin(), wav_data.end()); + float audio[] = {0.0f, 1.0f, 0.0f, -1.0f}; + string result; + TF_EXPECT_OK(EncodeAudioAsS16LEWav(audio, 44100, 2, 2, &result)); + EXPECT_EQ(expected, result); +} + } // namespace wav } // namespace tensorflow diff --git a/tensorflow/core/ops/audio_ops.cc b/tensorflow/core/ops/audio_ops.cc index 02b13a455c..91e81f2579 100644 --- a/tensorflow/core/ops/audio_ops.cc +++ b/tensorflow/core/ops/audio_ops.cc @@ -33,7 +33,7 @@ Status DecodeWavShapeFn(InferenceContext* c) { DimensionHandle channels_dim; int32 desired_channels; TF_RETURN_IF_ERROR(c->GetAttr("desired_channels", &desired_channels)); - if (desired_channels == 0) { + if (desired_channels == -1) { channels_dim = c->UnknownDim(); } else { if (desired_channels < 0) { @@ -45,7 +45,7 @@ Status DecodeWavShapeFn(InferenceContext* c) { DimensionHandle samples_dim; int32 desired_samples; TF_RETURN_IF_ERROR(c->GetAttr("desired_samples", &desired_samples)); - if (desired_samples == 0) { + if (desired_samples == -1) { samples_dim = c->UnknownDim(); } else { if (desired_samples < 0) { diff --git a/tensorflow/core/ops/ops.pbtxt b/tensorflow/core/ops/ops.pbtxt index ccae526e5f..449233e178 100644 --- a/tensorflow/core/ops/ops.pbtxt +++ b/tensorflow/core/ops/ops.pbtxt @@ -28468,6 +28468,33 @@ op { is_stateful: true } op { + name: "LMDBReader" + output_arg { + name: "reader_handle" + description: "The handle to reference the Reader." + type: DT_STRING + is_ref: true + } + attr { + name: "container" + type: "string" + default_value { + s: "" + } + description: "If non-empty, this reader is placed in the given container.\nOtherwise, a default container is used." + } + attr { + name: "shared_name" + type: "string" + default_value { + s: "" + } + description: "If non-empty, this reader is named in the given bucket\nwith this shared_name. Otherwise, the node name is used instead." + } + summary: "A Reader that outputs the records from a LMDB database." + is_stateful: true +} +op { name: "TakeDataset" input_arg { name: "input_dataset" diff --git a/tensorflow/core/protobuf/saver.proto b/tensorflow/core/protobuf/saver.proto index 65fe9c4c98..a757d3f756 100644 --- a/tensorflow/core/protobuf/saver.proto +++ b/tensorflow/core/protobuf/saver.proto @@ -37,9 +37,9 @@ message SaverDef { enum CheckpointFormatVersion { // Internal legacy format. LEGACY = 0; - // Current format: tf.Saver() which works with tensorflow::table::Table. + // Deprecated format: tf.Saver() which works with tensorflow::table::Table. V1 = 1; - // Experimental format under development. + // Current format: more efficient. V2 = 2; } CheckpointFormatVersion version = 7; diff --git a/tensorflow/docs_src/community/documentation.md b/tensorflow/docs_src/community/documentation.md index a757d8c66d..e8b3787c1f 100644 --- a/tensorflow/docs_src/community/documentation.md +++ b/tensorflow/docs_src/community/documentation.md @@ -31,7 +31,7 @@ TensorFlow. However, most developers will contribute documentation into the master Github branch, which is published, occasionally, -at [tensorflow.org/versions/master](https://tensorflow.org/versions/master). +at [tensorflow.org/versions/master](https://www.tensorflow.org/versions/master). If you want documentation changes to appear at root, you will need to also contribute that change to the current stable binary branch (and/or diff --git a/tensorflow/docs_src/extend/index.md b/tensorflow/docs_src/extend/index.md index 590023b065..5812caaffc 100644 --- a/tensorflow/docs_src/extend/index.md +++ b/tensorflow/docs_src/extend/index.md @@ -20,8 +20,8 @@ TensorFlow: Python is currently the only language supported by TensorFlow's API stability promises. However, TensorFlow also provides functionality in C++, Java, and Go, -plus community support for [Haskell](https://github.com/tensorflow/haskell) -and [Rust](https://github.com/tensorflow/rust). If you'd like to create or +plus community support for [Haskell](https://github.com/tensorflow/haskell) and +[Rust](https://github.com/tensorflow/rust). If you'd like to create or develop TensorFlow features in a language other than these languages, read the following guide: diff --git a/tensorflow/docs_src/get_started/get_started.md b/tensorflow/docs_src/get_started/get_started.md index 17d07cc582..852d41e9ed 100644 --- a/tensorflow/docs_src/get_started/get_started.md +++ b/tensorflow/docs_src/get_started/get_started.md @@ -33,7 +33,7 @@ tensors: ```python 3 # a rank 0 tensor; this is a scalar with shape [] -[1. ,2., 3.] # a rank 1 tensor; this is a vector with shape [3] +[1., 2., 3.] # a rank 1 tensor; this is a vector with shape [3] [[1., 2., 3.], [4., 5., 6.]] # a rank 2 tensor; a matrix with shape [2, 3] [[[1., 2., 3.]], [[7., 8., 9.]]] # a rank 3 tensor with shape [2, 1, 3] ``` @@ -100,20 +100,20 @@ we see the expected values of 3.0 and 4.0: ``` We can build more complicated computations by combining `Tensor` nodes with -operations (Operations are also nodes.). For example, we can add our two +operations (Operations are also nodes). For example, we can add our two constant nodes and produce a new graph as follows: ```python node3 = tf.add(node1, node2) -print("node3: ", node3) -print("sess.run(node3): ",sess.run(node3)) +print("node3:", node3) +print("sess.run(node3):", sess.run(node3)) ``` The last two print statements produce ``` -node3: Tensor("Add:0", shape=(), dtype=float32) -sess.run(node3): 7.0 +node3: Tensor("Add:0", shape=(), dtype=float32) +sess.run(node3): 7.0 ``` TensorFlow provides a utility called TensorBoard that can display a picture of @@ -140,8 +140,8 @@ the [run method](https://www.tensorflow.org/api_docs/python/tf/Session#run) to feed concrete values to the placeholders: ```python -print(sess.run(adder_node, {a: 3, b:4.5})) -print(sess.run(adder_node, {a: [1,3], b: [2, 4]})) +print(sess.run(adder_node, {a: 3, b: 4.5})) +print(sess.run(adder_node, {a: [1, 3], b: [2, 4]})) ``` resulting in the output @@ -159,7 +159,7 @@ For example, ```python add_and_triple = adder_node * 3. -print(sess.run(add_and_triple, {a: 3, b:4.5})) +print(sess.run(add_and_triple, {a: 3, b: 4.5})) ``` produces the output ``` @@ -202,7 +202,7 @@ Since `x` is a placeholder, we can evaluate `linear_model` for several values of `x` simultaneously as follows: ```python -print(sess.run(linear_model, {x:[1,2,3,4]})) +print(sess.run(linear_model, {x: [1, 2, 3, 4]})) ``` to produce the output ``` @@ -225,7 +225,7 @@ that abstracts the error of all examples using `tf.reduce_sum`: y = tf.placeholder(tf.float32) squared_deltas = tf.square(linear_model - y) loss = tf.reduce_sum(squared_deltas) -print(sess.run(loss, {x:[1,2,3,4], y:[0,-1,-2,-3]})) +print(sess.run(loss, {x: [1, 2, 3, 4], y: [0, -1, -2, -3]})) ``` producing the loss value ``` @@ -242,7 +242,7 @@ perfect values of -1 and 1. A variable is initialized to the value provided to fixW = tf.assign(W, [-1.]) fixb = tf.assign(b, [1.]) sess.run([fixW, fixb]) -print(sess.run(loss, {x:[1,2,3,4], y:[0,-1,-2,-3]})) +print(sess.run(loss, {x: [1, 2, 3, 4], y: [0, -1, -2, -3]})) ``` The final print shows the loss now is zero. ``` @@ -273,7 +273,7 @@ train = optimizer.minimize(loss) ```python sess.run(init) # reset values to incorrect defaults. for i in range(1000): - sess.run(train, {x:[1,2,3,4], y:[0,-1,-2,-3]}) + sess.run(train, {x: [1, 2, 3, 4], y: [0, -1, -2, -3]}) print(sess.run([W, b])) ``` @@ -319,10 +319,10 @@ init = tf.global_variables_initializer() sess = tf.Session() sess.run(init) # reset values to wrong for i in range(1000): - sess.run(train, {x:x_train, y:y_train}) + sess.run(train, {x: x_train, y: y_train}) # evaluate training accuracy -curr_W, curr_b, curr_loss = sess.run([W, b, loss], {x:x_train, y:y_train}) +curr_W, curr_b, curr_loss = sess.run([W, b, loss], {x: x_train, y: y_train}) print("W: %s b: %s loss: %s"%(curr_W, curr_b, curr_loss)) ``` When run, it produces @@ -377,11 +377,11 @@ y_train = np.array([0., -1., -2., -3.]) x_eval = np.array([2., 5., 8., 1.]) y_eval = np.array([-1.01, -4.1, -7, 0.]) input_fn = tf.estimator.inputs.numpy_input_fn( - {"x":x_train}, y_train, batch_size=4, num_epochs=None, shuffle=True) + {"x": x_train}, y_train, batch_size=4, num_epochs=None, shuffle=True) train_input_fn = tf.estimator.inputs.numpy_input_fn( - {"x":x_train}, y_train, batch_size=4, num_epochs=1000, shuffle=False) + {"x": x_train}, y_train, batch_size=4, num_epochs=1000, shuffle=False) eval_input_fn = tf.estimator.inputs.numpy_input_fn( - {"x":x_eval}, y_eval, batch_size=4, num_epochs=1000, shuffle=False) + {"x": x_eval}, y_eval, batch_size=4, num_epochs=1000, shuffle=False) # We can invoke 1000 training steps by invoking the method and passing the # training data set. @@ -449,11 +449,11 @@ y_train = np.array([0., -1., -2., -3.]) x_eval = np.array([2., 5., 8., 1.]) y_eval = np.array([-1.01, -4.1, -7, 0.]) input_fn = tf.estimator.inputs.numpy_input_fn( - {"x":x_train}, y_train, batch_size=4, num_epochs=None, shuffle=True) + {"x": x_train}, y_train, batch_size=4, num_epochs=None, shuffle=True) train_input_fn = tf.estimator.inputs.numpy_input_fn( - {"x":x_train}, y_train, batch_size=4, num_epochs=1000, shuffle=False) + {"x": x_train}, y_train, batch_size=4, num_epochs=1000, shuffle=False) eval_input_fn = tf.estimator.inputs.numpy_input_fn( - {"x":x_eval}, y_eval, batch_size=4, num_epochs=1000, shuffle=False) + {"x": x_eval}, y_eval, batch_size=4, num_epochs=1000, shuffle=False) # train estimator.train(input_fn=input_fn, steps=1000) diff --git a/tensorflow/docs_src/programmers_guide/faq.md b/tensorflow/docs_src/programmers_guide/faq.md index 74407ed315..56486a48b7 100644 --- a/tensorflow/docs_src/programmers_guide/faq.md +++ b/tensorflow/docs_src/programmers_guide/faq.md @@ -149,6 +149,8 @@ TensorFlow also has a to help build support for more client languages. We invite contributions of new language bindings. +Bindings for various other languages (such as [C#](https://github.com/migueldeicaza/TensorFlowSharp), [Julia](https://github.com/malmaud/TensorFlow.jl), [Ruby](https://github.com/somaticio/tensorflow.rb) and [Scala](https://github.com/eaplatanios/tensorflow_scala)) created and supported by the opensource community build on top of the C API supported by the TensorFlow maintainers. + #### Does TensorFlow make use of all the devices (GPUs and CPUs) available on my machine? TensorFlow supports multiple GPUs and CPUs. See the how-to documentation on diff --git a/tensorflow/examples/image_retraining/retrain.py b/tensorflow/examples/image_retraining/retrain.py index ac39639af1..2e2e578050 100644 --- a/tensorflow/examples/image_retraining/retrain.py +++ b/tensorflow/examples/image_retraining/retrain.py @@ -741,7 +741,7 @@ def add_final_training_ops(class_count, final_tensor_name, bottleneck_tensor, weights, and then sets up all the gradients for the backward pass. The set up for the softmax and fully-connected layers is based on: - https://tensorflow.org/versions/master/tutorials/mnist/beginners/index.html + https://www.tensorflow.org/versions/master/tutorials/mnist/beginners/index.html Args: class_count: Integer of how many categories of things we're trying to diff --git a/tensorflow/examples/label_image/README.md b/tensorflow/examples/label_image/README.md index c5857f394a..a9e44745e5 100644 --- a/tensorflow/examples/label_image/README.md +++ b/tensorflow/examples/label_image/README.md @@ -63,14 +63,14 @@ $ bazel-bin/tensorflow/examples/label_image/label_image --image=my_image.png ``` For a more detailed look at this code, you can check out the C++ section of the -[Inception tutorial](https://tensorflow.org/tutorials/image_recognition/). +[Inception tutorial](https://www.tensorflow.org/tutorials/image_recognition/). ## Python implementation label_image.py is a python implementation that provides code corresponding to the C++ code here. This gives more intuitive mapping between C++ and Python than the Python code mentioned in the -[Inception tutorial](https://tensorflow.org/tutorials/image_recognition/). +[Inception tutorial](https://www.tensorflow.org/tutorials/image_recognition/). and could be easier to add visualization or debug code. With tensorflow python package installed, you can run it like: diff --git a/tensorflow/go/op/wrappers.go b/tensorflow/go/op/wrappers.go index 6d0990cb52..5a67cfa441 100644 --- a/tensorflow/go/op/wrappers.go +++ b/tensorflow/go/op/wrappers.go @@ -14441,6 +14441,51 @@ func Tanh(scope *Scope, x tf.Output) (y tf.Output) { return op.Output(0) } +// Computes inverse hyperbolic sine of x element-wise. +func Asinh(scope *Scope, x tf.Output) (y tf.Output) { + if scope.Err() != nil { + return + } + opspec := tf.OpSpec{ + Type: "Asinh", + Input: []tf.Input{ + x, + }, + } + op := scope.AddOperation(opspec) + return op.Output(0) +} + +// Computes inverse hyperbolic cosine of x element-wise. +func Acosh(scope *Scope, x tf.Output) (y tf.Output) { + if scope.Err() != nil { + return + } + opspec := tf.OpSpec{ + Type: "Acosh", + Input: []tf.Input{ + x, + }, + } + op := scope.AddOperation(opspec) + return op.Output(0) +} + +// Computes inverse hyperbolic tangent of x element-wise. +func Atanh(scope *Scope, x tf.Output) (y tf.Output) { + if scope.Err() != nil { + return + } + opspec := tf.OpSpec{ + Type: "Atanh", + Input: []tf.Input{ + x, + }, + } + op := scope.AddOperation(opspec) + return op.Output(0) +} + // TextLineReaderV2Attr is an optional argument to TextLineReaderV2. type TextLineReaderV2Attr func(optionalAttr) @@ -18243,21 +18288,6 @@ func AssignAddVariableOp(scope *Scope, resource tf.Output, value tf.Output) (o * return scope.AddOperation(opspec) } -// Computes inverse hyperbolic sine of x element-wise. -func Asinh(scope *Scope, x tf.Output) (y tf.Output) { - if scope.Err() != nil { - return - } - opspec := tf.OpSpec{ - Type: "Asinh", - Input: []tf.Input{ - x, - }, - } - op := scope.AddOperation(opspec) - return op.Output(0) -} - // Real-valued fast Fourier transform. // // Computes the 1-dimensional discrete Fourier transform of a real-valued signal @@ -21096,21 +21126,6 @@ func SerializeManySparse(scope *Scope, sparse_indices tf.Output, sparse_values t return op.Output(0) } -// Computes inverse hyperbolic cosine of x element-wise. -func Acosh(scope *Scope, x tf.Output) (y tf.Output) { - if scope.Err() != nil { - return - } - opspec := tf.OpSpec{ - Type: "Acosh", - Input: []tf.Input{ - x, - }, - } - op := scope.AddOperation(opspec) - return op.Output(0) -} - // Computes the reverse mode backpropagated gradient of the Cholesky algorithm. // // For an explanation see "Differentiation of the Cholesky algorithm" by @@ -21139,21 +21154,6 @@ func CholeskyGrad(scope *Scope, l tf.Output, grad tf.Output) (output tf.Output) return op.Output(0) } -// Computes inverse hyperbolic tangent of x element-wise. -func Atanh(scope *Scope, x tf.Output) (y tf.Output) { - if scope.Err() != nil { - return - } - opspec := tf.OpSpec{ - Type: "Atanh", - Input: []tf.Input{ - x, - }, - } - op := scope.AddOperation(opspec) - return op.Output(0) -} - // Computes the log of the absolute value of `Gamma(x)` element-wise. func Lgamma(scope *Scope, x tf.Output) (y tf.Output) { if scope.Err() != nil { diff --git a/tensorflow/java/BUILD b/tensorflow/java/BUILD index 90372660cd..759f4c1b0e 100644 --- a/tensorflow/java/BUILD +++ b/tensorflow/java/BUILD @@ -162,6 +162,32 @@ java_test( ], ) +java_test( + name = "PrimitiveOpTest", + size = "small", + srcs = ["src/test/java/org/tensorflow/op/PrimitiveOpTest.java"], + javacopts = JAVACOPTS, + test_class = "org.tensorflow.op.PrimitiveOpTest", + deps = [ + ":tensorflow", + ":testutil", + "@junit", + ], +) + +java_test( + name = "OperandsTest", + size = "small", + srcs = ["src/test/java/org/tensorflow/op/OperandsTest.java"], + javacopts = JAVACOPTS, + test_class = "org.tensorflow.op.OperandsTest", + deps = [ + ":tensorflow", + ":testutil", + "@junit", + ], +) + filegroup( name = "libtensorflow_jni", srcs = select({ diff --git a/tensorflow/java/src/main/java/org/tensorflow/NativeLibrary.java b/tensorflow/java/src/main/java/org/tensorflow/NativeLibrary.java index 15f7b99135..ca06177017 100644 --- a/tensorflow/java/src/main/java/org/tensorflow/NativeLibrary.java +++ b/tensorflow/java/src/main/java/org/tensorflow/NativeLibrary.java @@ -49,7 +49,7 @@ final class NativeLibrary { // Either: // (1) The native library has already been statically loaded, OR // (2) The required native code has been statically linked (through a custom launcher), OR - // (3) The native code is part of another library (such as an an application-level libraryh) + // (3) The native code is part of another library (such as an application-level library) // that has already been loaded. For example, tensorflow/examples/android and // tensorflow/contrib/android include the required native code in differently named libraries. // diff --git a/tensorflow/java/src/main/java/org/tensorflow/Operand.java b/tensorflow/java/src/main/java/org/tensorflow/Operand.java new file mode 100644 index 0000000000..695c4c1060 --- /dev/null +++ b/tensorflow/java/src/main/java/org/tensorflow/Operand.java @@ -0,0 +1,48 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +package org.tensorflow; + +/** + * Interface implemented by operands of a TensorFlow operation. + * + * <p>Example usage: + * + * <pre>{@code + * // The "decodeJpeg" operation can be used as an operand to the "cast" operation + * Operand decodeJpeg = ops.image().decodeJpeg(...); + * ops.math().cast(decodeJpeg, DataType.FLOAT); + * + * // The output "y" of the "unique" operation can be used as an operand to the "cast" operation + * Output y = ops.array().unique(...).y(); + * ops.math().cast(y, DataType.FLOAT); + * + * // The "split" operation can be used as operand list to the "concat" operation + * Iterable<? extends Operand> split = ops.array().split(...); + * ops.array().concat(0, split); + * }</pre> + */ +public interface Operand { + + /** + * Returns the symbolic handle of a tensor. + * + * <p>Inputs to TensorFlow operations are outputs of another TensorFlow operation. This method is + * used to obtain a symbolic handle that represents the computation of the input. + * + * @see OperationBuilder#addInput(Output) + */ + Output asOutput(); +} diff --git a/tensorflow/java/src/main/java/org/tensorflow/Operation.java b/tensorflow/java/src/main/java/org/tensorflow/Operation.java index d584085282..ec26309fba 100644 --- a/tensorflow/java/src/main/java/org/tensorflow/Operation.java +++ b/tensorflow/java/src/main/java/org/tensorflow/Operation.java @@ -91,6 +91,21 @@ public final class Operation { } } + /** + * Returns symbolic handles to a list of tensors produced by this operation. + * + * @param idx index of the first tensor of the list + * @param length number of tensors in the list + * @return array of {@code Output} + */ + public Output[] outputList(int idx, int length) { + Output[] outputs = new Output[length]; + for (int i = 0; i < length; ++i) { + outputs[i] = output(idx + i); + } + return outputs; + } + /** Returns a symbolic handle to one of the tensors produced by this operation. */ public Output output(int idx) { return new Output(this, idx); diff --git a/tensorflow/java/src/main/java/org/tensorflow/Output.java b/tensorflow/java/src/main/java/org/tensorflow/Output.java index ad28eb2743..8dff50fafb 100644 --- a/tensorflow/java/src/main/java/org/tensorflow/Output.java +++ b/tensorflow/java/src/main/java/org/tensorflow/Output.java @@ -16,16 +16,17 @@ limitations under the License. package org.tensorflow; import java.util.Objects; + /** * A symbolic handle to a tensor produced by an {@link Operation}. * * <p>An Output is a symbolic handle to a tensor. The value of the Tensor is computed by executing * the {@link Operation} in a {@link Session}. * - * <p>By implementing the {@link Input} interface, instances of this class could also be passed - * directly in input to an operation. + * <p>By implementing the {@link Operand} interface, instances of this class also act as operands to + * {@link org.tensorflow.op.Op Op} instances. */ -public final class Output implements Input { +public final class Output implements Operand { /** Handle to the idx-th output of the Operation {@code op}. */ public Output(Operation op, int idx) { diff --git a/tensorflow/java/src/main/java/org/tensorflow/op/Op.java b/tensorflow/java/src/main/java/org/tensorflow/op/Op.java new file mode 100644 index 0000000000..aa6db40457 --- /dev/null +++ b/tensorflow/java/src/main/java/org/tensorflow/op/Op.java @@ -0,0 +1,35 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +package org.tensorflow.op; + +/** + * A marker interface for all operation wrappers. + * + * <p>Operation wrappers provide strongly typed interfaces for building operations and linking them + * into a graph without the use of literals and indexes required by the core classes. + * + * <p>This interface allows keeping references to any operation wrapper using a common type. + * + * <pre>{@code + * // All values returned by an Ops call can be referred as a Op + * Op split = ops.array().split(...); + * Op shape = ops.array().shape(...); + * + * // All operations could be added to an Op collection + * Collection<Op> allOps = Arrays.asList(split, shape); + * } + */ +public interface Op {} diff --git a/tensorflow/java/src/main/java/org/tensorflow/op/Operands.java b/tensorflow/java/src/main/java/org/tensorflow/op/Operands.java new file mode 100644 index 0000000000..5971103d6d --- /dev/null +++ b/tensorflow/java/src/main/java/org/tensorflow/op/Operands.java @@ -0,0 +1,46 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +package org.tensorflow.op; + +import java.util.ArrayList; +import java.util.List; +import org.tensorflow.Operand; +import org.tensorflow.OperationBuilder; +import org.tensorflow.Output; + +/** Utilities for manipulating operand related types and lists. */ +public final class Operands { + + /** + * Converts a list of {@link Operand} into an array of {@link Output}. + * + * <p>Operation wrappers need to convert back a list of inputs into an array of outputs in order + * to build an operation, see {@link OperationBuilder#addInputList(Output[])}. + * + * @param inputs an iteration of input operands + * @return an array of outputs + */ + public static Output[] asOutputs(Iterable<? extends Operand> inputs) { + List<Output> outputList = new ArrayList<>(); + for (Operand input : inputs) { + outputList.add(input.asOutput()); + } + return outputList.toArray(new Output[outputList.size()]); + } + + // Disabled constructor + private Operands() {} +} diff --git a/tensorflow/java/src/main/java/org/tensorflow/op/PrimitiveOp.java b/tensorflow/java/src/main/java/org/tensorflow/op/PrimitiveOp.java new file mode 100644 index 0000000000..8e56f97041 --- /dev/null +++ b/tensorflow/java/src/main/java/org/tensorflow/op/PrimitiveOp.java @@ -0,0 +1,65 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +package org.tensorflow.op; + +import org.tensorflow.Operation; + +/** + * A base class for {@link Op} implementations that are backed by a single {@link Operation}. + * + * <p>Each operation registered in the TensorFlow core is a primitive and is provided as a {@code + * PrimitiveOp}. Custom operations working with only one primitive may also derive from this class. + */ +public abstract class PrimitiveOp implements Op { + + @Override + public final int hashCode() { + return operation.hashCode(); + } + + @Override + public final boolean equals(Object obj) { + if (this == obj) { + return true; + } + // Note: we consider that all objects wrapping the same operation are equal, no matter their + // implementation + if (!(obj instanceof PrimitiveOp)) { + return false; + } + return operation.equals(((PrimitiveOp) obj).operation); + } + + @Override + public final String toString() { + return String.format("<%s '%s'>", operation.type(), operation.name()); + } + + /** + * Underlying operation. It is deliberately not exposed by a getter method to avoid any name + * conflict with generated methods of the subclasses. + */ + protected final Operation operation; + + /** + * Constructor. + * + * @param operation the underlying operation + */ + protected PrimitiveOp(Operation operation) { + this.operation = operation; + } +} diff --git a/tensorflow/java/src/test/java/org/tensorflow/OperationTest.java b/tensorflow/java/src/test/java/org/tensorflow/OperationTest.java index 27afc046ac..23b5abc077 100644 --- a/tensorflow/java/src/test/java/org/tensorflow/OperationTest.java +++ b/tensorflow/java/src/test/java/org/tensorflow/OperationTest.java @@ -153,6 +153,19 @@ public class OperationTest { } } + @Test + public void outputList() { + try (Graph g = new Graph()) { + Operation split = TestUtil.split(g, "split", new int[] {0, 1, 2}, 3); + Output[] outputs = split.outputList(1, 2); + assertNotNull(outputs); + assertEquals(2, outputs.length); + for (int i = 0; i < outputs.length; ++i) { + assertEquals(i + 1, outputs[i].index()); + } + } + } + private static int split(int[] values, int num_split) { try (Graph g = new Graph()) { return g.opBuilder("Split", "Split") diff --git a/tensorflow/java/src/test/java/org/tensorflow/TestUtil.java b/tensorflow/java/src/test/java/org/tensorflow/TestUtil.java index 6a3a16c2e1..e3415a696d 100644 --- a/tensorflow/java/src/test/java/org/tensorflow/TestUtil.java +++ b/tensorflow/java/src/test/java/org/tensorflow/TestUtil.java @@ -48,6 +48,14 @@ public class TestUtil { .output(0); } + public static Operation split(Graph g, String name, int[] values, int numSplit) { + return g.opBuilder("Split", name) + .addInput(constant(g, "split_dim", 0)) + .addInput(constant(g, "values", values)) + .setAttr("num_split", numSplit) + .build(); + } + public static void transpose_A_times_X(Graph g, int[][] a) { matmul(g, "Y", constant(g, "A", a), placeholder(g, "X", DataType.INT32), true, false); } diff --git a/tensorflow/java/src/test/java/org/tensorflow/op/OperandsTest.java b/tensorflow/java/src/test/java/org/tensorflow/op/OperandsTest.java new file mode 100644 index 0000000000..4fdd150acc --- /dev/null +++ b/tensorflow/java/src/test/java/org/tensorflow/op/OperandsTest.java @@ -0,0 +1,46 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +package org.tensorflow.op; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertSame; + +import java.util.Arrays; +import java.util.List; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.tensorflow.Graph; +import org.tensorflow.Operation; +import org.tensorflow.Output; +import org.tensorflow.TestUtil; + +/** Unit tests for {@link org.tensorflow.op.Operands}. */ +@RunWith(JUnit4.class) +public class OperandsTest { + + @Test + public void createOutputArrayFromOperandList() { + try (Graph g = new Graph()) { + Operation split = TestUtil.split(g, "split", new int[] {0, 1, 2}, 3); + List<Output> list = Arrays.asList(split.output(0), split.output(2)); + Output[] array = Operands.asOutputs(list); + assertEquals(list.size(), array.length); + assertSame(array[0], list.get(0)); + assertSame(array[1], list.get(1)); + } + } +} diff --git a/tensorflow/java/src/test/java/org/tensorflow/op/PrimitiveOpTest.java b/tensorflow/java/src/test/java/org/tensorflow/op/PrimitiveOpTest.java new file mode 100644 index 0000000000..b24bf5a476 --- /dev/null +++ b/tensorflow/java/src/test/java/org/tensorflow/op/PrimitiveOpTest.java @@ -0,0 +1,60 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +package org.tensorflow.op; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertTrue; + +import java.util.HashSet; +import java.util.Set; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.tensorflow.Graph; +import org.tensorflow.Output; +import org.tensorflow.TestUtil; + +/** Unit tests for {@link org.tensorflow.op.PrimitiveOp} */ +@RunWith(JUnit4.class) +public class PrimitiveOpTest { + + @Test + public void equalsHashcode() { + try (Graph g = new Graph()) { + Output array = TestUtil.constant(g, "array", new int[2]); + + PrimitiveOp test1 = + new PrimitiveOp(g.opBuilder("Shape", "shape1").addInput(array).build()) {}; + PrimitiveOp test2 = + new PrimitiveOp(g.opBuilder("Shape", "shape2").addInput(array).build()) {}; + PrimitiveOp test3 = new PrimitiveOp(test1.operation) {}; + + // equals() tests + assertNotEquals(test1, test2); + assertEquals(test1, test3); + assertEquals(test3, test1); + assertNotEquals(test2, test3); + + // hashcode() tests + Set<PrimitiveOp> ops = new HashSet<>(); + assertTrue(ops.add(test1)); + assertTrue(ops.add(test2)); + assertFalse(ops.add(test3)); + } + } +} diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD index 8d3d81eeba..0e56d6ed19 100644 --- a/tensorflow/python/BUILD +++ b/tensorflow/python/BUILD @@ -2379,6 +2379,7 @@ cuda_py_test( ":variables", "//third_party/py/numpy", ], + tags = ["no_windows"], ) cuda_py_test( diff --git a/tensorflow/python/framework/graph_io.py b/tensorflow/python/framework/graph_io.py index f909bcd62d..a0ea4ad48e 100644 --- a/tensorflow/python/framework/graph_io.py +++ b/tensorflow/python/framework/graph_io.py @@ -29,7 +29,7 @@ from tensorflow.python.lib.io import file_io def write_graph(graph_or_graph_def, logdir, name, as_text=True): """Writes a graph proto to a file. - The graph is written as a binary proto unless `as_text` is `True`. + The graph is written as a text proto unless `as_text` is `False`. ```python v = tf.Variable(0, name='my_variable') diff --git a/tensorflow/python/kernel_tests/BUILD b/tensorflow/python/kernel_tests/BUILD index 5d582351fd..f2cf90f303 100644 --- a/tensorflow/python/kernel_tests/BUILD +++ b/tensorflow/python/kernel_tests/BUILD @@ -72,6 +72,7 @@ tf_py_test( "//tensorflow/python:framework_test_lib", "//tensorflow/python:string_ops", ], + tags = ["nomac"], # b/35468214 ) tf_py_test( diff --git a/tensorflow/python/kernel_tests/sparse_ops_test.py b/tensorflow/python/kernel_tests/sparse_ops_test.py index 6664362226..d500f94dc2 100644 --- a/tensorflow/python/kernel_tests/sparse_ops_test.py +++ b/tensorflow/python/kernel_tests/sparse_ops_test.py @@ -548,7 +548,7 @@ class SparseReduceTest(test_util.TensorFlowTestCase): # [[1, ?, 2] # [?, 3, ?]] - # where ? is implictly-zero. + # where ? is implicitly-zero. ind = np.array([[0, 0], [0, 2], [1, 1]]).astype(np.int64) vals = np.array([1, 1, 1]).astype(np.int32) dense_shape = np.array([2, 3]).astype(np.int64) @@ -906,5 +906,16 @@ class SparseTransposeTest(test.TestCase): self.assertAllEqual(dn_trans, expected_trans) +class SparsePlaceholderTest(test.TestCase): + + def testPlaceholder(self): + with self.test_session(use_gpu=False): + foo = array_ops.sparse_placeholder(dtypes.float32, shape=(10, 47)) + self.assertAllEqual([10, 47], foo.get_shape()) + + foo = array_ops.sparse_placeholder(dtypes.float32, shape=(None, 47)) + self.assertAllEqual(None, foo.get_shape()) + + if __name__ == "__main__": googletest.main() diff --git a/tensorflow/python/lib/core/strings.i b/tensorflow/python/lib/core/strings.i index b74eb91cd5..938c13e30e 100644 --- a/tensorflow/python/lib/core/strings.i +++ b/tensorflow/python/lib/core/strings.i @@ -80,7 +80,7 @@ bool _BytesToStringPiece(PyObject* obj, tensorflow::StringPiece* result) { } } -// Converts a C++ string vector to a a list of Python bytes objects. +// Converts a C++ string vector to a list of Python bytes objects. %typemap(out) std::vector<string> { const int size = $1.size(); auto temp_string_list = tensorflow::make_safe(PyList_New(size)); diff --git a/tensorflow/python/ops/array_ops.py b/tensorflow/python/ops/array_ops.py index b42238a935..0956d914f5 100644 --- a/tensorflow/python/ops/array_ops.py +++ b/tensorflow/python/ops/array_ops.py @@ -721,10 +721,10 @@ def _SliceHelperVar(var, slice_spec): A = tf.Variable([[1,2,3], [4,5,6], [7,8,9]], dtype=tf.float32) with tf.Session() as sess: sess.run(tf.global_variables_initializer()) - print sess.run(A[:2, :2]) # => [[1,2], [4,5]] + print(sess.run(A[:2, :2])) # => [[1,2], [4,5]] op = A[:2,:2].assign(22. * tf.ones((2, 2))) - print sess.run(op) # => [[22, 22, 3], [22, 22, 6], [7,8,9]] + print(sess.run(op)) # => [[22, 22, 3], [22, 22, 6], [7,8,9]] ``` Note that assignments currently do not support NumPy broadcasting @@ -1556,7 +1556,7 @@ def _normalize_sparse_shape(shape, name): for el in shape: if el is None: return None - return ops.convert_to_tensor(shape, name=name) + return ops.convert_to_tensor(shape, dtype=dtypes.int64, name=name) def sparse_placeholder(dtype, shape=None, name=None): diff --git a/tensorflow/python/ops/lookup_ops.py b/tensorflow/python/ops/lookup_ops.py index dade053589..e0cf8bc5e1 100644 --- a/tensorflow/python/ops/lookup_ops.py +++ b/tensorflow/python/ops/lookup_ops.py @@ -240,7 +240,7 @@ class HashTable(InitializableLookupTableBase): tf.contrib.lookup.KeyValueTensorInitializer(keys, values), -1) out = table.lookup(input_tensor). table.init.run() - print out.eval() + print(out.eval()) ``` """ @@ -709,7 +709,7 @@ class IdTableWithHashBuckets(LookupInterface): num_oov_buckets) out = table.lookup(input_tensor). table.init.run() - print out.eval() + print(out.eval()) ``` The hash function used for generating out-of-vocabulary buckets ID is handled diff --git a/tensorflow/python/platform/flags.py b/tensorflow/python/platform/flags.py index f70dbf18d4..60ec4f84c4 100644 --- a/tensorflow/python/platform/flags.py +++ b/tensorflow/python/platform/flags.py @@ -44,7 +44,12 @@ class _FlagValues(object): def __getattr__(self, name): """Retrieves the 'value' attribute of the flag --name.""" - if not self.__dict__['__parsed']: + try: + parsed = self.__dict__['__parsed'] + except KeyError: + # May happen during pickle.load or copy.copy + raise AttributeError(name) + if not parsed: self._parse_flags() if name not in self.__dict__['__flags']: raise AttributeError(name) diff --git a/tensorflow/python/platform/flags_test.py b/tensorflow/python/platform/flags_test.py index 0dbaafd1fa..8b990975dd 100644 --- a/tensorflow/python/platform/flags_test.py +++ b/tensorflow/python/platform/flags_test.py @@ -17,6 +17,7 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +import copy import sys import unittest @@ -79,6 +80,10 @@ class FlagsTest(unittest.TestCase): FLAGS.float_foo = -1.0 self.assertEqual(-1.0, FLAGS.float_foo) + def test_copy(self): + copied = copy.copy(FLAGS) + self.assertEqual(copied.__dict__, FLAGS.__dict__) + def main(_): # unittest.main() tries to interpret the unknown flags, so use the diff --git a/tensorflow/python/util/tf_should_use.py b/tensorflow/python/util/tf_should_use.py index 05c99856d2..ab9e82a3cc 100644 --- a/tensorflow/python/util/tf_should_use.py +++ b/tensorflow/python/util/tf_should_use.py @@ -25,10 +25,15 @@ import types import six # pylint: disable=unused-import -from backports import weakref # pylint: disable=g-bad-import-order +# pylint: disable=g-bad-import-order,g-import-not-at-top +try: + from weakref import finalize +except ImportError: + from backports.weakref import finalize from tensorflow.python.platform import tf_logging from tensorflow.python.util import tf_decorator +# pylint: enable=g-bad-import-order,g-import-not-at-top class _RefInfoField( @@ -107,7 +112,7 @@ def _add_should_use_warning(x, fatal_error=False): # garbage collected. Can't add self as the args because the # loop will break garbage collection. We keep track of # ourselves via python ids. - weakref.finalize(self, _deleted, self._tf_ref_id, fatal_error) + finalize(self, _deleted, self._tf_ref_id, fatal_error) # Not sure why this pylint warning is being used; this is not an # old class form. diff --git a/tensorflow/tools/ci_build/windows/bazel/common_env.sh b/tensorflow/tools/ci_build/windows/bazel/common_env.sh index 8853dc53b1..05392c2724 100644 --- a/tensorflow/tools/ci_build/windows/bazel/common_env.sh +++ b/tensorflow/tools/ci_build/windows/bazel/common_env.sh @@ -56,3 +56,7 @@ export PATH="/c/tools/cuda/bin:$PATH" # Set the common build options on Windows export BUILD_OPTS='--copt=-w --host_copt=-w --verbose_failures --experimental_ui' + +# Build TF with wrapper-less CROSSTOOL +# TODO(pcloudy): Remove this after wrapper-less CROSSTOOL becomes default +export NO_MSVC_WRAPPER=1 diff --git a/tensorflow/tools/docker/Dockerfile b/tensorflow/tools/docker/Dockerfile index 5b3f1f936a..07a972400d 100644 --- a/tensorflow/tools/docker/Dockerfile +++ b/tensorflow/tools/docker/Dockerfile @@ -24,14 +24,15 @@ RUN curl -O https://bootstrap.pypa.io/get-pip.py && \ rm get-pip.py RUN pip --no-cache-dir install \ + Pillow \ + h5py \ ipykernel \ jupyter \ matplotlib \ numpy \ + pandas \ scipy \ sklearn \ - pandas \ - Pillow \ && \ python -m ipykernel.kernelspec diff --git a/tensorflow/tools/docker/Dockerfile.gpu b/tensorflow/tools/docker/Dockerfile.gpu index 3ba1e963f9..da83a30058 100644 --- a/tensorflow/tools/docker/Dockerfile.gpu +++ b/tensorflow/tools/docker/Dockerfile.gpu @@ -24,14 +24,15 @@ RUN curl -O https://bootstrap.pypa.io/get-pip.py && \ rm get-pip.py RUN pip --no-cache-dir install \ + Pillow \ + h5py \ ipykernel \ jupyter \ matplotlib \ numpy \ + pandas \ scipy \ sklearn \ - pandas \ - Pillow \ && \ python -m ipykernel.kernelspec diff --git a/tensorflow/tools/pip_package/setup.py b/tensorflow/tools/pip_package/setup.py index 39499e1775..c4284e10de 100644 --- a/tensorflow/tools/pip_package/setup.py +++ b/tensorflow/tools/pip_package/setup.py @@ -35,7 +35,6 @@ REQUIRED_PACKAGES = [ 'numpy >= 1.11.0', 'six >= 1.10.0', 'protobuf >= 3.2.0', - 'backports.weakref == 1.0rc1', 'tensorflow-tensorboard', ] @@ -54,6 +53,10 @@ else: # mock comes with unittest.mock for python3, need to install for python2 REQUIRED_PACKAGES.append('mock >= 2.0.0') +# weakref.finalize was introduced in Python 3.4 +if sys.version_info < (3, 4): + REQUIRED_PACKAGES.append('backports.weakref >= 1.0rc1') + # pylint: disable=line-too-long CONSOLE_SCRIPTS = [ 'saved_model_cli = tensorflow.python.tools.saved_model_cli:main', diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl index c1832eceb9..a5f0440601 100644 --- a/tensorflow/workspace.bzl +++ b/tensorflow/workspace.bzl @@ -327,11 +327,11 @@ def tf_workspace(path_prefix="", tf_repo_name=""): patched_http_archive( name = "protobuf", urls = [ - "http://mirror.bazel.build/github.com/google/protobuf/archive/v3.3.1.tar.gz", - "https://github.com/google/protobuf/archive/v3.3.1.tar.gz", + "https://github.com/google/protobuf/archive/0b059a3d8a8f8aa40dde7bea55edca4ec5dfea66.tar.gz", + "http://mirror.bazel.build/github.com/google/protobuf/archive/0b059a3d8a8f8aa40dde7bea55edca4ec5dfea66.tar.gz", ], - sha256 = "30f23a45c6f4515598702a6d19c4295ba92c4a635d7ad8d331a4db9fccff392d", - strip_prefix = "protobuf-3.3.1", + sha256 = "6d43b9d223ce09e5d4ce8b0060cb8a7513577a35a64c7e3dad10f0703bf3ad93", + strip_prefix = "protobuf-0b059a3d8a8f8aa40dde7bea55edca4ec5dfea66", # TODO: remove patching when tensorflow stops linking same protos into # multiple shared libraries loaded in runtime by python. # This patch fixes a runtime crash when tensorflow is compiled @@ -345,21 +345,21 @@ def tf_workspace(path_prefix="", tf_repo_name=""): native.http_archive( name = "com_google_protobuf", urls = [ - "http://mirror.bazel.build/github.com/google/protobuf/archive/v3.3.1.tar.gz", - "https://github.com/google/protobuf/archive/v3.3.1.tar.gz", + "https://github.com/google/protobuf/archive/0b059a3d8a8f8aa40dde7bea55edca4ec5dfea66.tar.gz", + "http://mirror.bazel.build/github.com/google/protobuf/archive/0b059a3d8a8f8aa40dde7bea55edca4ec5dfea66.tar.gz", ], - sha256 = "30f23a45c6f4515598702a6d19c4295ba92c4a635d7ad8d331a4db9fccff392d", - strip_prefix = "protobuf-3.3.1", + sha256 = "6d43b9d223ce09e5d4ce8b0060cb8a7513577a35a64c7e3dad10f0703bf3ad93", + strip_prefix = "protobuf-0b059a3d8a8f8aa40dde7bea55edca4ec5dfea66", ) native.http_archive( name = "com_google_protobuf_cc", urls = [ - "http://mirror.bazel.build/github.com/google/protobuf/archive/v3.3.1.tar.gz", - "https://github.com/google/protobuf/archive/v3.3.1.tar.gz", + "https://github.com/google/protobuf/archive/0b059a3d8a8f8aa40dde7bea55edca4ec5dfea66.tar.gz", + "http://mirror.bazel.build/github.com/google/protobuf/archive/0b059a3d8a8f8aa40dde7bea55edca4ec5dfea66.tar.gz", ], - sha256 = "30f23a45c6f4515598702a6d19c4295ba92c4a635d7ad8d331a4db9fccff392d", - strip_prefix = "protobuf-3.3.1", + sha256 = "6d43b9d223ce09e5d4ce8b0060cb8a7513577a35a64c7e3dad10f0703bf3ad93", + strip_prefix = "protobuf-0b059a3d8a8f8aa40dde7bea55edca4ec5dfea66", ) native.new_http_archive( |