diff options
author | Vijay Vasudevan <vrv@google.com> | 2017-04-26 19:02:10 -0800 |
---|---|---|
committer | TensorFlower Gardener <gardener@tensorflow.org> | 2017-04-26 20:31:03 -0700 |
commit | 95ca363c6caf5d152eef594c7fc3703a5d490070 (patch) | |
tree | e275d9b938d8c162d2a63e2677c80d590260c567 | |
parent | be43153b21fba4280f0f2a016242614a21d115f3 (diff) |
Make error message when explicitly adding an invalid device more clear.
Before, you would get something like:
`InvalidArgumentError: Cannot assign a device to node 'save/RestoreV2_45': Could not satisfy explicit device specification '/job:ps/task:1/device:CPU:0' because no devices matching that specification are registered in this process; available devices: /job:localhost/replica:0/task:0/cpu:0 [[Node: save/RestoreV2_45 = RestoreV2[dtypes=[DT_FLOAT], _device="/job:ps/task:1/device:CPU:0"](save/Const, save/RestoreV2_45/tensor_names, save/RestoreV2_45/shape_and_slices)]] Caused by op u'save/RestoreV2_45', defined`
And now this reads:
`InvalidArgumentError: Cannot assign a device for operation 'save/RestoreV2_45':
Operation was explicitly assigned to '/job:ps/task:1/device:CPU:0'
but available devices are [ /job:localhost/replica:0/task:0/cpu:0 ]. Make
sure the device specification refers to a valid device.
This drops the additional debug_info because this condition is tightly
scoped to lack of devices, and so node information is unnecessary.
Change: 154379360
-rw-r--r-- | tensorflow/core/common_runtime/simple_placer.cc | 15 | ||||
-rw-r--r-- | tensorflow/core/common_runtime/simple_placer_test.cc | 29 | ||||
-rw-r--r-- | tensorflow/python/framework/test_util_test.py | 3 | ||||
-rw-r--r-- | tensorflow/python/training/saver_test.py | 4 |
4 files changed, 17 insertions, 34 deletions
diff --git a/tensorflow/core/common_runtime/simple_placer.cc b/tensorflow/core/common_runtime/simple_placer.cc index f6e6bf0692..5ab3f20648 100644 --- a/tensorflow/core/common_runtime/simple_placer.cc +++ b/tensorflow/core/common_runtime/simple_placer.cc @@ -341,11 +341,10 @@ class ColocationGraph { std::sort(device_names.begin(), device_names.end()); return errors::InvalidArgument( - "Could not satisfy explicit device specification '", - node->def().device(), - "' because no devices matching that specification " - "are registered in this process; available devices: ", - str_util::Join(device_names, ", "), debug_info); + "Operation was explicitly assigned to ", node->def().device(), + " but available devices are [ ", + str_util::Join(device_names, ", "), " ]. Make sure ", + "the device specification refers to a valid device."); } else if (specified_device_name.has_type) { return errors::InvalidArgument( "Could not satisfy explicit device specification '", @@ -741,7 +740,7 @@ Status SimplePlacer::Run() { status = colocation_graph.GetDevicesForNode(node, &devices); if (!status.ok()) { return AttachDef( - errors::InvalidArgument("Cannot assign a device to node '", + errors::InvalidArgument("Cannot assign a device for operation '", node->name(), "': ", status.error_message()), node->def()); } @@ -783,7 +782,7 @@ Status SimplePlacer::Run() { status = colocation_graph.GetDevicesForNode(node, &devices); if (!status.ok()) { return AttachDef( - errors::InvalidArgument("Cannot assign a device to node '", + errors::InvalidArgument("Cannot assign a device for operation '", node->name(), "': ", status.error_message()), node->def()); } @@ -801,7 +800,7 @@ Status SimplePlacer::Run() { return e->dst()->assigned_device_name() == output_device_name; }); - if (consumers_on_same_device && + if (consumers_on_same_device && CanAssignToDevice(output_device_name, devices)) { assigned_device = output_device_name; } diff --git a/tensorflow/core/common_runtime/simple_placer_test.cc b/tensorflow/core/common_runtime/simple_placer_test.cc index c73ed041ed..bd84417b10 100644 --- a/tensorflow/core/common_runtime/simple_placer_test.cc +++ b/tensorflow/core/common_runtime/simple_placer_test.cc @@ -939,10 +939,7 @@ TEST_F(SimplePlacerTest, TestUnknownDevice) { Status s = Place(&g); EXPECT_EQ(error::INVALID_ARGUMENT, s.code()); - EXPECT_TRUE( - StringPiece(s.error_message()) - .contains( - "Could not satisfy explicit device specification '/job:foo'")); + EXPECT_TRUE(StringPiece(s.error_message()).contains("/job:foo")); } // Test that placement fails when the combination of partial @@ -957,10 +954,7 @@ TEST_F(SimplePlacerTest, TestUnknownMergedDevice) { Status s = Place(&g); EXPECT_EQ(error::INVALID_ARGUMENT, s.code()); - EXPECT_TRUE( - StringPiece(s.error_message()) - .contains( - "Could not satisfy explicit device specification '/job:foo'")); + EXPECT_TRUE(StringPiece(s.error_message()).contains("/job:foo")); } // Test that placement fails when the previously-assigned device for a @@ -1107,10 +1101,7 @@ TEST_F(SimplePlacerTest, TestNonexistentGpuNoAllowSoftPlacement) { SessionOptions options; Status s = Place(&g, &options); EXPECT_EQ(error::INVALID_ARGUMENT, s.code()); - EXPECT_TRUE(StringPiece(s.error_message()) - .contains("Could not satisfy explicit " - "device specification " - "'/device:fakegpu:11'")); + EXPECT_TRUE(StringPiece(s.error_message()).contains("/device:fakegpu:11")); } // Test that placement fails when a node requests an explicit device that is not @@ -1127,10 +1118,7 @@ TEST_F(SimplePlacerTest, TestUnsupportedDeviceNoAllowSoftPlacement) { SessionOptions options; Status s = Place(&g, &options); EXPECT_EQ(error::INVALID_ARGUMENT, s.code()); - EXPECT_TRUE(StringPiece(s.error_message()) - .contains("Could not satisfy explicit " - "device specification " - "'/device:fakecpu:0'")); + EXPECT_TRUE(StringPiece(s.error_message()).contains("/device:fakecpu:0")); EXPECT_TRUE( StringPiece(s.error_message()) .contains("no supported kernel for fakecpu devices is available")); @@ -1151,12 +1139,9 @@ TEST_F(SimplePlacerTest, TestNonExistentDevice) { Status s = Place(&g, &options); EXPECT_EQ(error::INVALID_ARGUMENT, s.code()); LOG(WARNING) << s.error_message(); - EXPECT_TRUE( - StringPiece(s.error_message()) - .contains("Could not satisfy explicit device specification " - "'/job:foo/replica:17' " - "because no devices matching that specification are " - "registered in this process")); + EXPECT_TRUE(StringPiece(s.error_message()) + .contains("was explicitly assigned to /job:foo/replica:17 " + "but available devices")); } TEST_F(SimplePlacerTest, TestUnsupportedDeviceAllowSoftPlacement) { diff --git a/tensorflow/python/framework/test_util_test.py b/tensorflow/python/framework/test_util_test.py index cccba94fb0..6129fa2e0d 100644 --- a/tensorflow/python/framework/test_util_test.py +++ b/tensorflow/python/framework/test_util_test.py @@ -252,8 +252,7 @@ class TestUtilTest(test_util.TensorFlowTestCase): self.assertArrayNear(a, b, 0.001) def testForceGPU(self): - with self.assertRaisesRegexp(errors.InvalidArgumentError, - "Cannot assign a device to node"): + with self.assertRaises(errors.InvalidArgumentError): with self.test_session(force_gpu=True): # this relies on us not having a GPU implementation for assert, which # seems sensible diff --git a/tensorflow/python/training/saver_test.py b/tensorflow/python/training/saver_test.py index ec370afee1..d862244f8f 100644 --- a/tensorflow/python/training/saver_test.py +++ b/tensorflow/python/training/saver_test.py @@ -1845,8 +1845,8 @@ class MetaGraphTest(test.TestCase): with session.Session(graph=ops_lib.Graph()) as sess: saver_module.import_meta_graph( meta_graph_def, clear_devices=False, import_scope="new_model") - with self.assertRaisesRegexp(errors_impl.InvalidArgumentError, - "Cannot assign a device to node"): + # Device refers to GPU, which is not available here. + with self.assertRaises(errors_impl.InvalidArgumentError): sess.run(variables.global_variables_initializer()) with session.Session(graph=ops_lib.Graph()) as sess: |