aboutsummaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
authorGravatar Vijay Vasudevan <vrv@google.com>2017-04-26 19:02:10 -0800
committerGravatar TensorFlower Gardener <gardener@tensorflow.org>2017-04-26 20:31:03 -0700
commit95ca363c6caf5d152eef594c7fc3703a5d490070 (patch)
treee275d9b938d8c162d2a63e2677c80d590260c567
parentbe43153b21fba4280f0f2a016242614a21d115f3 (diff)
Make error message when explicitly adding an invalid device more clear.
Before, you would get something like: `InvalidArgumentError: Cannot assign a device to node 'save/RestoreV2_45': Could not satisfy explicit device specification '/job:ps/task:1/device:CPU:0' because no devices matching that specification are registered in this process; available devices: /job:localhost/replica:0/task:0/cpu:0 [[Node: save/RestoreV2_45 = RestoreV2[dtypes=[DT_FLOAT], _device="/job:ps/task:1/device:CPU:0"](save/Const, save/RestoreV2_45/tensor_names, save/RestoreV2_45/shape_and_slices)]] Caused by op u'save/RestoreV2_45', defined` And now this reads: `InvalidArgumentError: Cannot assign a device for operation 'save/RestoreV2_45': Operation was explicitly assigned to '/job:ps/task:1/device:CPU:0' but available devices are [ /job:localhost/replica:0/task:0/cpu:0 ]. Make sure the device specification refers to a valid device. This drops the additional debug_info because this condition is tightly scoped to lack of devices, and so node information is unnecessary. Change: 154379360
-rw-r--r--tensorflow/core/common_runtime/simple_placer.cc15
-rw-r--r--tensorflow/core/common_runtime/simple_placer_test.cc29
-rw-r--r--tensorflow/python/framework/test_util_test.py3
-rw-r--r--tensorflow/python/training/saver_test.py4
4 files changed, 17 insertions, 34 deletions
diff --git a/tensorflow/core/common_runtime/simple_placer.cc b/tensorflow/core/common_runtime/simple_placer.cc
index f6e6bf0692..5ab3f20648 100644
--- a/tensorflow/core/common_runtime/simple_placer.cc
+++ b/tensorflow/core/common_runtime/simple_placer.cc
@@ -341,11 +341,10 @@ class ColocationGraph {
std::sort(device_names.begin(), device_names.end());
return errors::InvalidArgument(
- "Could not satisfy explicit device specification '",
- node->def().device(),
- "' because no devices matching that specification "
- "are registered in this process; available devices: ",
- str_util::Join(device_names, ", "), debug_info);
+ "Operation was explicitly assigned to ", node->def().device(),
+ " but available devices are [ ",
+ str_util::Join(device_names, ", "), " ]. Make sure ",
+ "the device specification refers to a valid device.");
} else if (specified_device_name.has_type) {
return errors::InvalidArgument(
"Could not satisfy explicit device specification '",
@@ -741,7 +740,7 @@ Status SimplePlacer::Run() {
status = colocation_graph.GetDevicesForNode(node, &devices);
if (!status.ok()) {
return AttachDef(
- errors::InvalidArgument("Cannot assign a device to node '",
+ errors::InvalidArgument("Cannot assign a device for operation '",
node->name(), "': ", status.error_message()),
node->def());
}
@@ -783,7 +782,7 @@ Status SimplePlacer::Run() {
status = colocation_graph.GetDevicesForNode(node, &devices);
if (!status.ok()) {
return AttachDef(
- errors::InvalidArgument("Cannot assign a device to node '",
+ errors::InvalidArgument("Cannot assign a device for operation '",
node->name(), "': ", status.error_message()),
node->def());
}
@@ -801,7 +800,7 @@ Status SimplePlacer::Run() {
return e->dst()->assigned_device_name() == output_device_name;
});
- if (consumers_on_same_device &&
+ if (consumers_on_same_device &&
CanAssignToDevice(output_device_name, devices)) {
assigned_device = output_device_name;
}
diff --git a/tensorflow/core/common_runtime/simple_placer_test.cc b/tensorflow/core/common_runtime/simple_placer_test.cc
index c73ed041ed..bd84417b10 100644
--- a/tensorflow/core/common_runtime/simple_placer_test.cc
+++ b/tensorflow/core/common_runtime/simple_placer_test.cc
@@ -939,10 +939,7 @@ TEST_F(SimplePlacerTest, TestUnknownDevice) {
Status s = Place(&g);
EXPECT_EQ(error::INVALID_ARGUMENT, s.code());
- EXPECT_TRUE(
- StringPiece(s.error_message())
- .contains(
- "Could not satisfy explicit device specification '/job:foo'"));
+ EXPECT_TRUE(StringPiece(s.error_message()).contains("/job:foo"));
}
// Test that placement fails when the combination of partial
@@ -957,10 +954,7 @@ TEST_F(SimplePlacerTest, TestUnknownMergedDevice) {
Status s = Place(&g);
EXPECT_EQ(error::INVALID_ARGUMENT, s.code());
- EXPECT_TRUE(
- StringPiece(s.error_message())
- .contains(
- "Could not satisfy explicit device specification '/job:foo'"));
+ EXPECT_TRUE(StringPiece(s.error_message()).contains("/job:foo"));
}
// Test that placement fails when the previously-assigned device for a
@@ -1107,10 +1101,7 @@ TEST_F(SimplePlacerTest, TestNonexistentGpuNoAllowSoftPlacement) {
SessionOptions options;
Status s = Place(&g, &options);
EXPECT_EQ(error::INVALID_ARGUMENT, s.code());
- EXPECT_TRUE(StringPiece(s.error_message())
- .contains("Could not satisfy explicit "
- "device specification "
- "'/device:fakegpu:11'"));
+ EXPECT_TRUE(StringPiece(s.error_message()).contains("/device:fakegpu:11"));
}
// Test that placement fails when a node requests an explicit device that is not
@@ -1127,10 +1118,7 @@ TEST_F(SimplePlacerTest, TestUnsupportedDeviceNoAllowSoftPlacement) {
SessionOptions options;
Status s = Place(&g, &options);
EXPECT_EQ(error::INVALID_ARGUMENT, s.code());
- EXPECT_TRUE(StringPiece(s.error_message())
- .contains("Could not satisfy explicit "
- "device specification "
- "'/device:fakecpu:0'"));
+ EXPECT_TRUE(StringPiece(s.error_message()).contains("/device:fakecpu:0"));
EXPECT_TRUE(
StringPiece(s.error_message())
.contains("no supported kernel for fakecpu devices is available"));
@@ -1151,12 +1139,9 @@ TEST_F(SimplePlacerTest, TestNonExistentDevice) {
Status s = Place(&g, &options);
EXPECT_EQ(error::INVALID_ARGUMENT, s.code());
LOG(WARNING) << s.error_message();
- EXPECT_TRUE(
- StringPiece(s.error_message())
- .contains("Could not satisfy explicit device specification "
- "'/job:foo/replica:17' "
- "because no devices matching that specification are "
- "registered in this process"));
+ EXPECT_TRUE(StringPiece(s.error_message())
+ .contains("was explicitly assigned to /job:foo/replica:17 "
+ "but available devices"));
}
TEST_F(SimplePlacerTest, TestUnsupportedDeviceAllowSoftPlacement) {
diff --git a/tensorflow/python/framework/test_util_test.py b/tensorflow/python/framework/test_util_test.py
index cccba94fb0..6129fa2e0d 100644
--- a/tensorflow/python/framework/test_util_test.py
+++ b/tensorflow/python/framework/test_util_test.py
@@ -252,8 +252,7 @@ class TestUtilTest(test_util.TensorFlowTestCase):
self.assertArrayNear(a, b, 0.001)
def testForceGPU(self):
- with self.assertRaisesRegexp(errors.InvalidArgumentError,
- "Cannot assign a device to node"):
+ with self.assertRaises(errors.InvalidArgumentError):
with self.test_session(force_gpu=True):
# this relies on us not having a GPU implementation for assert, which
# seems sensible
diff --git a/tensorflow/python/training/saver_test.py b/tensorflow/python/training/saver_test.py
index ec370afee1..d862244f8f 100644
--- a/tensorflow/python/training/saver_test.py
+++ b/tensorflow/python/training/saver_test.py
@@ -1845,8 +1845,8 @@ class MetaGraphTest(test.TestCase):
with session.Session(graph=ops_lib.Graph()) as sess:
saver_module.import_meta_graph(
meta_graph_def, clear_devices=False, import_scope="new_model")
- with self.assertRaisesRegexp(errors_impl.InvalidArgumentError,
- "Cannot assign a device to node"):
+ # Device refers to GPU, which is not available here.
+ with self.assertRaises(errors_impl.InvalidArgumentError):
sess.run(variables.global_variables_initializer())
with session.Session(graph=ops_lib.Graph()) as sess: