aboutsummaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
Diffstat
-rw-r--r--tensorflow/core/debug/BUILD10
-rw-r--r--tensorflow/core/debug/debug_grpc_testlib.cc23
-rw-r--r--tensorflow/core/debug/debug_io_utils.cc29
-rw-r--r--tensorflow/core/debug/debug_io_utils_test.cc33
-rw-r--r--tensorflow/core/debug/debugger_event_metadata.proto9
-rw-r--r--tensorflow/core/kernels/summary_tensor_op_test.cc9
-rw-r--r--tensorflow/python/ops/summary_ops.py1
-rw-r--r--tensorflow/python/summary/summary.py2
-rw-r--r--tensorflow/python/summary/text_summary.py28
-rw-r--r--tensorflow/python/summary/text_summary_test.py10
-rw-r--r--tensorflow/python/summary/writer/writer.py26
-rw-r--r--tensorflow/python/summary/writer/writer_test.py57
12 files changed, 215 insertions, 22 deletions
diff --git a/tensorflow/core/debug/BUILD b/tensorflow/core/debug/BUILD
index 2fc49d4412..30824a9072 100644
--- a/tensorflow/core/debug/BUILD
+++ b/tensorflow/core/debug/BUILD
@@ -123,6 +123,7 @@ tf_cuda_library(
linkstatic = 1,
deps = [
":debug_service_proto_cc",
+ ":debugger_event_metadata_proto_cc",
"//tensorflow/core:core_cpu_internal",
"//tensorflow/core:framework",
"//tensorflow/core:lib",
@@ -144,6 +145,7 @@ tf_cuda_library(
":debug_graph_utils",
":debug_io_utils",
":debug_service_proto_cc",
+ ":debugger_event_metadata_proto_cc",
"//tensorflow/core:framework",
"//tensorflow/core:lib",
"//tensorflow/core:lib_internal",
@@ -194,6 +196,7 @@ tf_cc_test(
deps = [
":debug_grpc_testlib",
":debug_io_utils",
+ ":debugger_event_metadata_proto_cc",
"//tensorflow/core:core_cpu",
"//tensorflow/core:core_cpu_internal",
"//tensorflow/core:framework",
@@ -204,6 +207,7 @@ tf_cc_test(
"//tensorflow/core:test",
"//tensorflow/core:test_main",
"//tensorflow/core:testlib",
+ "//tensorflow/core/platform/default/build_config:platformlib",
],
)
@@ -247,6 +251,12 @@ tf_cc_test(
],
)
+tf_proto_library_cc(
+ name = "debugger_event_metadata_proto",
+ srcs = ["debugger_event_metadata.proto"],
+ cc_api_version = 2,
+)
+
# TODO(cais): Add the following back in when tfdbg is supported on Android.
# filegroup(
# name = "android_srcs",
diff --git a/tensorflow/core/debug/debug_grpc_testlib.cc b/tensorflow/core/debug/debug_grpc_testlib.cc
index c19842a2f6..7317aa0372 100644
--- a/tensorflow/core/debug/debug_grpc_testlib.cc
+++ b/tensorflow/core/debug/debug_grpc_testlib.cc
@@ -16,10 +16,12 @@ limitations under the License.
#include "tensorflow/core/debug/debug_grpc_testlib.h"
#include "tensorflow/core/debug/debug_graph_utils.h"
+#include "tensorflow/core/debug/debugger_event_metadata.pb.h"
#include "tensorflow/core/framework/summary.pb.h"
#include "tensorflow/core/lib/io/path.h"
#include "tensorflow/core/lib/strings/str_util.h"
#include "tensorflow/core/platform/env.h"
+#include "tensorflow/core/platform/protobuf.h"
#include "tensorflow/core/platform/tracing.h"
namespace tensorflow {
@@ -44,8 +46,6 @@ namespace test {
tensorflow::str_util::Split(val.node_name(), ':');
const string node_name = name_items[0];
- int32 output_slot = 0;
- tensorflow::strings::safe_strto32(name_items[1], &output_slot);
const string debug_op = name_items[2];
const TensorProto& tensor_proto = val.tensor();
@@ -54,9 +54,24 @@ namespace test {
return ::grpc::Status::CANCELLED;
}
- device_names.push_back(val.tag());
+ // Obtain the device name, which is encoded in JSON.
+ third_party::tensorflow::core::debug::DebuggerEventMetadata metadata;
+ for (int i = 0; i < val.metadata().plugin_data_size(); i++) {
+ if (val.metadata().plugin_data(i).plugin_name() != "debugger") {
+ // This plugin data was meant for another plugin.
+ continue;
+ }
+ auto status = tensorflow::protobuf::util::JsonStringToMessage(
+ val.metadata().plugin_data(i).content(), &metadata);
+ if (status.ok()) {
+ // The device name has been determined.
+ break;
+ }
+ }
+
+ device_names.push_back(metadata.device());
node_names.push_back(node_name);
- output_slots.push_back(output_slot);
+ output_slots.push_back(metadata.output_slot());
debug_ops.push_back(debug_op);
debug_tensors.push_back(tensor);
}
diff --git a/tensorflow/core/debug/debug_io_utils.cc b/tensorflow/core/debug/debug_io_utils.cc
index 88dffb934c..69fc367789 100644
--- a/tensorflow/core/debug/debug_io_utils.cc
+++ b/tensorflow/core/debug/debug_io_utils.cc
@@ -26,11 +26,13 @@ limitations under the License.
#pragma comment(lib,"Ws2_32.lib")
#endif
+#include "tensorflow/core/debug/debugger_event_metadata.pb.h"
#include "tensorflow/core/framework/summary.pb.h"
#include "tensorflow/core/lib/hash/hash.h"
#include "tensorflow/core/lib/io/path.h"
#include "tensorflow/core/lib/strings/str_util.h"
#include "tensorflow/core/lib/strings/stringprintf.h"
+#include "tensorflow/core/platform/protobuf.h"
#include "tensorflow/core/util/event.pb.h"
#define GRPC_OSS_UNIMPLEMENTED_ERROR \
@@ -55,7 +57,32 @@ Event WrapTensorAsEvent(const DebugNodeKey& debug_node_key,
// "DebugIdentity", the debug node_name in the Summary proto will be
// "foo/node_a:0:DebugIdentity".
summ_val->set_node_name(debug_node_key.debug_node_name);
- summ_val->set_tag(debug_node_key.device_name);
+
+ // Tag by the node name. This allows TensorBoard to quickly fetch data per op.
+ summ_val->set_tag(debug_node_key.node_name);
+
+ // Store data within debugger metadata to be stored for each event.
+ third_party::tensorflow::core::debug::DebuggerEventMetadata metadata;
+ metadata.set_device(debug_node_key.device_name);
+ metadata.set_output_slot(debug_node_key.output_slot);
+
+ // Encode the data in JSON.
+ string json_output;
+ tensorflow::protobuf::util::JsonPrintOptions json_options;
+ json_options.always_print_primitive_fields = true;
+ auto status = tensorflow::protobuf::util::MessageToJsonString(
+ metadata, &json_output, json_options);
+ if (status.ok()) {
+ // Store summary metadata. Set the plugin to use this data as "debugger".
+ SummaryMetadata::PluginData* plugin_data =
+ summ_val->mutable_metadata()->add_plugin_data();
+ plugin_data->set_plugin_name("debugger");
+ plugin_data->set_content(json_output);
+ } else {
+ LOG(WARNING) << "Failed to convert DebuggerEventMetadata proto to JSON. "
+ << "The debug_node_name is " << debug_node_key.debug_node_name
+ << ".";
+ }
if (tensor.dtype() == DT_STRING) {
// Treat DT_STRING specially, so that tensor_util.MakeNdarray can convert
diff --git a/tensorflow/core/debug/debug_io_utils_test.cc b/tensorflow/core/debug/debug_io_utils_test.cc
index 35c95fb98c..08ef4001bc 100644
--- a/tensorflow/core/debug/debug_io_utils_test.cc
+++ b/tensorflow/core/debug/debug_io_utils_test.cc
@@ -15,6 +15,7 @@ limitations under the License.
#include "tensorflow/core/debug/debug_io_utils.h"
+#include "tensorflow/core/debug/debugger_event_metadata.pb.h"
#include "tensorflow/core/framework/summary.pb.h"
#include "tensorflow/core/framework/tensor_testutil.h"
#include "tensorflow/core/lib/core/notification.h"
@@ -124,10 +125,18 @@ TEST_F(DebugIOUtilsTest, DumpStringTensorToFileSunnyDay) {
ASSERT_GE(wall_time, event.wall_time());
ASSERT_EQ(1, event.summary().value().size());
- ASSERT_EQ(kDebugNodeKey.device_name, event.summary().value(0).tag());
+ ASSERT_EQ(kDebugNodeKey.node_name, event.summary().value(0).tag());
ASSERT_EQ(kDebugNodeKey.debug_node_name,
event.summary().value(0).node_name());
+ // Determine and validate some information from the metadata.
+ third_party::tensorflow::core::debug::DebuggerEventMetadata metadata;
+ auto status = tensorflow::protobuf::util::JsonStringToMessage(
+ event.summary().value(0).metadata().plugin_data(0).content(), &metadata);
+ ASSERT_TRUE(status.ok());
+ ASSERT_EQ(kDebugNodeKey.device_name, metadata.device());
+ ASSERT_EQ(kDebugNodeKey.output_slot, metadata.output_slot());
+
Tensor b_prime(DT_STRING);
ASSERT_TRUE(b_prime.FromProto(event.summary().value(0).tensor()));
@@ -229,10 +238,19 @@ TEST_F(DebugIOUtilsTest, PublishTensorToMultipleFileURLs) {
ASSERT_GE(wall_time, event.wall_time());
ASSERT_EQ(1, event.summary().value().size());
- ASSERT_EQ(kDebugNodeKey.device_name, event.summary().value(0).tag());
+ ASSERT_EQ(kDebugNodeKey.node_name, event.summary().value(0).tag());
ASSERT_EQ(kDebugNodeKey.debug_node_name,
event.summary().value(0).node_name());
+ // Determine and validate some information from the metadata.
+ third_party::tensorflow::core::debug::DebuggerEventMetadata metadata;
+ auto status = tensorflow::protobuf::util::JsonStringToMessage(
+ event.summary().value(0).metadata().plugin_data(0).content(),
+ &metadata);
+ ASSERT_TRUE(status.ok());
+ ASSERT_EQ(kDebugNodeKey.device_name, metadata.device());
+ ASSERT_EQ(kDebugNodeKey.output_slot, metadata.output_slot());
+
Tensor a_prime(DT_FLOAT);
ASSERT_TRUE(a_prime.FromProto(event.summary().value(0).tensor()));
@@ -333,10 +351,19 @@ TEST_F(DebugIOUtilsTest, PublishTensorConcurrentlyToPartiallyOverlappingPaths) {
ASSERT_GE(wall_time, event.wall_time());
ASSERT_EQ(1, event.summary().value().size());
- ASSERT_EQ(kDebugNodeKey.device_name, event.summary().value(0).tag());
+ ASSERT_EQ(kDebugNodeKey.node_name, event.summary().value(0).tag());
ASSERT_EQ(kDebugNodeKey.debug_node_name,
event.summary().value(0).node_name());
+ // Determine and validate some information from the metadata.
+ third_party::tensorflow::core::debug::DebuggerEventMetadata metadata;
+ auto status = tensorflow::protobuf::util::JsonStringToMessage(
+ event.summary().value(0).metadata().plugin_data(0).content(),
+ &metadata);
+ ASSERT_TRUE(status.ok());
+ ASSERT_EQ(kDebugNodeKey.device_name, metadata.device());
+ ASSERT_EQ(kDebugNodeKey.output_slot, metadata.output_slot());
+
Tensor a_prime(DT_FLOAT);
ASSERT_TRUE(a_prime.FromProto(event.summary().value(0).tensor()));
diff --git a/tensorflow/core/debug/debugger_event_metadata.proto b/tensorflow/core/debug/debugger_event_metadata.proto
new file mode 100644
index 0000000000..44ef305f5a
--- /dev/null
+++ b/tensorflow/core/debug/debugger_event_metadata.proto
@@ -0,0 +1,9 @@
+syntax = "proto3";
+
+package third_party.tensorflow.core.debug;
+
+// Encapsulates per-event data related to debugging.
+message DebuggerEventMetadata {
+ string device = 1;
+ int32 output_slot = 2;
+};
diff --git a/tensorflow/core/kernels/summary_tensor_op_test.cc b/tensorflow/core/kernels/summary_tensor_op_test.cc
index 0006a71bd7..010ff443fa 100644
--- a/tensorflow/core/kernels/summary_tensor_op_test.cc
+++ b/tensorflow/core/kernels/summary_tensor_op_test.cc
@@ -85,8 +85,15 @@ TEST_F(SummaryTensorOpV2Test, BasicPluginData) {
ASSERT_EQ(0, out_tensor->dims());
Summary summary;
ParseProtoUnlimited(&summary, out_tensor->scalar<string>()());
-
ASSERT_EQ(1, summary.value_size());
+
+ // Check the content of the tensor stored in the summary.
+ Tensor string_content_tensor;
+ CHECK(string_content_tensor.FromProto(summary.value(0).tensor()));
+ ASSERT_EQ("some string tensor content",
+ string_content_tensor.scalar<string>()());
+
+ // Check plugin-related data.
ASSERT_EQ("tag_foo", summary.value(0).tag());
ASSERT_EQ(2, summary.value(0).metadata().plugin_data_size());
ASSERT_EQ("foo", summary.value(0).metadata().plugin_data(0).plugin_name());
diff --git a/tensorflow/python/ops/summary_ops.py b/tensorflow/python/ops/summary_ops.py
index 4ad0862dcc..3d19bc2526 100644
--- a/tensorflow/python/ops/summary_ops.py
+++ b/tensorflow/python/ops/summary_ops.py
@@ -131,7 +131,6 @@ def _tensor_summary_v2( # pylint: disable=invalid-name
val = gen_logging_ops._tensor_summary_v2(
tensor=tensor,
tag=tag,
- description="",
name=scope,
serialized_summary_metadata=serialized_summary_metadata)
summary_op_util.collect(val, collections, [ops.GraphKeys.SUMMARIES])
diff --git a/tensorflow/python/summary/summary.py b/tensorflow/python/summary/summary.py
index 7ff01a51f3..f3600793a6 100644
--- a/tensorflow/python/summary/summary.py
+++ b/tensorflow/python/summary/summary.py
@@ -20,6 +20,7 @@ See the @{$python/summary} guide.
@@FileWriter
@@FileWriterCache
@@tensor_summary
+@@_tensor_summary_v2
@@scalar
@@histogram
@@audio
@@ -28,6 +29,7 @@ See the @{$python/summary} guide.
@@merge
@@merge_all
@@get_summary_description
+@@PluginAsset
@@get_plugin_asset
@@get_all_plugin_assets
"""
diff --git a/tensorflow/python/summary/text_summary.py b/tensorflow/python/summary/text_summary.py
index 52bc913b2a..2132dc6eb8 100644
--- a/tensorflow/python/summary/text_summary.py
+++ b/tensorflow/python/summary/text_summary.py
@@ -23,13 +23,26 @@ from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
+from collections import namedtuple
import json
+from tensorflow.core.framework import summary_pb2
from tensorflow.python.framework import dtypes
-from tensorflow.python.ops.summary_ops import tensor_summary
+from tensorflow.python.ops.summary_ops import _tensor_summary_v2
from tensorflow.python.summary import plugin_asset
+from tensorflow.python.util import deprecation
+PLUGIN_NAME = "text"
+# Contains event-related data specific to the text plugin.
+_TextPluginData = namedtuple("_TextPluginData", [])
+
+
+@deprecation.deprecated_args(
+ "2017-06-13",
+ "collections is deprecated. Instead of using collections to associate "
+ "plugins to events, add a PluginData field to the SummaryMetadata of a "
+ "Value proto.", "collections")
def text_summary(name, tensor, collections=None):
"""Summarizes textual data.
@@ -60,9 +73,16 @@ def text_summary(name, tensor, collections=None):
raise ValueError("Expected tensor %s to have dtype string, got %s" %
(tensor.name, tensor.dtype))
- t_summary = tensor_summary(name, tensor, collections=collections)
- text_assets = plugin_asset.get_plugin_asset(TextSummaryPluginAsset)
- text_assets.register_tensor(t_summary.op.name)
+ summary_metadata = summary_pb2.SummaryMetadata()
+ text_plugin_data = _TextPluginData()
+ data_dict = text_plugin_data._asdict() # pylint: disable=protected-access
+ summary_metadata.plugin_data.add(
+ plugin_name=PLUGIN_NAME, content=json.dumps(data_dict))
+ t_summary = _tensor_summary_v2(
+ name=name,
+ tensor=tensor,
+ summary_metadata=summary_metadata,
+ collections=collections)
return t_summary
diff --git a/tensorflow/python/summary/text_summary_test.py b/tensorflow/python/summary/text_summary_test.py
index 31009702ca..4d357918f6 100644
--- a/tensorflow/python/summary/text_summary_test.py
+++ b/tensorflow/python/summary/text_summary_test.py
@@ -17,7 +17,6 @@ from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
-from tensorflow.python.framework import ops as framework_ops
from tensorflow.python.framework import test_util
from tensorflow.python.ops import array_ops
from tensorflow.python.platform import googletest
@@ -43,16 +42,11 @@ class TextPluginTest(test_util.TensorFlowTestCase):
# The API accepts vectors.
arr = array_ops.constant(["one", "two", "three"])
summ = text_summary.text_summary("foo", arr)
- self.assertEqual(summ.op.type, "TensorSummary")
+ self.assertEqual(summ.op.type, "TensorSummaryV2")
# the API accepts scalars
summ = text_summary.text_summary("foo", array_ops.constant("one"))
- self.assertEqual(summ.op.type, "TensorSummary")
-
- def testTextSummaryCollections(self):
- text_summary.text_summary("bar", array_ops.constant("2"), collections=[])
- summaries = framework_ops.get_collection(framework_ops.GraphKeys.SUMMARIES)
- self.assertEqual(len(summaries), 0)
+ self.assertEqual(summ.op.type, "TensorSummaryV2")
if __name__ == "__main__":
diff --git a/tensorflow/python/summary/writer/writer.py b/tensorflow/python/summary/writer/writer.py
index 05f97fb284..8ce49d623d 100644
--- a/tensorflow/python/summary/writer/writer.py
+++ b/tensorflow/python/summary/writer/writer.py
@@ -86,6 +86,14 @@ class SummaryToEventTransformer(object):
meta_graph.create_meta_graph_def(graph_def=graph_def or
maybe_graph_as_def))
+ # This set contains tags of Summary Values that have been encountered
+ # already. The motivation here is that the SummaryWriter only keeps the
+ # metadata property (which is a SummaryMetadata proto) of the first Summary
+ # Value encountered for each tag. The SummaryWriter strips away the
+ # SummaryMetadata for all subsequent Summary Values with tags seen
+ # previously. This saves space.
+ self._seen_summary_tags = set()
+
def add_summary(self, summary, global_step=None):
"""Adds a `Summary` protocol buffer to the event file.
@@ -108,6 +116,24 @@ class SummaryToEventTransformer(object):
summ = summary_pb2.Summary()
summ.ParseFromString(summary)
summary = summ
+
+ # We strip metadata from values with tags that we have seen before in order
+ # to save space - we just store the metadata on the first value with a
+ # specific tag.
+ for value in summary.value:
+ if not value.metadata:
+ continue
+
+ if value.tag in self._seen_summary_tags:
+ # This tag has been encountered before. Strip the metadata.
+ value.ClearField("metadata")
+ continue
+
+ # We encounter a value with a tag we have not encountered previously. And
+ # it has metadata. Remember to strip metadata from future values with this
+ # tag string.
+ self._seen_summary_tags.add(value.tag)
+
event = event_pb2.Event(summary=summary)
self._add_event(event, global_step)
diff --git a/tensorflow/python/summary/writer/writer_test.py b/tensorflow/python/summary/writer/writer_test.py
index 8c34eb82e3..3d27b11cb9 100644
--- a/tensorflow/python/summary/writer/writer_test.py
+++ b/tensorflow/python/summary/writer/writer_test.py
@@ -317,6 +317,63 @@ class SummaryWriterTestCase(test.TestCase):
# We should be done.
self.assertRaises(StopIteration, lambda: next(rr))
+ def testPluginMetadataStrippedFromSubsequentEvents(self):
+ test_dir = self._CleanTestDir("basics")
+ sw = writer.FileWriter(test_dir)
+
+ sw.add_session_log(event_pb2.SessionLog(status=SessionLog.START), 1)
+
+ # We add 2 summaries with the same tags. They both have metadata. The writer
+ # should strip the metadata from the second one.
+ value = summary_pb2.Summary.Value(tag="foo", simple_value=10.0)
+ value.metadata.plugin_data.add(plugin_name="bar", content="... content ...")
+ sw.add_summary(summary_pb2.Summary(value=[value]), 10)
+ value = summary_pb2.Summary.Value(tag="foo", simple_value=10.0)
+ value.metadata.plugin_data.add(plugin_name="bar", content="... content ...")
+ sw.add_summary(summary_pb2.Summary(value=[value]), 10)
+
+ sw.close()
+ rr = self._EventsReader(test_dir)
+
+ # The first event should list the file_version.
+ ev = next(rr)
+ self._assertRecent(ev.wall_time)
+ self.assertEquals("brain.Event:2", ev.file_version)
+
+ # The next event should be the START message.
+ ev = next(rr)
+ self._assertRecent(ev.wall_time)
+ self.assertEquals(1, ev.step)
+ self.assertEquals(SessionLog.START, ev.session_log.status)
+
+ # This is the first event with tag foo. It should contain SummaryMetadata.
+ ev = next(rr)
+ self.assertProtoEquals("""
+ value {
+ tag: "foo"
+ simple_value: 10.0
+ metadata {
+ plugin_data {
+ plugin_name: "bar"
+ content: "... content ..."
+ }
+ }
+ }
+ """, ev.summary)
+
+ # This is the second event with tag foo. It should lack SummaryMetadata
+ # because the file writer should have stripped it.
+ ev = next(rr)
+ self.assertProtoEquals("""
+ value {
+ tag: "foo"
+ simple_value: 10.0
+ }
+ """, ev.summary)
+
+ # We should be done.
+ self.assertRaises(StopIteration, lambda: next(rr))
+
def testFileWriterWithSuffix(self):
test_dir = self._CleanTestDir("test_suffix")
sw = writer.FileWriter(test_dir, filename_suffix="_test_suffix")
generated by cgit v1.2.3 (git 2.39.1) at 2024-06-18 11:53:11 +0000