Make TensorBoard purging of orphaned or out-of-order events optional.

Now it is controlled by a flag.
Default behavior is the same.

Will be quite useful for debugging missing data type issues.
Change: 115623272

1 files changed, 164 insertions, 68 deletions

diff --git a/tensorflow/python/summary/event_accumulator_test.py b/tensorflow/python/summary/event_accumulator_test.py
index ada914bdf3..9a92008a0d 100644
--- a/tensorflow/python/summary/event_accumulator_test.py
+++ b/tensorflow/python/summary/event_accumulator_test.py
@@ -41,34 +41,50 @@ class _EventGenerator(object):
 
   def AddScalar(self, tag, wall_time=0, step=0, value=0):
     event = tf.Event(
-        wall_time=wall_time, step=step,
-        summary=tf.Summary(
-            value=[tf.Summary.Value(tag=tag, simple_value=value)]
-        )
-    )
+        wall_time=wall_time,
+        step=step,
+        summary=tf.Summary(value=[tf.Summary.Value(tag=tag,
+                                                   simple_value=value)]))
     self.AddEvent(event)
 
-  def AddHistogram(self, tag, wall_time=0, step=0, hmin=1, hmax=2, hnum=3,
-                   hsum=4, hsum_squares=5, hbucket_limit=None, hbucket=None):
-    histo = tf.HistogramProto(min=hmin, max=hmax, num=hnum, sum=hsum,
+  def AddHistogram(self,
+                   tag,
+                   wall_time=0,
+                   step=0,
+                   hmin=1,
+                   hmax=2,
+                   hnum=3,
+                   hsum=4,
+                   hsum_squares=5,
+                   hbucket_limit=None,
+                   hbucket=None):
+    histo = tf.HistogramProto(min=hmin,
+                              max=hmax,
+                              num=hnum,
+                              sum=hsum,
                               sum_squares=hsum_squares,
                               bucket_limit=hbucket_limit,
                               bucket=hbucket)
-    event = tf.Event(
-        wall_time=wall_time,
-        step=step,
-        summary=tf.Summary(value=[tf.Summary.Value(tag=tag, histo=histo)]))
+    event = tf.Event(wall_time=wall_time,
+                     step=step,
+                     summary=tf.Summary(value=[tf.Summary.Value(tag=tag,
+                                                                histo=histo)]))
     self.AddEvent(event)
 
-  def AddImage(self, tag, wall_time=0, step=0, encoded_image_string=b'imgstr',
-               width=150, height=100):
+  def AddImage(self,
+               tag,
+               wall_time=0,
+               step=0,
+               encoded_image_string=b'imgstr',
+               width=150,
+               height=100):
     image = tf.Summary.Image(encoded_image_string=encoded_image_string,
-                             width=width, height=height)
-    event = tf.Event(
-        wall_time=wall_time,
-        step=step,
-        summary=tf.Summary(
-            value=[tf.Summary.Value(tag=tag, image=image)]))
+                             width=width,
+                             height=height)
+    event = tf.Event(wall_time=wall_time,
+                     step=step,
+                     summary=tf.Summary(value=[tf.Summary.Value(tag=tag,
+                                                                image=image)]))
     self.AddEvent(event)
 
   def AddEvent(self, event):
@@ -103,9 +119,11 @@ class MockingEventAccumulatorTest(EventAccumulatorTest):
                   ea.GRAPH: False}
     self._real_constructor = ea.EventAccumulator
     self._real_generator = ea._GeneratorFromPath
+
     def _FakeAccumulatorConstructor(generator, *args, **kwargs):
       ea._GeneratorFromPath = lambda x: generator
       return self._real_constructor(generator, *args, **kwargs)
+
     ea.EventAccumulator = _FakeAccumulatorConstructor
 
   def tearDown(self):
@@ -129,13 +147,13 @@ class MockingEventAccumulatorTest(EventAccumulatorTest):
     gen.AddImage('im2')
     acc = ea.EventAccumulator(gen)
     acc.Reload()
-    self.assertTagsEqual(
-        acc.Tags(), {
-            ea.IMAGES: ['im1', 'im2'],
-            ea.SCALARS: ['s1', 's2'],
-            ea.HISTOGRAMS: ['hst1', 'hst2'],
-            ea.COMPRESSED_HISTOGRAMS: ['hst1', 'hst2'],
-            ea.GRAPH: False})
+    self.assertTagsEqual(acc.Tags(), {
+        ea.IMAGES: ['im1', 'im2'],
+        ea.SCALARS: ['s1', 's2'],
+        ea.HISTOGRAMS: ['hst1', 'hst2'],
+        ea.COMPRESSED_HISTOGRAMS: ['hst1', 'hst2'],
+        ea.GRAPH: False
+    })
 
   def testReload(self):
     gen = _EventGenerator()
@@ -155,7 +173,8 @@ class MockingEventAccumulatorTest(EventAccumulatorTest):
         ea.SCALARS: ['s1', 's2'],
         ea.HISTOGRAMS: ['hst1', 'hst2'],
         ea.COMPRESSED_HISTOGRAMS: ['hst1', 'hst2'],
-        ea.GRAPH: False})
+        ea.GRAPH: False
+    })
 
   def testScalars(self):
     gen = _EventGenerator()
@@ -172,18 +191,42 @@ class MockingEventAccumulatorTest(EventAccumulatorTest):
     gen = _EventGenerator()
     acc = ea.EventAccumulator(gen)
 
-    val1 = ea.HistogramValue(min=1, max=2, num=3, sum=4, sum_squares=5,
-                             bucket_limit=[1, 2, 3], bucket=[0, 3, 0])
-    val2 = ea.HistogramValue(min=-2, max=3, num=4, sum=5, sum_squares=6,
-                             bucket_limit=[2, 3, 4], bucket=[1, 3, 0])
+    val1 = ea.HistogramValue(min=1,
+                             max=2,
+                             num=3,
+                             sum=4,
+                             sum_squares=5,
+                             bucket_limit=[1, 2, 3],
+                             bucket=[0, 3, 0])
+    val2 = ea.HistogramValue(min=-2,
+                             max=3,
+                             num=4,
+                             sum=5,
+                             sum_squares=6,
+                             bucket_limit=[2, 3, 4],
+                             bucket=[1, 3, 0])
 
     hst1 = ea.HistogramEvent(wall_time=1, step=10, histogram_value=val1)
     hst2 = ea.HistogramEvent(wall_time=2, step=12, histogram_value=val2)
-    gen.AddHistogram('hst1', wall_time=1, step=10, hmin=1, hmax=2, hnum=3,
-                     hsum=4, hsum_squares=5, hbucket_limit=[1, 2, 3],
+    gen.AddHistogram('hst1',
+                     wall_time=1,
+                     step=10,
+                     hmin=1,
+                     hmax=2,
+                     hnum=3,
+                     hsum=4,
+                     hsum_squares=5,
+                     hbucket_limit=[1, 2, 3],
                      hbucket=[0, 3, 0])
-    gen.AddHistogram('hst2', wall_time=2, step=12, hmin=-2, hmax=3, hnum=4,
-                     hsum=5, hsum_squares=6, hbucket_limit=[2, 3, 4],
+    gen.AddHistogram('hst2',
+                     wall_time=2,
+                     step=12,
+                     hmin=-2,
+                     hmax=3,
+                     hnum=4,
+                     hsum=5,
+                     hsum_squares=6,
+                     hbucket_limit=[2, 3, 4],
                      hbucket=[1, 3, 0])
     acc.Reload()
     self.assertEqual(acc.Histograms('hst1'), [hst1])
@@ -193,17 +236,32 @@ class MockingEventAccumulatorTest(EventAccumulatorTest):
     gen = _EventGenerator()
     acc = ea.EventAccumulator(gen, compression_bps=(0, 2500, 5000, 7500, 10000))
 
-    gen.AddHistogram('hst1', wall_time=1, step=10, hmin=1, hmax=2, hnum=3,
-                     hsum=4, hsum_squares=5, hbucket_limit=[1, 2, 3],
+    gen.AddHistogram('hst1',
+                     wall_time=1,
+                     step=10,
+                     hmin=1,
+                     hmax=2,
+                     hnum=3,
+                     hsum=4,
+                     hsum_squares=5,
+                     hbucket_limit=[1, 2, 3],
                      hbucket=[0, 3, 0])
-    gen.AddHistogram('hst2', wall_time=2, step=12, hmin=-2, hmax=3, hnum=4,
-                     hsum=5, hsum_squares=6, hbucket_limit=[2, 3, 4],
+    gen.AddHistogram('hst2',
+                     wall_time=2,
+                     step=12,
+                     hmin=-2,
+                     hmax=3,
+                     hnum=4,
+                     hsum=5,
+                     hsum_squares=6,
+                     hbucket_limit=[2, 3, 4],
                      hbucket=[1, 3, 0])
     acc.Reload()
 
     # Create the expected values after compressing hst1
-    expected_vals1 = [ea.CompressedHistogramValue(bp, val) for bp, val in [(
-        0, 1.0), (2500, 1.25), (5000, 1.5), (7500, 1.75), (10000, 2.0)]]
+    expected_vals1 = [ea.CompressedHistogramValue(bp, val)
+                      for bp, val in [(0, 1.0), (2500, 1.25), (5000, 1.5), (
+                          7500, 1.75), (10000, 2.0)]]
     expected_cmphst1 = ea.CompressedHistogramEvent(
         wall_time=1,
         step=10,
@@ -225,8 +283,8 @@ class MockingEventAccumulatorTest(EventAccumulatorTest):
   def testPercentile(self):
 
     def AssertExpectedForBps(bps, expected):
-      output = acc._Percentile(
-          bps, bucket_limit, cumsum_weights, histo_min, histo_max, histo_num)
+      output = acc._Percentile(bps, bucket_limit, cumsum_weights, histo_min,
+                               histo_max, histo_num)
       self.assertAlmostEqual(expected, output)
 
     gen = _EventGenerator()
@@ -311,14 +369,28 @@ class MockingEventAccumulatorTest(EventAccumulatorTest):
   def testImages(self):
     gen = _EventGenerator()
     acc = ea.EventAccumulator(gen)
-    im1 = ea.ImageEvent(wall_time=1, step=10, encoded_image_string=b'big',
-                        width=400, height=300)
-    im2 = ea.ImageEvent(wall_time=2, step=12, encoded_image_string=b'small',
-                        width=40, height=30)
-    gen.AddImage('im1', wall_time=1, step=10, encoded_image_string=b'big',
-                 width=400, height=300)
-    gen.AddImage('im2', wall_time=2, step=12, encoded_image_string=b'small',
-                 width=40, height=30)
+    im1 = ea.ImageEvent(wall_time=1,
+                        step=10,
+                        encoded_image_string=b'big',
+                        width=400,
+                        height=300)
+    im2 = ea.ImageEvent(wall_time=2,
+                        step=12,
+                        encoded_image_string=b'small',
+                        width=40,
+                        height=30)
+    gen.AddImage('im1',
+                 wall_time=1,
+                 step=10,
+                 encoded_image_string=b'big',
+                 width=400,
+                 height=300)
+    gen.AddImage('im2',
+                 wall_time=2,
+                 step=12,
+                 encoded_image_string=b'small',
+                 width=40,
+                 height=30)
     acc.Reload()
     self.assertEqual(acc.Images('im1'), [im1])
     self.assertEqual(acc.Images('im2'), [im2])
@@ -359,8 +431,7 @@ class MockingEventAccumulatorTest(EventAccumulatorTest):
     gen = _EventGenerator()
     acc = ea.EventAccumulator(gen)
     gen.AddScalar('s1', wall_time=1, step=10, value=20)
-    gen.AddEvent(tf.Event(
-        wall_time=2, step=20, file_version='nots2'))
+    gen.AddEvent(tf.Event(wall_time=2, step=20, file_version='nots2'))
     gen.AddScalar('s3', wall_time=3, step=100, value=1)
     gen.AddHistogram('hst1')
     gen.AddImage('im1')
@@ -371,7 +442,8 @@ class MockingEventAccumulatorTest(EventAccumulatorTest):
         ea.SCALARS: ['s1', 's3'],
         ea.HISTOGRAMS: ['hst1'],
         ea.COMPRESSED_HISTOGRAMS: ['hst1'],
-        ea.GRAPH: False})
+        ea.GRAPH: False
+    })
 
   def testExpiredDataDiscardedAfterRestartForFileVersionLessThan2(self):
     """Tests that events are discarded after a restart is detected.
@@ -404,6 +476,28 @@ class MockingEventAccumulatorTest(EventAccumulatorTest):
     ## Check that we have discarded 200 and 300 from s1
     self.assertEqual([x.step for x in acc.Scalars('s1')], [100, 101, 201, 301])
 
+  def testOrphanedDataNotDiscardedIfFlagUnset(self):
+    """Tests that events are not discarded if purge_orphaned_data is false.
+    """
+    gen = _EventGenerator()
+    acc = ea.EventAccumulator(gen, purge_orphaned_data=False)
+
+    gen.AddEvent(tf.Event(wall_time=0, step=0, file_version='brain.Event:1'))
+    gen.AddScalar('s1', wall_time=1, step=100, value=20)
+    gen.AddScalar('s1', wall_time=1, step=200, value=20)
+    gen.AddScalar('s1', wall_time=1, step=300, value=20)
+    acc.Reload()
+    ## Check that number of items are what they should be
+    self.assertEqual([x.step for x in acc.Scalars('s1')], [100, 200, 300])
+
+    gen.AddScalar('s1', wall_time=1, step=101, value=20)
+    gen.AddScalar('s1', wall_time=1, step=201, value=20)
+    gen.AddScalar('s1', wall_time=1, step=301, value=20)
+    acc.Reload()
+    ## Check that we have discarded 200 and 300 from s1
+    self.assertEqual([x.step for x in acc.Scalars('s1')], [100, 200, 300, 101,
+                                                           201, 301])
+
   def testEventsDiscardedPerTagAfterRestartForFileVersionLessThan2(self):
     """Tests that event discards after restart, only affect the misordered tag.
 
@@ -486,6 +580,7 @@ class RealisticEventAccumulatorTest(EventAccumulatorTest):
 
   def testScalarsRealistically(self):
     """Test accumulator by writing values and then reading them."""
+
     def FakeScalarSummary(tag, value):
       value = tf.Summary.Value(tag=tag, simple_value=value)
       summary = tf.Summary(value=[value])
@@ -504,9 +599,9 @@ class RealisticEventAccumulatorTest(EventAccumulatorTest):
     # Write a bunch of events using the writer
     for i in xrange(30):
       summ_id = FakeScalarSummary('id', i)
-      summ_sq = FakeScalarSummary('sq', i*i)
-      writer.add_summary(summ_id, i*5)
-      writer.add_summary(summ_sq, i*5)
+      summ_sq = FakeScalarSummary('sq', i * i)
+      writer.add_summary(summ_id, i * 5)
+      writer.add_summary(summ_sq, i * 5)
     writer.flush()
 
     # Verify that we can load those events properly
@@ -517,23 +612,24 @@ class RealisticEventAccumulatorTest(EventAccumulatorTest):
         ea.SCALARS: ['id', 'sq'],
         ea.HISTOGRAMS: [],
         ea.COMPRESSED_HISTOGRAMS: [],
-        ea.GRAPH: True})
+        ea.GRAPH: True
+    })
     id_events = acc.Scalars('id')
     sq_events = acc.Scalars('sq')
     self.assertEqual(30, len(id_events))
     self.assertEqual(30, len(sq_events))
     for i in xrange(30):
-      self.assertEqual(i*5, id_events[i].step)
-      self.assertEqual(i*5, sq_events[i].step)
+      self.assertEqual(i * 5, id_events[i].step)
+      self.assertEqual(i * 5, sq_events[i].step)
       self.assertEqual(i, id_events[i].value)
-      self.assertEqual(i*i, sq_events[i].value)
+      self.assertEqual(i * i, sq_events[i].value)
 
     # Write a few more events to test incremental reloading
     for i in xrange(30, 40):
       summ_id = FakeScalarSummary('id', i)
-      summ_sq = FakeScalarSummary('sq', i*i)
-      writer.add_summary(summ_id, i*5)
-      writer.add_summary(summ_sq, i*5)
+      summ_sq = FakeScalarSummary('sq', i * i)
+      writer.add_summary(summ_id, i * 5)
+      writer.add_summary(summ_sq, i * 5)
     writer.flush()
 
     # Verify we can now see all of the data
@@ -541,10 +637,10 @@ class RealisticEventAccumulatorTest(EventAccumulatorTest):
     self.assertEqual(40, len(id_events))
     self.assertEqual(40, len(sq_events))
     for i in xrange(40):
-      self.assertEqual(i*5, id_events[i].step)
-      self.assertEqual(i*5, sq_events[i].step)
+      self.assertEqual(i * 5, id_events[i].step)
+      self.assertEqual(i * 5, sq_events[i].step)
       self.assertEqual(i, id_events[i].value)
-      self.assertEqual(i*i, sq_events[i].value)
+      self.assertEqual(i * i, sq_events[i].value)
     self.assertProtoEquals(graph_def, acc.Graph())