From f41959ccb2d9d4c722fe8fc3351401d53bcf4900 Mon Sep 17 00:00:00 2001 From: Manjunath Kudlur Date: Fri, 6 Nov 2015 16:27:58 -0800 Subject: TensorFlow: Initial commit of TensorFlow library. TensorFlow is an open source software library for numerical computation using data flow graphs. Base CL: 107276108 --- .../python/summary/event_accumulator_test.py | 422 +++++++++++++++++++++ 1 file changed, 422 insertions(+) create mode 100644 tensorflow/python/summary/event_accumulator_test.py (limited to 'tensorflow/python/summary/event_accumulator_test.py') diff --git a/tensorflow/python/summary/event_accumulator_test.py b/tensorflow/python/summary/event_accumulator_test.py new file mode 100644 index 0000000000..c8de80ccba --- /dev/null +++ b/tensorflow/python/summary/event_accumulator_test.py @@ -0,0 +1,422 @@ +import os + +import tensorflow.python.platform + +import tensorflow as tf + +from tensorflow.python.platform import gfile +from tensorflow.python.summary import event_accumulator as ea + + +class _EventGenerator(object): + + def __init__(self): + self.items = [] + + def Load(self): + while self.items: + yield self.items.pop(0) + + def AddScalar(self, tag, wall_time=0, step=0, value=0): + event = tf.Event( + wall_time=wall_time, step=step, + summary=tf.Summary( + value=[tf.Summary.Value(tag=tag, simple_value=value)] + ) + ) + self.AddEvent(event) + + def AddHistogram(self, tag, wall_time=0, step=0, hmin=1, hmax=2, hnum=3, + hsum=4, hsum_squares=5, hbucket_limit=None, hbucket=None): + histo = tf.HistogramProto(min=hmin, max=hmax, num=hnum, sum=hsum, + sum_squares=hsum_squares, + bucket_limit=hbucket_limit, + bucket=hbucket) + event = tf.Event( + wall_time=wall_time, + step=step, + summary=tf.Summary(value=[tf.Summary.Value(tag=tag, histo=histo)])) + self.AddEvent(event) + + def AddImage(self, tag, wall_time=0, step=0, encoded_image_string='imgstr', + width=150, height=100): + image = tf.Summary.Image(encoded_image_string=encoded_image_string, + width=width, height=height) + event = tf.Event( + wall_time=wall_time, + step=step, + summary=tf.Summary( + value=[tf.Summary.Value(tag=tag, image=image)])) + self.AddEvent(event) + + def AddEvent(self, event): + self.items.append(event) + + +class EventAccumulatorTest(tf.test.TestCase): + + def assertTagsEqual(self, tags1, tags2): + # Make sure the two dictionaries have the same keys. + self.assertItemsEqual(tags1, tags2) + # Additionally, make sure each key in the dictionary maps to the same value. + for key in tags1: + if isinstance(tags1[key], list): + # We don't care about the order of the values in lists, thus asserting + # only if the items are equal. + self.assertItemsEqual(tags1[key], tags2[key]) + else: + # Make sure the values are equal. + self.assertEqual(tags1[key], tags2[key]) + + +class MockingEventAccumulatorTest(EventAccumulatorTest): + + def setUp(self): + super(MockingEventAccumulatorTest, self).setUp() + self.empty = {ea.IMAGES: [], + ea.SCALARS: [], + ea.HISTOGRAMS: [], + ea.COMPRESSED_HISTOGRAMS: [], + ea.GRAPH: False} + self._real_constructor = ea.EventAccumulator + self._real_generator = ea._GeneratorFromPath + def _FakeAccumulatorConstructor(generator, *args, **kwargs): + ea._GeneratorFromPath = lambda x: generator + return self._real_constructor(generator, *args, **kwargs) + ea.EventAccumulator = _FakeAccumulatorConstructor + + def tearDown(self): + ea.EventAccumulator = self._real_constructor + ea._GeneratorFromPath = self._real_generator + + def testEmptyAccumulator(self): + gen = _EventGenerator() + x = ea.EventAccumulator(gen) + x.Reload() + self.assertEqual(x.Tags(), self.empty) + + def testTags(self): + gen = _EventGenerator() + gen.AddScalar('sv1') + gen.AddScalar('sv2') + gen.AddHistogram('hst1') + gen.AddHistogram('hst2') + gen.AddImage('im1') + gen.AddImage('im2') + acc = ea.EventAccumulator(gen) + acc.Reload() + self.assertTagsEqual( + acc.Tags(), { + ea.IMAGES: ['im1', 'im2'], + ea.SCALARS: ['sv1', 'sv2'], + ea.HISTOGRAMS: ['hst1', 'hst2'], + ea.COMPRESSED_HISTOGRAMS: ['hst1', 'hst2'], + ea.GRAPH: False}) + + def testReload(self): + gen = _EventGenerator() + acc = ea.EventAccumulator(gen) + acc.Reload() + self.assertEqual(acc.Tags(), self.empty) + gen.AddScalar('sv1') + gen.AddScalar('sv2') + gen.AddHistogram('hst1') + gen.AddHistogram('hst2') + gen.AddImage('im1') + gen.AddImage('im2') + self.assertEqual(acc.Tags(), self.empty) + acc.Reload() + self.assertTagsEqual(acc.Tags(), { + ea.IMAGES: ['im1', 'im2'], + ea.SCALARS: ['sv1', 'sv2'], + ea.HISTOGRAMS: ['hst1', 'hst2'], + ea.COMPRESSED_HISTOGRAMS: ['hst1', 'hst2'], + ea.GRAPH: False}) + + def testScalars(self): + gen = _EventGenerator() + acc = ea.EventAccumulator(gen) + sv1 = ea.ScalarEvent(wall_time=1, step=10, value=32) + sv2 = ea.ScalarEvent(wall_time=2, step=12, value=64) + gen.AddScalar('sv1', wall_time=1, step=10, value=32) + gen.AddScalar('sv2', wall_time=2, step=12, value=64) + acc.Reload() + self.assertEqual(acc.Scalars('sv1'), [sv1]) + self.assertEqual(acc.Scalars('sv2'), [sv2]) + + def testHistograms(self): + gen = _EventGenerator() + acc = ea.EventAccumulator(gen) + + val1 = ea.HistogramValue(min=1, max=2, num=3, sum=4, sum_squares=5, + bucket_limit=[1, 2, 3], bucket=[0, 3, 0]) + val2 = ea.HistogramValue(min=-2, max=3, num=4, sum=5, sum_squares=6, + bucket_limit=[2, 3, 4], bucket=[1, 3, 0]) + + hst1 = ea.HistogramEvent(wall_time=1, step=10, histogram_value=val1) + hst2 = ea.HistogramEvent(wall_time=2, step=12, histogram_value=val2) + gen.AddHistogram('hst1', wall_time=1, step=10, hmin=1, hmax=2, hnum=3, + hsum=4, hsum_squares=5, hbucket_limit=[1, 2, 3], + hbucket=[0, 3, 0]) + gen.AddHistogram('hst2', wall_time=2, step=12, hmin=-2, hmax=3, hnum=4, + hsum=5, hsum_squares=6, hbucket_limit=[2, 3, 4], + hbucket=[1, 3, 0]) + acc.Reload() + self.assertEqual(acc.Histograms('hst1'), [hst1]) + self.assertEqual(acc.Histograms('hst2'), [hst2]) + + def testCompressedHistograms(self): + gen = _EventGenerator() + acc = ea.EventAccumulator(gen, compression_bps=(0, 2500, 5000, 7500, 10000)) + + gen.AddHistogram('hst1', wall_time=1, step=10, hmin=1, hmax=2, hnum=3, + hsum=4, hsum_squares=5, hbucket_limit=[1, 2, 3], + hbucket=[0, 3, 0]) + gen.AddHistogram('hst2', wall_time=2, step=12, hmin=-2, hmax=3, hnum=4, + hsum=5, hsum_squares=6, hbucket_limit=[2, 3, 4], + hbucket=[1, 3, 0]) + acc.Reload() + + # Create the expected values after compressing hst1 + expected_vals1 = [ea.CompressedHistogramValue(bp, val) for bp, val in [( + 0, 1.0), (2500, 1.25), (5000, 1.5), (7500, 1.75), (10000, 2.0)]] + expected_cmphst1 = ea.CompressedHistogramEvent( + wall_time=1, + step=10, + compressed_histogram_values=expected_vals1) + self.assertEqual(acc.CompressedHistograms('hst1'), [expected_cmphst1]) + + # Create the expected values after compressing hst2 + expected_vals2 = [ + ea.CompressedHistogramValue(bp, val) + for bp, val in [(0, -2), (2500, 2), (5000, 2 + float(1) / 3), ( + 7500, 2 + float(2) / 3), (10000, 3)] + ] + expected_cmphst2 = ea.CompressedHistogramEvent( + wall_time=2, + step=12, + compressed_histogram_values=expected_vals2) + self.assertEqual(acc.CompressedHistograms('hst2'), [expected_cmphst2]) + + def testPercentile(self): + + def AssertExpectedForBps(bps, expected): + output = acc._Percentile( + bps, bucket_limit, cumsum_weights, histo_min, histo_max, histo_num) + self.assertAlmostEqual(expected, output) + + gen = _EventGenerator() + acc = ea.EventAccumulator(gen) + + bucket_limit = [1, 2, 3, 4] + histo_num = 100 + + ## All weights in the first bucket + cumsum_weights = [10000, 10000, 10000, 10000] + histo_min = -1 + histo_max = .9 + AssertExpectedForBps(0, histo_min) + AssertExpectedForBps(2500, acc._Remap(2500, 0, 10000, histo_min, histo_max)) + AssertExpectedForBps(5000, acc._Remap(5000, 0, 10000, histo_min, histo_max)) + AssertExpectedForBps(7500, acc._Remap(7500, 0, 10000, histo_min, histo_max)) + AssertExpectedForBps(10000, histo_max) + + ## All weights in second bucket + cumsum_weights = [0, 10000, 10000, 10000] + histo_min = 1.1 + histo_max = 1.8 + AssertExpectedForBps(0, histo_min) + AssertExpectedForBps(2500, acc._Remap(2500, 0, 10000, histo_min, histo_max)) + AssertExpectedForBps(5000, acc._Remap(5000, 0, 10000, histo_min, histo_max)) + AssertExpectedForBps(7500, acc._Remap(7500, 0, 10000, histo_min, histo_max)) + AssertExpectedForBps(10000, histo_max) + + ## All weights in the last bucket + cumsum_weights = [0, 0, 0, 10000] + histo_min = 3.1 + histo_max = 3.6 + AssertExpectedForBps(0, histo_min) + AssertExpectedForBps(2500, acc._Remap(2500, 0, 10000, histo_min, histo_max)) + AssertExpectedForBps(5000, acc._Remap(5000, 0, 10000, histo_min, histo_max)) + AssertExpectedForBps(7500, acc._Remap(7500, 0, 10000, histo_min, histo_max)) + AssertExpectedForBps(10000, histo_max) + + ## Weights distributed between two buckets + cumsum_weights = [0, 4000, 10000, 10000] + histo_min = 1.1 + histo_max = 2.9 + AssertExpectedForBps(0, histo_min) + AssertExpectedForBps(2500, acc._Remap(2500, 0, 4000, histo_min, + bucket_limit[1])) + AssertExpectedForBps(5000, acc._Remap(5000, 4000, 10000, bucket_limit[1], + histo_max)) + AssertExpectedForBps(7500, acc._Remap(7500, 4000, 10000, bucket_limit[1], + histo_max)) + AssertExpectedForBps(10000, histo_max) + + ## Weights distributed between all buckets + cumsum_weights = [1000, 4000, 8000, 10000] + histo_min = -1 + histo_max = 3.9 + AssertExpectedForBps(0, histo_min) + AssertExpectedForBps(2500, acc._Remap(2500, 1000, 4000, bucket_limit[0], + bucket_limit[1])) + AssertExpectedForBps(5000, acc._Remap(5000, 4000, 8000, bucket_limit[1], + bucket_limit[2])) + AssertExpectedForBps(7500, acc._Remap(7500, 4000, 8000, bucket_limit[1], + bucket_limit[2])) + AssertExpectedForBps(9000, acc._Remap(9000, 8000, 10000, bucket_limit[2], + histo_max)) + AssertExpectedForBps(10000, histo_max) + + ## Most weight in first bucket + cumsum_weights = [9000, 10000, 10000, 10000] + histo_min = -1 + histo_max = 1.1 + AssertExpectedForBps(0, histo_min) + AssertExpectedForBps(2500, acc._Remap(2500, 0, 9000, histo_min, + bucket_limit[0])) + AssertExpectedForBps(5000, acc._Remap(5000, 0, 9000, histo_min, + bucket_limit[0])) + AssertExpectedForBps(7500, acc._Remap(7500, 0, 9000, histo_min, + bucket_limit[0])) + AssertExpectedForBps(9500, acc._Remap(9500, 9000, 10000, bucket_limit[0], + histo_max)) + AssertExpectedForBps(10000, histo_max) + + def testImages(self): + gen = _EventGenerator() + acc = ea.EventAccumulator(gen) + im1 = ea.ImageEvent(wall_time=1, step=10, encoded_image_string='big', + width=400, height=300) + im2 = ea.ImageEvent(wall_time=2, step=12, encoded_image_string='small', + width=40, height=30) + gen.AddImage('im1', wall_time=1, step=10, encoded_image_string='big', + width=400, height=300) + gen.AddImage('im2', wall_time=2, step=12, encoded_image_string='small', + width=40, height=30) + acc.Reload() + self.assertEqual(acc.Images('im1'), [im1]) + self.assertEqual(acc.Images('im2'), [im2]) + + def testActivation(self): + gen = _EventGenerator() + acc = ea.EventAccumulator(gen) + self.assertFalse(acc._activated) + with self.assertRaises(RuntimeError): + acc.Tags() + with self.assertRaises(RuntimeError): + acc.Scalars('sv1') + acc.Reload() + self.assertTrue(acc._activated) + acc._activated = False + + def testKeyError(self): + gen = _EventGenerator() + acc = ea.EventAccumulator(gen) + acc.Reload() + with self.assertRaises(KeyError): + acc.Scalars('sv1') + with self.assertRaises(KeyError): + acc.Scalars('hst1') + with self.assertRaises(KeyError): + acc.Scalars('im1') + with self.assertRaises(KeyError): + acc.Histograms('sv1') + with self.assertRaises(KeyError): + acc.Histograms('im1') + with self.assertRaises(KeyError): + acc.Images('sv1') + with self.assertRaises(KeyError): + acc.Images('hst1') + + def testNonValueEvents(self): + """Tests that non-value events in the generator don't cause early exits.""" + gen = _EventGenerator() + acc = ea.EventAccumulator(gen) + gen.AddScalar('sv1', wall_time=1, step=10, value=20) + gen.AddEvent(tf.Event( + wall_time=2, step=20, file_version='notsv2')) + gen.AddScalar('sv3', wall_time=3, step=100, value=1) + gen.AddHistogram('hst1') + gen.AddImage('im1') + + acc.Reload() + self.assertTagsEqual(acc.Tags(), { + ea.IMAGES: ['im1'], + ea.SCALARS: ['sv1', 'sv3'], + ea.HISTOGRAMS: ['hst1'], + ea.COMPRESSED_HISTOGRAMS: ['hst1'], + ea.GRAPH: False}) + + +class RealisticEventAccumulatorTest(EventAccumulatorTest): + + def setUp(self): + super(RealisticEventAccumulatorTest, self).setUp() + + def testScalarsRealistically(self): + """Test accumulator by writing values and then reading them.""" + def FakeScalarSummary(tag, value): + value = tf.Summary.Value(tag=tag, simple_value=value) + summary = tf.Summary(value=[value]) + return summary + + directory = os.path.join(self.get_temp_dir(), 'values_dir') + if gfile.IsDirectory(directory): + gfile.DeleteRecursively(directory) + gfile.MkDir(directory) + + writer = tf.train.SummaryWriter(directory, max_queue=100) + graph_def = tf.GraphDef(node=[tf.NodeDef(name='A', op='Mul')]) + # Add a graph to the summary writer. + writer.add_graph(graph_def) + + # Write a bunch of events using the writer + for i in xrange(30): + summ_id = FakeScalarSummary('id', i) + summ_sq = FakeScalarSummary('sq', i*i) + writer.add_summary(summ_id, i*5) + writer.add_summary(summ_sq, i*5) + writer.flush() + + # Verify that we can load those events properly + acc = ea.EventAccumulator(directory) + acc.Reload() + self.assertTagsEqual(acc.Tags(), { + ea.IMAGES: [], + ea.SCALARS: ['id', 'sq'], + ea.HISTOGRAMS: [], + ea.COMPRESSED_HISTOGRAMS: [], + ea.GRAPH: True}) + id_events = acc.Scalars('id') + sq_events = acc.Scalars('sq') + self.assertEqual(30, len(id_events)) + self.assertEqual(30, len(sq_events)) + for i in xrange(30): + self.assertEqual(i*5, id_events[i].step) + self.assertEqual(i*5, sq_events[i].step) + self.assertEqual(i, id_events[i].value) + self.assertEqual(i*i, sq_events[i].value) + + # Write a few more events to test incremental reloading + for i in xrange(30, 40): + summ_id = FakeScalarSummary('id', i) + summ_sq = FakeScalarSummary('sq', i*i) + writer.add_summary(summ_id, i*5) + writer.add_summary(summ_sq, i*5) + writer.flush() + + # Verify we can now see all of the data + acc.Reload() + self.assertEqual(40, len(id_events)) + self.assertEqual(40, len(sq_events)) + for i in xrange(40): + self.assertEqual(i*5, id_events[i].step) + self.assertEqual(i*5, sq_events[i].step) + self.assertEqual(i, id_events[i].value) + self.assertEqual(i*i, sq_events[i].value) + + +if __name__ == '__main__': + tf.test.main() -- cgit v1.2.3