diff options
author | A. Unique TensorFlower <gardener@tensorflow.org> | 2017-12-15 10:09:07 -0800 |
---|---|---|
committer | TensorFlower Gardener <gardener@tensorflow.org> | 2017-12-15 10:13:19 -0800 |
commit | 65c108990382ecfdb9a4a8a58f2303096300e5e0 (patch) | |
tree | a4ae4215512ff2e5070b9477dec99c22d237dee9 | |
parent | 897076aea855c8cfcb8b365689d0a34259c67f6a (diff) |
Fix bfloat16 numerics issues in the tests.
PiperOrigin-RevId: 179207115
-rw-r--r-- | tensorflow/compiler/tests/ftrl_test.py | 40 | ||||
-rw-r--r-- | tensorflow/compiler/tests/momentum_test.py | 23 | ||||
-rw-r--r-- | tensorflow/compiler/tests/unary_ops_test.py | 2 | ||||
-rw-r--r-- | tensorflow/python/framework/test_util.py | 10 |
4 files changed, 39 insertions, 36 deletions
diff --git a/tensorflow/compiler/tests/ftrl_test.py b/tensorflow/compiler/tests/ftrl_test.py index 7e3871312c..f9db4cf201 100644 --- a/tensorflow/compiler/tests/ftrl_test.py +++ b/tensorflow/compiler/tests/ftrl_test.py @@ -161,9 +161,9 @@ class FtrlOptimizerTest(XLATestCase): ftrl_update.run() # Validate updated params - self.assertAllClose( + self.assertAllCloseAccordingToType( np.array([-2.55607247, -3.98729396]), var0.eval(), 1e-5, 1e-5) - self.assertAllClose( + self.assertAllCloseAccordingToType( np.array([-0.28232238, -0.56096673]), var1.eval(), 1e-5, 1e-5) def testFtrlWithL1(self): @@ -189,10 +189,10 @@ class FtrlOptimizerTest(XLATestCase): ftrl_update.run() # Validate updated params - self.assertAllClose(np.array([-7.66718769, -10.91273689]), var0.eval(), - rtol=1e-4) - self.assertAllClose(np.array([-0.93460727, -1.86147261]), var1.eval(), - rtol=1e-4) + self.assertAllCloseAccordingToType( + np.array([-7.66718769, -10.91273689]), var0.eval(), rtol=1e-4) + self.assertAllCloseAccordingToType( + np.array([-0.93460727, -1.86147261]), var1.eval(), rtol=1e-4) def testFtrlWithL1_L2(self): for dtype in self.float_types: @@ -217,10 +217,10 @@ class FtrlOptimizerTest(XLATestCase): ftrl_update.run() # Validate updated params - self.assertAllClose(np.array([-0.24059935, -0.46829352]), var0.eval(), - rtol=1e-5) - self.assertAllClose(np.array([-0.02406147, -0.04830509]), var1.eval(), - rtol=1e-5) + self.assertAllCloseAccordingToType( + np.array([-0.24059935, -0.46829352]), var0.eval(), rtol=1e-5) + self.assertAllCloseAccordingToType( + np.array([-0.02406147, -0.04830509]), var1.eval(), rtol=1e-5) def testFtrlWithL1_L2_L2Shrinkage(self): """Test the new FTRL op with support for l2 shrinkage. @@ -244,18 +244,18 @@ class FtrlOptimizerTest(XLATestCase): ftrl_update = opt.apply_gradients(zip([grads0, grads1], [var0, var1])) variables.global_variables_initializer().run() # Fetch params to validate initial values - self.assertAllClose([1.0, 2.0], var0.eval()) - self.assertAllClose([4.0, 3.0], var1.eval()) + self.assertAllCloseAccordingToType([1.0, 2.0], var0.eval()) + self.assertAllCloseAccordingToType([4.0, 3.0], var1.eval()) # Run 10 steps FTRL for _ in range(10): ftrl_update.run() # Validate updated params - self.assertAllClose(np.array([-0.21931979, -0.40642974]), var0.eval(), - rtol=1e-4) - self.assertAllClose(np.array([-0.0282721, -0.07188385]), var1.eval(), - rtol=1e-4) + self.assertAllCloseAccordingToType( + np.array([-0.21931979, -0.40642974]), var0.eval(), rtol=1e-4) + self.assertAllCloseAccordingToType( + np.array([-0.0282721, -0.07188385]), var1.eval(), rtol=1e-4) # When variables are initialized with Zero, FTRL-Proximal has two properties: # 1. Without L1&L2 but with fixed learning rate, FTRL-Proximal is identical @@ -272,8 +272,8 @@ class FtrlOptimizerTest(XLATestCase): with self.test_session(), self.test_scope(): val2, val3 = self.equivAdagradTest_AdagradPart(steps, dtype) - self.assertAllClose(val0, val2, rtol=1e-4) - self.assertAllClose(val1, val3, rtol=1e-4) + self.assertAllCloseAccordingToType(val0, val2, rtol=1e-4) + self.assertAllCloseAccordingToType(val1, val3, rtol=1e-4) def testEquivGradientDescentwithoutRegularization(self): steps = 5 @@ -284,8 +284,8 @@ class FtrlOptimizerTest(XLATestCase): val2, val3 = self.equivGradientDescentTest_GradientDescentPart( steps, dtype) - self.assertAllClose(val0, val2, rtol=1e-5) - self.assertAllClose(val1, val3, rtol=1e-5) + self.assertAllCloseAccordingToType(val0, val2, rtol=1e-5) + self.assertAllCloseAccordingToType(val1, val3, rtol=1e-5) if __name__ == "__main__": diff --git a/tensorflow/compiler/tests/momentum_test.py b/tensorflow/compiler/tests/momentum_test.py index c00e3035a0..af9394e7d7 100644 --- a/tensorflow/compiler/tests/momentum_test.py +++ b/tensorflow/compiler/tests/momentum_test.py @@ -96,28 +96,27 @@ class MomentumOptimizerTest(XLATestCase): def testNesterovMomentum(self): for dtype in self.float_types: with self.test_session(), self.test_scope(): - var0 = resource_variable_ops.ResourceVariable([1.0, 2.0], dtype=dtype) - var1 = resource_variable_ops.ResourceVariable([3.0, 4.0], dtype=dtype) - var0_np = np.array([1.0, 2.0], dtype=dtype) - var1_np = np.array([3.0, 4.0], dtype=dtype) + var0 = resource_variable_ops.ResourceVariable([0.1, 0.2], dtype=dtype) + var1 = resource_variable_ops.ResourceVariable([0.3, 0.4], dtype=dtype) + var0_np = np.array([0.1, 0.2], dtype=dtype) + var1_np = np.array([0.3, 0.4], dtype=dtype) accum0_np = np.array([0.0, 0.0], dtype=dtype) accum1_np = np.array([0.0, 0.0], dtype=dtype) - cost = 5 * var0 * var0 + 3 * var1 + cost = 0.4 * var0 * var0 + 0.9 * var1 global_step = resource_variable_ops.ResourceVariable( array_ops.zeros([], dtypes.int32), name="global_step") mom_op = momentum_lib.MomentumOptimizer( - learning_rate=2.0, momentum=0.9, use_nesterov=True) + learning_rate=0.1, momentum=0.9, use_nesterov=True) opt_op = mom_op.minimize(cost, global_step, [var0, var1]) variables.global_variables_initializer().run() for _ in range(1, 5): opt_op.run() var0_np, accum0_np = self._update_nesterov_momentum_numpy( - var0_np, accum0_np, var0_np * 10, 2.0, 0.9) - var1_np, accum1_np = self._update_nesterov_momentum_numpy(var1_np, - accum1_np, - 3, 2.0, 0.9) - self.assertAllClose(var0_np, var0.eval()) - self.assertAllClose(var1_np, var1.eval()) + var0_np, accum0_np, var0_np * 0.8, 0.1, 0.9) + var1_np, accum1_np = self._update_nesterov_momentum_numpy( + var1_np, accum1_np, 0.9, 0.1, 0.9) + self.assertAllCloseAccordingToType(var0_np, var0.eval()) + self.assertAllCloseAccordingToType(var1_np, var1.eval()) def testTensorLearningRateAndMomentum(self): for dtype in self.float_types: diff --git a/tensorflow/compiler/tests/unary_ops_test.py b/tensorflow/compiler/tests/unary_ops_test.py index b0623c0fbc..ecba5a4fb0 100644 --- a/tensorflow/compiler/tests/unary_ops_test.py +++ b/tensorflow/compiler/tests/unary_ops_test.py @@ -67,7 +67,7 @@ class UnaryOpsTest(XLATestCase): output = op(pinp) result = session.run(output, {pinp: inp}) if equality_test is None: - equality_test = self.assertAllClose + equality_test = self.assertAllCloseAccordingToType equality_test(result, expected, rtol=rtol, atol=atol) def ListsAreClose(self, result, expected, rtol, atol): diff --git a/tensorflow/python/framework/test_util.py b/tensorflow/python/framework/test_util.py index 509c5ec8d6..8875d45a07 100644 --- a/tensorflow/python/framework/test_util.py +++ b/tensorflow/python/framework/test_util.py @@ -1091,7 +1091,9 @@ class TensorFlowTestCase(googletest.TestCase): float_rtol=1e-6, float_atol=1e-6, half_rtol=1e-3, - half_atol=1e-3): + half_atol=1e-3, + bfloat16_rtol=1e-2, + bfloat16_atol=1e-2): """Like assertAllClose, but also suitable for comparing fp16 arrays. In particular, the tolerance is reduced to 1e-3 if at least @@ -1106,6 +1108,8 @@ class TensorFlowTestCase(googletest.TestCase): float_atol: absolute tolerance for float32. half_rtol: relative tolerance for float16. half_atol: absolute tolerance for float16. + bfloat16_rtol: relative tolerance for bfloat16. + bfloat16_atol: absolute tolerance for bfloat16. """ a = self._GetNdArray(a) b = self._GetNdArray(b) @@ -1119,8 +1123,8 @@ class TensorFlowTestCase(googletest.TestCase): atol = max(atol, half_atol) if (a.dtype == dtypes.bfloat16.as_numpy_dtype or b.dtype == dtypes.bfloat16.as_numpy_dtype): - rtol = max(rtol, half_rtol) - atol = max(atol, half_atol) + rtol = max(rtol, bfloat16_rtol) + atol = max(atol, bfloat16_atol) self.assertAllClose(a, b, rtol=rtol, atol=atol) |