diff options
author | 2016-07-13 11:12:24 -0800 | |
---|---|---|
committer | 2016-07-13 12:18:12 -0700 | |
commit | 4ad8912996a25136a280312de3801f30dd4d4a74 (patch) | |
tree | 2cfa1dbb38401d816c9168210ac92e3efc2a8375 /tensorflow/contrib/quantization | |
parent | ef76625346d46ebab8eebdaa3b55b7180e8b1b89 (diff) |
Avoid overflow in float-int32 cast when making test case for quantization.
Also fix bug in creation of test for 1-byte values.
Change: 127344565
Diffstat (limited to 'tensorflow/contrib/quantization')
-rw-r--r-- | tensorflow/contrib/quantization/kernels/quantization_utils_test.cc | 10 |
1 files changed, 6 insertions, 4 deletions
diff --git a/tensorflow/contrib/quantization/kernels/quantization_utils_test.cc b/tensorflow/contrib/quantization/kernels/quantization_utils_test.cc index f4a4e8d962..d62610b2ca 100644 --- a/tensorflow/contrib/quantization/kernels/quantization_utils_test.cc +++ b/tensorflow/contrib/quantization/kernels/quantization_utils_test.cc @@ -260,18 +260,20 @@ class QuantizationUtilsTest : public ::testing::Test { FPair(-31.0f, 13.0f)}) { const float f_min = min_and_max.first; const float f_max = min_and_max.second; - const int values_count = sizeof(T) == 256 ? 256 : 50000; + const int values_count = sizeof(T) == 1 ? 256 : 50000; Tensor input(DataTypeToEnum<T>::v(), TensorShape{values_count}); auto input_array = input.flat<T>(); const double q_range = static_cast<double>(Eigen::NumTraits<T>::highest()) - Eigen::NumTraits<T>::lowest(); for (int i = 0; i < values_count; ++i) { - if (sizeof(T) == 256) { + if (sizeof(T) == 1) { input_array(i) = Eigen::NumTraits<T>::lowest() + i; } else { - input_array(i) = Eigen::NumTraits<T>::lowest() + - static_cast<int32>(q_range / values_count * i); + int64 offset = static_cast<int64>(q_range / values_count * i); + input_array(i) = static_cast<int32>( + Eigen::NumTraits<T>::lowest() + + std::min<int64>(Eigen::NumTraits<T>::highest(), offset)); } } |