aboutsummaryrefslogtreecommitdiffhomepage
path: root/tensorflow/contrib/quantization
diff options
context:
space:
mode:
authorGravatar A. Unique TensorFlower <gardener@tensorflow.org>2016-07-13 11:12:24 -0800
committerGravatar TensorFlower Gardener <gardener@tensorflow.org>2016-07-13 12:18:12 -0700
commit4ad8912996a25136a280312de3801f30dd4d4a74 (patch)
tree2cfa1dbb38401d816c9168210ac92e3efc2a8375 /tensorflow/contrib/quantization
parentef76625346d46ebab8eebdaa3b55b7180e8b1b89 (diff)
Avoid overflow in float-int32 cast when making test case for quantization.
Also fix bug in creation of test for 1-byte values. Change: 127344565
Diffstat (limited to 'tensorflow/contrib/quantization')
-rw-r--r--tensorflow/contrib/quantization/kernels/quantization_utils_test.cc10
1 files changed, 6 insertions, 4 deletions
diff --git a/tensorflow/contrib/quantization/kernels/quantization_utils_test.cc b/tensorflow/contrib/quantization/kernels/quantization_utils_test.cc
index f4a4e8d962..d62610b2ca 100644
--- a/tensorflow/contrib/quantization/kernels/quantization_utils_test.cc
+++ b/tensorflow/contrib/quantization/kernels/quantization_utils_test.cc
@@ -260,18 +260,20 @@ class QuantizationUtilsTest : public ::testing::Test {
FPair(-31.0f, 13.0f)}) {
const float f_min = min_and_max.first;
const float f_max = min_and_max.second;
- const int values_count = sizeof(T) == 256 ? 256 : 50000;
+ const int values_count = sizeof(T) == 1 ? 256 : 50000;
Tensor input(DataTypeToEnum<T>::v(), TensorShape{values_count});
auto input_array = input.flat<T>();
const double q_range =
static_cast<double>(Eigen::NumTraits<T>::highest()) -
Eigen::NumTraits<T>::lowest();
for (int i = 0; i < values_count; ++i) {
- if (sizeof(T) == 256) {
+ if (sizeof(T) == 1) {
input_array(i) = Eigen::NumTraits<T>::lowest() + i;
} else {
- input_array(i) = Eigen::NumTraits<T>::lowest() +
- static_cast<int32>(q_range / values_count * i);
+ int64 offset = static_cast<int64>(q_range / values_count * i);
+ input_array(i) = static_cast<int32>(
+ Eigen::NumTraits<T>::lowest() +
+ std::min<int64>(Eigen::NumTraits<T>::highest(), offset));
}
}