aboutsummaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
authorGravatar Justin Lebar <jlebar@google.com>2018-07-13 15:55:05 -0700
committerGravatar TensorFlower Gardener <gardener@tensorflow.org>2018-07-13 15:58:13 -0700
commitb3137565ac28ca6cd9f17342cb4213206e7ec5c8 (patch)
treec56d09197e79fc3228efd7c64066e8b69f0ddad8
parent0cc772c00fdffad5d8416fcde6b9fd3f566cca8e (diff)
[XLA] Update bad-ptxas-version warning to warn for anything less than 9.2.88.
We have hit another ptxas bug, which appears to be fixed in 9.2.88. We're no longer testing 8.x, so we can't say those versions work either. Just warn if it's less than 9.2.88. Also update the warning to suggest that people can cherry-pick the new ptxas if they can't upgrade to 9.2.88 properly. PiperOrigin-RevId: 204541857
-rw-r--r--tensorflow/compiler/xla/service/gpu/gpu_compiler.cc24
1 files changed, 17 insertions, 7 deletions
diff --git a/tensorflow/compiler/xla/service/gpu/gpu_compiler.cc b/tensorflow/compiler/xla/service/gpu/gpu_compiler.cc
index e1da8d940c..6a441548ca 100644
--- a/tensorflow/compiler/xla/service/gpu/gpu_compiler.cc
+++ b/tensorflow/compiler/xla/service/gpu/gpu_compiler.cc
@@ -357,13 +357,19 @@ void WarnIfBadPtxasVersion(const string& ptxas_path) {
// ptxas 9.0 before 9.0.276 and ptxas 9.1 before 9.1.121 miscompile some
// address calculations with large offsets (e.g. "load ptr + large_constant"),
// b/70245379.
- if ((vmaj == 9 && vmin == 0 && vdot < 276) ||
- (vmaj == 9 && vmin == 1 && vdot < 121)) {
- LOG(WARNING) << "*** WARNING *** You are using ptxas " << vmaj << "."
- << vmin << "." << vdot
- << ", which is in range [9.0.0, 9.0.276) + [9.1.0, 9.1.121). "
- "These versions are known to miscompile XLA code, leading "
- "to incorrect results or invalid-address errors.";
+ //
+ // ptxas 9.1.121 miscompiles some large multioutput fusions, again in a way
+ // that appears related to address calculations. ptxas 9.2.88 appears to
+ // work, as far as we can tell.
+ if ((vmaj < 9 || vmin < 2 || vdot < 88)) {
+ LOG(WARNING)
+ << "*** WARNING *** You are using ptxas " << vmaj << "." << vmin << "."
+ << vdot
+ << ", which older than 9.2.88. XLA doesn't support ptxas 8.x, and "
+ "ptxas 9.x before 9.2.88 is known to miscompile XLA code, leading "
+ "to incorrect results or invalid-address errors.\n\nYou do not need "
+ "to update to CUDA 9.2.88; cherry-picking the ptxas binary is "
+ "sufficient.";
}
}
@@ -391,6 +397,10 @@ void WarnIfBadDriverJITVersion() {
// - 384.x before 384.108
// - 387.x before 387.40
// - 390.x before 390.10.
+ //
+ // TODO(jlebar): This list does not cover the address-calculation bug we've
+ // observed in ptxas 9.1.121. Need to get a new safe range from nvidia
+ // corresponding to ptxas >= 9.2.88.
auto vmaj = std::get<0>(version);
auto vmin = std::get<1>(version);
if ((vmaj == 384 && vmin < 108) || //