From 154f00e9eacaec5667215784c7601b55024e2f61 Mon Sep 17 00:00:00 2001 From: Antonio Sanchez Date: Thu, 1 Jul 2021 13:41:04 -0700 Subject: Fix inverse nullptr/asan errors for LU. For empty or single-column matrices, the current `PartialPivLU` currently dereferences a `nullptr` or accesses memory out-of-bounds. Here we adjust the checks to avoid this. --- Eigen/src/LU/PartialPivLU.h | 7 ++++++- test/inverse.cpp | 2 ++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/Eigen/src/LU/PartialPivLU.h b/Eigen/src/LU/PartialPivLU.h index 46ffdd320..34aed7249 100644 --- a/Eigen/src/LU/PartialPivLU.h +++ b/Eigen/src/LU/PartialPivLU.h @@ -504,8 +504,13 @@ struct partial_lu_impl template void partial_lu_inplace(MatrixType& lu, TranspositionType& row_transpositions, typename TranspositionType::StorageIndex& nb_transpositions) { + // Special-case of zero matrix. + if (lu.rows() == 0 || lu.cols() == 0) { + nb_transpositions = 0; + return; + } eigen_assert(lu.cols() == row_transpositions.size()); - eigen_assert((&row_transpositions.coeffRef(1)-&row_transpositions.coeffRef(0)) == 1); + eigen_assert(row_transpositions.size() < 2 || (&row_transpositions.coeffRef(1)-&row_transpositions.coeffRef(0)) == 1); partial_lu_impl < typename MatrixType::Scalar, MatrixType::Flags&RowMajorBit?RowMajor:ColMajor, diff --git a/test/inverse.cpp b/test/inverse.cpp index 99f9e0c9b..9cedfa1e1 100644 --- a/test/inverse.cpp +++ b/test/inverse.cpp @@ -135,6 +135,8 @@ EIGEN_DECLARE_TEST(inverse) CALL_SUBTEST_5( inverse(MatrixXf(s,s)) ); TEST_SET_BUT_UNUSED_VARIABLE(s) CALL_SUBTEST_5( inverse_zerosized() ); + CALL_SUBTEST_5( inverse(MatrixXf(0, 0)) ); + CALL_SUBTEST_5( inverse(MatrixXf(1, 1)) ); s = internal::random(25,100); CALL_SUBTEST_6( inverse(MatrixXcd(s,s)) ); -- cgit v1.2.3