diff options
author | Antonio Sanchez <cantonios@google.com> | 2020-12-02 14:00:57 -0800 |
---|---|---|
committer | Antonio Sanchez <cantonios@google.com> | 2020-12-04 10:16:29 -0800 |
commit | e2f21465fea76a80966f12a20d0be36597f19b44 (patch) | |
tree | 1ae9b0e3ae489b028902166a343f796d196fde82 /unsupported/Eigen | |
parent | 305b8bd2777bda99f65791468f305b76021bf579 (diff) |
Special function implementations for half/bfloat16 packets.
Current implementations fail to consider half-float packets, only
half-float scalars. Added specializations for packets on AVX, AVX512 and
NEON. Added tests to `special_packetmath`.
The current `special_functions` tests would fail for half and bfloat16 due to
lack of precision. The NEON tests also fail with precision issues and
due to different handling of `sqrt(inf)`, so special functions bessel, ndtri
have been disabled.
Tested with AVX, AVX512.
Diffstat (limited to 'unsupported/Eigen')
-rw-r--r-- | unsupported/Eigen/SpecialFunctions | 19 | ||||
-rw-r--r-- | unsupported/Eigen/src/SpecialFunctions/BesselFunctionsImpl.h | 132 | ||||
-rw-r--r-- | unsupported/Eigen/src/SpecialFunctions/BesselFunctionsPacketMath.h | 36 | ||||
-rw-r--r-- | unsupported/Eigen/src/SpecialFunctions/SpecialFunctionsImpl.h | 5 | ||||
-rw-r--r-- | unsupported/Eigen/src/SpecialFunctions/arch/AVX/BesselFunctions.h | 46 | ||||
-rw-r--r-- | unsupported/Eigen/src/SpecialFunctions/arch/AVX/SpecialFunctions.h | 16 | ||||
-rw-r--r-- | unsupported/Eigen/src/SpecialFunctions/arch/AVX512/BesselFunctions.h | 46 | ||||
-rw-r--r-- | unsupported/Eigen/src/SpecialFunctions/arch/AVX512/SpecialFunctions.h | 16 | ||||
-rw-r--r-- | unsupported/Eigen/src/SpecialFunctions/arch/GPU/SpecialFunctions.h (renamed from unsupported/Eigen/src/SpecialFunctions/arch/GPU/GpuSpecialFunctions.h) | 0 | ||||
-rw-r--r-- | unsupported/Eigen/src/SpecialFunctions/arch/NEON/BesselFunctions.h | 54 | ||||
-rw-r--r-- | unsupported/Eigen/src/SpecialFunctions/arch/NEON/SpecialFunctions.h | 34 |
11 files changed, 309 insertions, 95 deletions
diff --git a/unsupported/Eigen/SpecialFunctions b/unsupported/Eigen/SpecialFunctions index dda6618de..f6a2460e6 100644 --- a/unsupported/Eigen/SpecialFunctions +++ b/unsupported/Eigen/SpecialFunctions @@ -61,23 +61,36 @@ namespace Eigen { } #include "src/SpecialFunctions/BesselFunctionsImpl.h" -#include "src/SpecialFunctions/BesselFunctionsPacketMath.h" #include "src/SpecialFunctions/BesselFunctionsBFloat16.h" #include "src/SpecialFunctions/BesselFunctionsHalf.h" +#include "src/SpecialFunctions/BesselFunctionsPacketMath.h" #include "src/SpecialFunctions/BesselFunctionsFunctors.h" #include "src/SpecialFunctions/BesselFunctionsArrayAPI.h" #include "src/SpecialFunctions/SpecialFunctionsImpl.h" #if defined(EIGEN_HIPCC) #include "src/SpecialFunctions/HipVectorCompatibility.h" #endif -#include "src/SpecialFunctions/SpecialFunctionsPacketMath.h" #include "src/SpecialFunctions/SpecialFunctionsBFloat16.h" #include "src/SpecialFunctions/SpecialFunctionsHalf.h" +#include "src/SpecialFunctions/SpecialFunctionsPacketMath.h" #include "src/SpecialFunctions/SpecialFunctionsFunctors.h" #include "src/SpecialFunctions/SpecialFunctionsArrayAPI.h" +#if defined EIGEN_VECTORIZE_AVX512 + #include "src/SpecialFunctions/arch/AVX/BesselFunctions.h" + #include "src/SpecialFunctions/arch/AVX/SpecialFunctions.h" + #include "src/SpecialFunctions/arch/AVX512/BesselFunctions.h" + #include "src/SpecialFunctions/arch/AVX512/SpecialFunctions.h" +#elif defined EIGEN_VECTORIZE_AVX + #include "src/SpecialFunctions/arch/AVX/BesselFunctions.h" + #include "src/SpecialFunctions/arch/AVX/SpecialFunctions.h" +#elif defined EIGEN_VECTORIZE_NEON + #include "src/SpecialFunctions/arch/NEON/BesselFunctions.h" + #include "src/SpecialFunctions/arch/NEON/SpecialFunctions.h" +#endif + #if defined EIGEN_VECTORIZE_GPU - #include "src/SpecialFunctions/arch/GPU/GpuSpecialFunctions.h" + #include "src/SpecialFunctions/arch/GPU/SpecialFunctions.h" #endif namespace Eigen { diff --git a/unsupported/Eigen/src/SpecialFunctions/BesselFunctionsImpl.h b/unsupported/Eigen/src/SpecialFunctions/BesselFunctionsImpl.h index a9b6ad940..24812be1b 100644 --- a/unsupported/Eigen/src/SpecialFunctions/BesselFunctionsImpl.h +++ b/unsupported/Eigen/src/SpecialFunctions/BesselFunctionsImpl.h @@ -46,7 +46,7 @@ struct bessel_i0e_retval { typedef Scalar type; }; -template <typename T, typename ScalarType> +template <typename T, typename ScalarType = typename unpacket_traits<T>::type> struct generic_i0e { EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE T run(const T&) { @@ -201,11 +201,11 @@ struct generic_i0e<T, double> { } }; -template <typename Scalar> +template <typename T> struct bessel_i0e_impl { EIGEN_DEVICE_FUNC - static EIGEN_STRONG_INLINE Scalar run(const Scalar x) { - return generic_i0e<Scalar, Scalar>::run(x); + static EIGEN_STRONG_INLINE T run(const T x) { + return generic_i0e<T>::run(x); } }; @@ -214,7 +214,7 @@ struct bessel_i0_retval { typedef Scalar type; }; -template <typename T, typename ScalarType> +template <typename T, typename ScalarType = typename unpacket_traits<T>::type> struct generic_i0 { EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE T run(const T& x) { @@ -224,11 +224,11 @@ struct generic_i0 { } }; -template <typename Scalar> +template <typename T> struct bessel_i0_impl { EIGEN_DEVICE_FUNC - static EIGEN_STRONG_INLINE Scalar run(const Scalar x) { - return generic_i0<Scalar, Scalar>::run(x); + static EIGEN_STRONG_INLINE T run(const T x) { + return generic_i0<T>::run(x); } }; @@ -237,7 +237,7 @@ struct bessel_i1e_retval { typedef Scalar type; }; -template <typename T, typename ScalarType> +template <typename T, typename ScalarType = typename unpacket_traits<T>::type > struct generic_i1e { EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE T run(const T&) { @@ -396,20 +396,20 @@ struct generic_i1e<T, double> { } }; -template <typename Scalar> +template <typename T> struct bessel_i1e_impl { EIGEN_DEVICE_FUNC - static EIGEN_STRONG_INLINE Scalar run(const Scalar x) { - return generic_i1e<Scalar, Scalar>::run(x); + static EIGEN_STRONG_INLINE T run(const T x) { + return generic_i1e<T>::run(x); } }; -template <typename Scalar> +template <typename T> struct bessel_i1_retval { - typedef Scalar type; + typedef T type; }; -template <typename T, typename ScalarType> +template <typename T, typename ScalarType = typename unpacket_traits<T>::type> struct generic_i1 { EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE T run(const T& x) { @@ -419,20 +419,20 @@ struct generic_i1 { } }; -template <typename Scalar> +template <typename T> struct bessel_i1_impl { EIGEN_DEVICE_FUNC - static EIGEN_STRONG_INLINE Scalar run(const Scalar x) { - return generic_i1<Scalar, Scalar>::run(x); + static EIGEN_STRONG_INLINE T run(const T x) { + return generic_i1<T>::run(x); } }; -template <typename Scalar> +template <typename T> struct bessel_k0e_retval { - typedef Scalar type; + typedef T type; }; -template <typename T, typename ScalarType> +template <typename T, typename ScalarType = typename unpacket_traits<T>::type> struct generic_k0e { EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE T run(const T&) { @@ -582,20 +582,20 @@ struct generic_k0e<T, double> { } }; -template <typename Scalar> +template <typename T> struct bessel_k0e_impl { EIGEN_DEVICE_FUNC - static EIGEN_STRONG_INLINE Scalar run(const Scalar x) { - return generic_k0e<Scalar, Scalar>::run(x); + static EIGEN_STRONG_INLINE T run(const T x) { + return generic_k0e<T>::run(x); } }; -template <typename Scalar> +template <typename T> struct bessel_k0_retval { - typedef Scalar type; + typedef T type; }; -template <typename T, typename ScalarType> +template <typename T, typename ScalarType = typename unpacket_traits<T>::type> struct generic_k0 { EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE T run(const T&) { @@ -754,20 +754,20 @@ struct generic_k0<T, double> { } }; -template <typename Scalar> +template <typename T> struct bessel_k0_impl { EIGEN_DEVICE_FUNC - static EIGEN_STRONG_INLINE Scalar run(const Scalar x) { - return generic_k0<Scalar, Scalar>::run(x); + static EIGEN_STRONG_INLINE T run(const T x) { + return generic_k0<T>::run(x); } }; -template <typename Scalar> +template <typename T> struct bessel_k1e_retval { - typedef Scalar type; + typedef T type; }; -template <typename T, typename ScalarType> +template <typename T, typename ScalarType = typename unpacket_traits<T>::type> struct generic_k1e { EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE T run(const T&) { @@ -910,20 +910,20 @@ struct generic_k1e<T, double> { } }; -template <typename Scalar> +template <typename T> struct bessel_k1e_impl { EIGEN_DEVICE_FUNC - static EIGEN_STRONG_INLINE Scalar run(const Scalar x) { - return generic_k1e<Scalar, Scalar>::run(x); + static EIGEN_STRONG_INLINE T run(const T x) { + return generic_k1e<T>::run(x); } }; -template <typename Scalar> +template <typename T> struct bessel_k1_retval { - typedef Scalar type; + typedef T type; }; -template <typename T, typename ScalarType> +template <typename T, typename ScalarType = typename unpacket_traits<T>::type> struct generic_k1 { EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE T run(const T&) { @@ -1076,20 +1076,20 @@ struct generic_k1<T, double> { } }; -template <typename Scalar> +template <typename T> struct bessel_k1_impl { EIGEN_DEVICE_FUNC - static EIGEN_STRONG_INLINE Scalar run(const Scalar x) { - return generic_k1<Scalar, Scalar>::run(x); + static EIGEN_STRONG_INLINE T run(const T x) { + return generic_k1<T>::run(x); } }; -template <typename Scalar> +template <typename T> struct bessel_j0_retval { - typedef Scalar type; + typedef T type; }; -template <typename T, typename ScalarType> +template <typename T, typename ScalarType = typename unpacket_traits<T>::type> struct generic_j0 { EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE T run(const T&) { @@ -1276,20 +1276,20 @@ struct generic_j0<T, double> { } }; -template <typename Scalar> +template <typename T> struct bessel_j0_impl { EIGEN_DEVICE_FUNC - static EIGEN_STRONG_INLINE Scalar run(const Scalar x) { - return generic_j0<Scalar, Scalar>::run(x); + static EIGEN_STRONG_INLINE T run(const T x) { + return generic_j0<T>::run(x); } }; -template <typename Scalar> +template <typename T> struct bessel_y0_retval { - typedef Scalar type; + typedef T type; }; -template <typename T, typename ScalarType> +template <typename T, typename ScalarType = typename unpacket_traits<T>::type> struct generic_y0 { EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE T run(const T&) { @@ -1474,20 +1474,20 @@ struct generic_y0<T, double> { } }; -template <typename Scalar> +template <typename T> struct bessel_y0_impl { EIGEN_DEVICE_FUNC - static EIGEN_STRONG_INLINE Scalar run(const Scalar x) { - return generic_y0<Scalar, Scalar>::run(x); + static EIGEN_STRONG_INLINE T run(const T x) { + return generic_y0<T>::run(x); } }; -template <typename Scalar> +template <typename T> struct bessel_j1_retval { - typedef Scalar type; + typedef T type; }; -template <typename T, typename ScalarType> +template <typename T, typename ScalarType = typename unpacket_traits<T>::type> struct generic_j1 { EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE T run(const T&) { @@ -1665,20 +1665,20 @@ struct generic_j1<T, double> { } }; -template <typename Scalar> +template <typename T> struct bessel_j1_impl { EIGEN_DEVICE_FUNC - static EIGEN_STRONG_INLINE Scalar run(const Scalar x) { - return generic_j1<Scalar, Scalar>::run(x); + static EIGEN_STRONG_INLINE T run(const T x) { + return generic_j1<T>::run(x); } }; -template <typename Scalar> +template <typename T> struct bessel_y1_retval { - typedef Scalar type; + typedef T type; }; -template <typename T, typename ScalarType> +template <typename T, typename ScalarType = typename unpacket_traits<T>::type> struct generic_y1 { EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE T run(const T&) { @@ -1868,11 +1868,11 @@ struct generic_y1<T, double> { } }; -template <typename Scalar> +template <typename T> struct bessel_y1_impl { EIGEN_DEVICE_FUNC - static EIGEN_STRONG_INLINE Scalar run(const Scalar x) { - return generic_y1<Scalar, Scalar>::run(x); + static EIGEN_STRONG_INLINE T run(const T x) { + return generic_y1<T>::run(x); } }; diff --git a/unsupported/Eigen/src/SpecialFunctions/BesselFunctionsPacketMath.h b/unsupported/Eigen/src/SpecialFunctions/BesselFunctionsPacketMath.h index efc6d9c8f..943d10f6a 100644 --- a/unsupported/Eigen/src/SpecialFunctions/BesselFunctionsPacketMath.h +++ b/unsupported/Eigen/src/SpecialFunctions/BesselFunctionsPacketMath.h @@ -19,8 +19,7 @@ namespace internal { template <typename Packet> EIGEN_DEVICE_FUNC EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet pbessel_i0(const Packet& x) { - typedef typename unpacket_traits<Packet>::type ScalarType; - using internal::generic_i0; return generic_i0<Packet, ScalarType>::run(x); + return numext::bessel_i0(x); } /** \internal \returns the exponentially scaled modified Bessel function of @@ -28,8 +27,7 @@ Packet pbessel_i0(const Packet& x) { template <typename Packet> EIGEN_DEVICE_FUNC EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet pbessel_i0e(const Packet& x) { - typedef typename unpacket_traits<Packet>::type ScalarType; - using internal::generic_i0e; return generic_i0e<Packet, ScalarType>::run(x); + return numext::bessel_i0e(x); } /** \internal \returns the exponentially scaled modified Bessel function of @@ -37,8 +35,7 @@ Packet pbessel_i0e(const Packet& x) { template <typename Packet> EIGEN_DEVICE_FUNC EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet pbessel_i1(const Packet& x) { - typedef typename unpacket_traits<Packet>::type ScalarType; - using internal::generic_i1; return generic_i1<Packet, ScalarType>::run(x); + return numext::bessel_i1(x); } /** \internal \returns the exponentially scaled modified Bessel function of @@ -46,8 +43,7 @@ Packet pbessel_i1(const Packet& x) { template <typename Packet> EIGEN_DEVICE_FUNC EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet pbessel_i1e(const Packet& x) { - typedef typename unpacket_traits<Packet>::type ScalarType; - using internal::generic_i1e; return generic_i1e<Packet, ScalarType>::run(x); + return numext::bessel_i1e(x); } /** \internal \returns the exponentially scaled modified Bessel function of @@ -55,8 +51,7 @@ Packet pbessel_i1e(const Packet& x) { template <typename Packet> EIGEN_DEVICE_FUNC EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet pbessel_j0(const Packet& x) { - typedef typename unpacket_traits<Packet>::type ScalarType; - using internal::generic_j0; return generic_j0<Packet, ScalarType>::run(x); + return numext::bessel_j0(x); } /** \internal \returns the exponentially scaled modified Bessel function of @@ -64,8 +59,7 @@ Packet pbessel_j0(const Packet& x) { template <typename Packet> EIGEN_DEVICE_FUNC EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet pbessel_j1(const Packet& x) { - typedef typename unpacket_traits<Packet>::type ScalarType; - using internal::generic_j1; return generic_j1<Packet, ScalarType>::run(x); + return numext::bessel_j1(x); } /** \internal \returns the exponentially scaled modified Bessel function of @@ -73,8 +67,7 @@ Packet pbessel_j1(const Packet& x) { template <typename Packet> EIGEN_DEVICE_FUNC EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet pbessel_y0(const Packet& x) { - typedef typename unpacket_traits<Packet>::type ScalarType; - using internal::generic_y0; return generic_y0<Packet, ScalarType>::run(x); + return numext::bessel_y0(x); } /** \internal \returns the exponentially scaled modified Bessel function of @@ -82,8 +75,7 @@ Packet pbessel_y0(const Packet& x) { template <typename Packet> EIGEN_DEVICE_FUNC EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet pbessel_y1(const Packet& x) { - typedef typename unpacket_traits<Packet>::type ScalarType; - using internal::generic_y1; return generic_y1<Packet, ScalarType>::run(x); + return numext::bessel_y1(x); } /** \internal \returns the exponentially scaled modified Bessel function of @@ -91,8 +83,7 @@ Packet pbessel_y1(const Packet& x) { template <typename Packet> EIGEN_DEVICE_FUNC EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet pbessel_k0(const Packet& x) { - typedef typename unpacket_traits<Packet>::type ScalarType; - using internal::generic_k0; return generic_k0<Packet, ScalarType>::run(x); + return numext::bessel_k0(x); } /** \internal \returns the exponentially scaled modified Bessel function of @@ -100,8 +91,7 @@ Packet pbessel_k0(const Packet& x) { template <typename Packet> EIGEN_DEVICE_FUNC EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet pbessel_k0e(const Packet& x) { - typedef typename unpacket_traits<Packet>::type ScalarType; - using internal::generic_k0e; return generic_k0e<Packet, ScalarType>::run(x); + return numext::bessel_k0e(x); } /** \internal \returns the exponentially scaled modified Bessel function of @@ -109,8 +99,7 @@ Packet pbessel_k0e(const Packet& x) { template <typename Packet> EIGEN_DEVICE_FUNC EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet pbessel_k1(const Packet& x) { - typedef typename unpacket_traits<Packet>::type ScalarType; - using internal::generic_k1; return generic_k1<Packet, ScalarType>::run(x); + return numext::bessel_k1(x); } /** \internal \returns the exponentially scaled modified Bessel function of @@ -118,8 +107,7 @@ Packet pbessel_k1(const Packet& x) { template <typename Packet> EIGEN_DEVICE_FUNC EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet pbessel_k1e(const Packet& x) { - typedef typename unpacket_traits<Packet>::type ScalarType; - using internal::generic_k1e; return generic_k1e<Packet, ScalarType>::run(x); + return numext::bessel_k1e(x); } } // end namespace internal diff --git a/unsupported/Eigen/src/SpecialFunctions/SpecialFunctionsImpl.h b/unsupported/Eigen/src/SpecialFunctions/SpecialFunctionsImpl.h index 648eb053e..cfc13aff7 100644 --- a/unsupported/Eigen/src/SpecialFunctions/SpecialFunctionsImpl.h +++ b/unsupported/Eigen/src/SpecialFunctions/SpecialFunctionsImpl.h @@ -348,7 +348,7 @@ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T generic_fast_erf_float(const T& a_x) { template <typename T> struct erf_impl { EIGEN_DEVICE_FUNC - static EIGEN_STRONG_INLINE T run(const T x) { + static EIGEN_STRONG_INLINE T run(const T& x) { return generic_fast_erf_float(x); } }; @@ -490,7 +490,8 @@ struct erfc_impl<double> { template<typename T> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T flipsign( const T& should_flipsign, const T& x) { - const T sign_mask = pset1<T>(-0.0); + typedef typename unpacket_traits<T>::type Scalar; + const T sign_mask = pset1<T>(Scalar(-0.0)); T sign_bit = pand<T>(should_flipsign, sign_mask); return pxor<T>(sign_bit, x); } diff --git a/unsupported/Eigen/src/SpecialFunctions/arch/AVX/BesselFunctions.h b/unsupported/Eigen/src/SpecialFunctions/arch/AVX/BesselFunctions.h new file mode 100644 index 000000000..2d7669209 --- /dev/null +++ b/unsupported/Eigen/src/SpecialFunctions/arch/AVX/BesselFunctions.h @@ -0,0 +1,46 @@ +#ifndef EIGEN_AVX_BESSELFUNCTIONS_H +#define EIGEN_AVX_BESSELFUNCTIONS_H + +namespace Eigen { +namespace internal { + +F16_PACKET_FUNCTION(Packet8f, Packet8h, pbessel_i0) +BF16_PACKET_FUNCTION(Packet8f, Packet8bf, pbessel_i0) + +F16_PACKET_FUNCTION(Packet8f, Packet8h, pbessel_i0e) +BF16_PACKET_FUNCTION(Packet8f, Packet8bf, pbessel_i0e) + +F16_PACKET_FUNCTION(Packet8f, Packet8h, pbessel_i1) +BF16_PACKET_FUNCTION(Packet8f, Packet8bf, pbessel_i1) + +F16_PACKET_FUNCTION(Packet8f, Packet8h, pbessel_i1e) +BF16_PACKET_FUNCTION(Packet8f, Packet8bf, pbessel_i1e) + +F16_PACKET_FUNCTION(Packet8f, Packet8h, pbessel_j0) +BF16_PACKET_FUNCTION(Packet8f, Packet8bf, pbessel_j0) + +F16_PACKET_FUNCTION(Packet8f, Packet8h, pbessel_j1) +BF16_PACKET_FUNCTION(Packet8f, Packet8bf, pbessel_j1) + +F16_PACKET_FUNCTION(Packet8f, Packet8h, pbessel_k0) +BF16_PACKET_FUNCTION(Packet8f, Packet8bf, pbessel_k0) + +F16_PACKET_FUNCTION(Packet8f, Packet8h, pbessel_k0e) +BF16_PACKET_FUNCTION(Packet8f, Packet8bf, pbessel_k0e) + +F16_PACKET_FUNCTION(Packet8f, Packet8h, pbessel_k1) +BF16_PACKET_FUNCTION(Packet8f, Packet8bf, pbessel_k1) + +F16_PACKET_FUNCTION(Packet8f, Packet8h, pbessel_k1e) +BF16_PACKET_FUNCTION(Packet8f, Packet8bf, pbessel_k1e) + +F16_PACKET_FUNCTION(Packet8f, Packet8h, pbessel_y0) +BF16_PACKET_FUNCTION(Packet8f, Packet8bf, pbessel_y0) + +F16_PACKET_FUNCTION(Packet8f, Packet8h, pbessel_y1) +BF16_PACKET_FUNCTION(Packet8f, Packet8bf, pbessel_y1) + +} // namespace internal +} // namespace Eigen + +#endif // EIGEN_AVX_BESSELFUNCTIONS_H diff --git a/unsupported/Eigen/src/SpecialFunctions/arch/AVX/SpecialFunctions.h b/unsupported/Eigen/src/SpecialFunctions/arch/AVX/SpecialFunctions.h new file mode 100644 index 000000000..35e62a8ac --- /dev/null +++ b/unsupported/Eigen/src/SpecialFunctions/arch/AVX/SpecialFunctions.h @@ -0,0 +1,16 @@ +#ifndef EIGEN_AVX_SPECIALFUNCTIONS_H +#define EIGEN_AVX_SPECIALFUNCTIONS_H + +namespace Eigen { +namespace internal { + +F16_PACKET_FUNCTION(Packet8f, Packet8h, perf) +BF16_PACKET_FUNCTION(Packet8f, Packet8bf, perf) + +F16_PACKET_FUNCTION(Packet8f, Packet8h, pndtri) +BF16_PACKET_FUNCTION(Packet8f, Packet8bf, pndtri) + +} // namespace internal +} // namespace Eigen + +#endif // EIGEN_AVX_SPECIAL_FUNCTIONS_H diff --git a/unsupported/Eigen/src/SpecialFunctions/arch/AVX512/BesselFunctions.h b/unsupported/Eigen/src/SpecialFunctions/arch/AVX512/BesselFunctions.h new file mode 100644 index 000000000..7dd3c3e5b --- /dev/null +++ b/unsupported/Eigen/src/SpecialFunctions/arch/AVX512/BesselFunctions.h @@ -0,0 +1,46 @@ +#ifndef EIGEN_AVX512_BESSELFUNCTIONS_H +#define EIGEN_AVX512_BESSELFUNCTIONS_H + +namespace Eigen { +namespace internal { + +F16_PACKET_FUNCTION(Packet16f, Packet16h, pbessel_i0) +BF16_PACKET_FUNCTION(Packet16f, Packet16bf, pbessel_i0) + +F16_PACKET_FUNCTION(Packet16f, Packet16h, pbessel_i0e) +BF16_PACKET_FUNCTION(Packet16f, Packet16bf, pbessel_i0e) + +F16_PACKET_FUNCTION(Packet16f, Packet16h, pbessel_i1) +BF16_PACKET_FUNCTION(Packet16f, Packet16bf, pbessel_i1) + +F16_PACKET_FUNCTION(Packet16f, Packet16h, pbessel_i1e) +BF16_PACKET_FUNCTION(Packet16f, Packet16bf, pbessel_i1e) + +F16_PACKET_FUNCTION(Packet16f, Packet16h, pbessel_j0) +BF16_PACKET_FUNCTION(Packet16f, Packet16bf, pbessel_j0) + +F16_PACKET_FUNCTION(Packet16f, Packet16h, pbessel_j1) +BF16_PACKET_FUNCTION(Packet16f, Packet16bf, pbessel_j1) + +F16_PACKET_FUNCTION(Packet16f, Packet16h, pbessel_k0) +BF16_PACKET_FUNCTION(Packet16f, Packet16bf, pbessel_k0) + +F16_PACKET_FUNCTION(Packet16f, Packet16h, pbessel_k0e) +BF16_PACKET_FUNCTION(Packet16f, Packet16bf, pbessel_k0e) + +F16_PACKET_FUNCTION(Packet16f, Packet16h, pbessel_k1) +BF16_PACKET_FUNCTION(Packet16f, Packet16bf, pbessel_k1) + +F16_PACKET_FUNCTION(Packet16f, Packet16h, pbessel_k1e) +BF16_PACKET_FUNCTION(Packet16f, Packet16bf, pbessel_k1e) + +F16_PACKET_FUNCTION(Packet16f, Packet16h, pbessel_y0) +BF16_PACKET_FUNCTION(Packet16f, Packet16bf, pbessel_y0) + +F16_PACKET_FUNCTION(Packet16f, Packet16h, pbessel_y1) +BF16_PACKET_FUNCTION(Packet16f, Packet16bf, pbessel_y1) + +} // namespace internal +} // namespace Eigen + +#endif // EIGEN_AVX512_BESSELFUNCTIONS_H diff --git a/unsupported/Eigen/src/SpecialFunctions/arch/AVX512/SpecialFunctions.h b/unsupported/Eigen/src/SpecialFunctions/arch/AVX512/SpecialFunctions.h new file mode 100644 index 000000000..79878f2b6 --- /dev/null +++ b/unsupported/Eigen/src/SpecialFunctions/arch/AVX512/SpecialFunctions.h @@ -0,0 +1,16 @@ +#ifndef EIGEN_AVX512_SPECIALFUNCTIONS_H +#define EIGEN_AVX512_SPECIALFUNCTIONS_H + +namespace Eigen { +namespace internal { + +F16_PACKET_FUNCTION(Packet16f, Packet16h, perf) +BF16_PACKET_FUNCTION(Packet16f, Packet16bf, perf) + +F16_PACKET_FUNCTION(Packet16f, Packet16h, pndtri) +BF16_PACKET_FUNCTION(Packet16f, Packet16bf, pndtri) + +} // namespace internal +} // namespace Eigen + +#endif // EIGEN_AVX512_SPECIAL_FUNCTIONS_H diff --git a/unsupported/Eigen/src/SpecialFunctions/arch/GPU/GpuSpecialFunctions.h b/unsupported/Eigen/src/SpecialFunctions/arch/GPU/SpecialFunctions.h index dd3bf4dd1..dd3bf4dd1 100644 --- a/unsupported/Eigen/src/SpecialFunctions/arch/GPU/GpuSpecialFunctions.h +++ b/unsupported/Eigen/src/SpecialFunctions/arch/GPU/SpecialFunctions.h diff --git a/unsupported/Eigen/src/SpecialFunctions/arch/NEON/BesselFunctions.h b/unsupported/Eigen/src/SpecialFunctions/arch/NEON/BesselFunctions.h new file mode 100644 index 000000000..67433b057 --- /dev/null +++ b/unsupported/Eigen/src/SpecialFunctions/arch/NEON/BesselFunctions.h @@ -0,0 +1,54 @@ +#ifndef EIGEN_NEON_BESSELFUNCTIONS_H +#define EIGEN_NEON_BESSELFUNCTIONS_H + +namespace Eigen { +namespace internal { + +#if EIGEN_HAS_ARM64_FP16_VECTOR_ARITHMETIC + +#define NEON_HALF_TO_FLOAT_FUNCTIONS(METHOD) \ +template <> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE \ +Packet8hf METHOD<Packet8hf>(const Packet8hf& x) { \ + const Packet4f lo = METHOD<Packet4f>(vcvt_f32_f16(vget_low_f16(x))); \ + const Packet4f hi = METHOD<Packet4f>(vcvt_f32_f16(vget_high_f16(x))); \ + return vcombine_f16(vcvt_f16_f32(lo), vcvt_f16_f32(hi)); \ +} \ + \ +template <> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE \ +Packet4hf METHOD<Packet4hf>(const Packet4hf& x) { \ + return vcvt_f16_f32(METHOD<Packet4f>(vcvt_f32_f16(x))); \ +} + +NEON_HALF_TO_FLOAT_FUNCTIONS(pbessel_i0) +NEON_HALF_TO_FLOAT_FUNCTIONS(pbessel_i0e) +NEON_HALF_TO_FLOAT_FUNCTIONS(pbessel_i1) +NEON_HALF_TO_FLOAT_FUNCTIONS(pbessel_i1e) +NEON_HALF_TO_FLOAT_FUNCTIONS(pbessel_j0) +NEON_HALF_TO_FLOAT_FUNCTIONS(pbessel_j1) +NEON_HALF_TO_FLOAT_FUNCTIONS(pbessel_k0) +NEON_HALF_TO_FLOAT_FUNCTIONS(pbessel_k0e) +NEON_HALF_TO_FLOAT_FUNCTIONS(pbessel_k1) +NEON_HALF_TO_FLOAT_FUNCTIONS(pbessel_k1e) +NEON_HALF_TO_FLOAT_FUNCTIONS(pbessel_y0) +NEON_HALF_TO_FLOAT_FUNCTIONS(pbessel_y1) + +#undef NEON_HALF_TO_FLOAT_FUNCTIONS +#endif + +BF16_PACKET_FUNCTION(Packet4f, Packet4bf, pbessel_i0) +BF16_PACKET_FUNCTION(Packet4f, Packet4bf, pbessel_i0e) +BF16_PACKET_FUNCTION(Packet4f, Packet4bf, pbessel_i1) +BF16_PACKET_FUNCTION(Packet4f, Packet4bf, pbessel_i1e) +BF16_PACKET_FUNCTION(Packet4f, Packet4bf, pbessel_j0) +BF16_PACKET_FUNCTION(Packet4f, Packet4bf, pbessel_j1) +BF16_PACKET_FUNCTION(Packet4f, Packet4bf, pbessel_k0) +BF16_PACKET_FUNCTION(Packet4f, Packet4bf, pbessel_k0e) +BF16_PACKET_FUNCTION(Packet4f, Packet4bf, pbessel_k1) +BF16_PACKET_FUNCTION(Packet4f, Packet4bf, pbessel_k1e) +BF16_PACKET_FUNCTION(Packet4f, Packet4bf, pbessel_y0) +BF16_PACKET_FUNCTION(Packet4f, Packet4bf, pbessel_y1) + +} // namespace internal +} // namespace Eigen + +#endif // EIGEN_NEON_BESSELFUNCTIONS_H diff --git a/unsupported/Eigen/src/SpecialFunctions/arch/NEON/SpecialFunctions.h b/unsupported/Eigen/src/SpecialFunctions/arch/NEON/SpecialFunctions.h new file mode 100644 index 000000000..f8dda28fc --- /dev/null +++ b/unsupported/Eigen/src/SpecialFunctions/arch/NEON/SpecialFunctions.h @@ -0,0 +1,34 @@ +#ifndef EIGEN_NEON_SPECIALFUNCTIONS_H +#define EIGEN_NEON_SPECIALFUNCTIONS_H + +namespace Eigen { +namespace internal { + +#ifdef EIGEN_HAS_ARM64_FP16_VECTOR_ARITHMETIC + +#define NEON_HALF_TO_FLOAT_FUNCTIONS(METHOD) \ +template <> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE \ +Packet8hf METHOD<Packet8hf>(const Packet8hf& x) { \ + const Packet4f lo = METHOD<Packet4f>(vcvt_f32_f16(vget_low_f16(x))); \ + const Packet4f hi = METHOD<Packet4f>(vcvt_f32_f16(vget_high_f16(x))); \ + return vcombine_f16(vcvt_f16_f32(lo), vcvt_f16_f32(hi)); \ +} \ + \ +template <> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE \ +Packet4hf METHOD<Packet4hf>(const Packet4hf& x) { \ + return vcvt_f16_f32(METHOD<Packet4f>(vcvt_f32_f16(x))); \ +} + +NEON_HALF_TO_FLOAT_FUNCTIONS(perf) +NEON_HALF_TO_FLOAT_FUNCTIONS(pndtri) + +#undef NEON_HALF_TO_FLOAT_FUNCTIONS +#endif + +BF16_PACKET_FUNCTION(Packet4f, Packet4bf, perf) +BF16_PACKET_FUNCTION(Packet4f, Packet4bf, pndtri) + +} // namespace internal +} // namespace Eigen + +#endif // EIGEN_NEON_SPECIALFUNCTIONS_H |