diff options
| author |  Hannah Lin <hylin@google.com> | 2023-08-14 14:34:51 -0700 |
|---|---|---|
| committer |  Copybara-Service <copybara-worker@google.com> | 2023-08-14 14:35:57 -0700 |
| commit | 71910654b4452f13f543f7c7f433fb3c6d932c33 (patch) | |
| tree | 7534cfa548090c2a8cc4b5c6647876705a8c29b2 /absl/base | |
| parent | e8f98b24eb3f16f0964c63d1bec9cf1bc3c2606e (diff) | |
Always inline prefetches.
PiperOrigin-RevId: 556914455
Change-Id: Ic0169e1099384eefe285e6d354e448eb5189e397
Diffstat (limited to 'absl/base')
| -rw-r--r-- | absl/base/prefetch.h | 28 |
1 files changed, 19 insertions, 9 deletions
diff --git a/absl/base/prefetch.h b/absl/base/prefetch.h index de7a180d..6fd2a820 100644 --- a/absl/base/prefetch.h +++ b/absl/base/prefetch.h @@ -24,6 +24,7 @@ #ifndef ABSL_BASE_PREFETCH_H_ #define ABSL_BASE_PREFETCH_H_ +#include "absl/base/attributes.h" #include "absl/base/config.h" #if defined(ABSL_INTERNAL_HAVE_SSE) @@ -140,15 +141,18 @@ void PrefetchToLocalCacheForWrite(const void* addr); // See __builtin_prefetch: // https://gcc.gnu.org/onlinedocs/gcc/Other-Builtins.html. // -inline void PrefetchToLocalCache(const void* addr) { +ABSL_ATTRIBUTE_ALWAYS_INLINE inline void PrefetchToLocalCache( + const void* addr) { __builtin_prefetch(addr, 0, 3); } -inline void PrefetchToLocalCacheNta(const void* addr) { +ABSL_ATTRIBUTE_ALWAYS_INLINE inline void PrefetchToLocalCacheNta( + const void* addr) { __builtin_prefetch(addr, 0, 0); } -inline void PrefetchToLocalCacheForWrite(const void* addr) { +ABSL_ATTRIBUTE_ALWAYS_INLINE inline void PrefetchToLocalCacheForWrite( + const void* addr) { // [x86] gcc/clang don't generate PREFETCHW for __builtin_prefetch(.., 1) // unless -march=broadwell or newer; this is not generally the default, so we // manually emit prefetchw. PREFETCHW is recognized as a no-op on older Intel @@ -164,15 +168,18 @@ inline void PrefetchToLocalCacheForWrite(const void* addr) { #define ABSL_HAVE_PREFETCH 1 -inline void PrefetchToLocalCache(const void* addr) { +ABSL_ATTRIBUTE_ALWAYS_INLINE inline void PrefetchToLocalCache( + const void* addr) { _mm_prefetch(reinterpret_cast<const char*>(addr), _MM_HINT_T0); } -inline void PrefetchToLocalCacheNta(const void* addr) { +ABSL_ATTRIBUTE_ALWAYS_INLINE inline void PrefetchToLocalCacheNta( + const void* addr) { _mm_prefetch(reinterpret_cast<const char*>(addr), _MM_HINT_NTA); } -inline void PrefetchToLocalCacheForWrite(const void* addr) { +ABSL_ATTRIBUTE_ALWAYS_INLINE inline void PrefetchToLocalCacheForWrite( + const void* addr) { #if defined(_MM_HINT_ET0) _mm_prefetch(reinterpret_cast<const char*>(addr), _MM_HINT_ET0); #elif !defined(_MSC_VER) && defined(__x86_64__) @@ -186,9 +193,12 @@ inline void PrefetchToLocalCacheForWrite(const void* addr) { #else -inline void PrefetchToLocalCache(const void* addr) {} -inline void PrefetchToLocalCacheNta(const void* addr) {} -inline void PrefetchToLocalCacheForWrite(const void* addr) {} +ABSL_ATTRIBUTE_ALWAYS_INLINE inline void PrefetchToLocalCache( + const void* addr) {} +ABSL_ATTRIBUTE_ALWAYS_INLINE inline void PrefetchToLocalCacheNta( + const void* addr) {} +ABSL_ATTRIBUTE_ALWAYS_INLINE inline void PrefetchToLocalCacheForWrite( + const void* addr) {} #endif |