summaryrefslogtreecommitdiff
path: root/absl/base
diff options
context:
space:
mode:
authorGravatar Abseil Team <absl-team@google.com>2017-12-14 12:36:12 -0800
committerGravatar Titus Winters <titus@google.com>2017-12-15 08:52:46 -0500
commit6280bddf55e675219cacc25a6a12bc5ddc0fdc74 (patch)
tree15b87def4e6978fa40eee2cb8db76302bedc7282 /absl/base
parent720c017e30339fd1786ce4aac68bc8559736e53f (diff)
Changes imported from Abseil "staging" branch:
- 8320b38cd9f4f271fb6b278bd1e10d93f6ac3856 Use overloads for int32/int64/uint32/uint64 rather than i... by Jorg Brown <jorg@google.com> - f8b582b8deb3f78a3c6de2114b3ec4640f5427dd Internal change by Juemin Yang <jueminyang@google.com> - 240ff55ebf493ab1233ebe6976853a5fa2b3ec46 Remove the internal LowLevelAlloc's dependence on kLinker... by Greg Falcon <gfalcon@google.com> GitOrigin-RevId: 8320b38cd9f4f271fb6b278bd1e10d93f6ac3856 Change-Id: If5004efa2b43856948390ab357b8e9403e4461b4
Diffstat (limited to 'absl/base')
-rw-r--r--absl/base/attributes.h3
-rw-r--r--absl/base/internal/low_level_alloc.cc279
2 files changed, 152 insertions, 130 deletions
diff --git a/absl/base/attributes.h b/absl/base/attributes.h
index 6f3cfe4c..02bb030f 100644
--- a/absl/base/attributes.h
+++ b/absl/base/attributes.h
@@ -305,6 +305,7 @@
__attribute__((section(#name))) __attribute__((noinline))
#endif
+
// ABSL_ATTRIBUTE_SECTION_VARIABLE
//
// Tells the compiler/linker to put a given variable into a section and define
@@ -344,6 +345,7 @@
(reinterpret_cast<void *>(__start_##name))
#define ABSL_ATTRIBUTE_SECTION_STOP(name) \
(reinterpret_cast<void *>(__stop_##name))
+
#else // !ABSL_HAVE_ATTRIBUTE_SECTION
#define ABSL_HAVE_ATTRIBUTE_SECTION 0
@@ -356,6 +358,7 @@
#define ABSL_DECLARE_ATTRIBUTE_SECTION_VARS(name)
#define ABSL_ATTRIBUTE_SECTION_START(name) (reinterpret_cast<void *>(0))
#define ABSL_ATTRIBUTE_SECTION_STOP(name) (reinterpret_cast<void *>(0))
+
#endif // ABSL_ATTRIBUTE_SECTION
// ABSL_ATTRIBUTE_STACK_ALIGN_FOR_OLD_LIBC
diff --git a/absl/base/internal/low_level_alloc.cc b/absl/base/internal/low_level_alloc.cc
index 8e2f9c98..96223243 100644
--- a/absl/base/internal/low_level_alloc.cc
+++ b/absl/base/internal/low_level_alloc.cc
@@ -19,6 +19,9 @@
#include "absl/base/internal/low_level_alloc.h"
+#include <type_traits>
+
+#include "absl/base/call_once.h"
#include "absl/base/config.h"
#include "absl/base/internal/scheduling_mode.h"
#include "absl/base/macros.h"
@@ -194,43 +197,80 @@ static void LLA_SkiplistDelete(AllocList *head, AllocList *e,
// ---------------------------------------------------------------------------
// Arena implementation
+// Metadata for an LowLevelAlloc arena instance.
struct LowLevelAlloc::Arena {
- // This constructor does nothing, and relies on zero-initialization to get
- // the proper initial state.
- Arena() : mu(base_internal::kLinkerInitialized) {} // NOLINT
- explicit Arena(int) // NOLINT(readability/casting)
- : // Avoid recursive cooperative scheduling w/ kernel scheduling.
- mu(base_internal::SCHEDULE_KERNEL_ONLY),
- // Set pagesize to zero explicitly for non-static init.
- pagesize(0),
- random(0) {}
-
- base_internal::SpinLock mu; // protects freelist, allocation_count,
- // pagesize, roundup, min_size
- AllocList freelist; // head of free list; sorted by addr (under mu)
- int32_t allocation_count; // count of allocated blocks (under mu)
- std::atomic<uint32_t> flags; // flags passed to NewArena (ro after init)
- size_t pagesize; // ==getpagesize() (init under mu, then ro)
- size_t roundup; // lowest 2^n >= max(16,sizeof (AllocList))
- // (init under mu, then ro)
- size_t min_size; // smallest allocation block size
- // (init under mu, then ro)
- uint32_t random; // PRNG state
+ // Constructs an arena with the given LowLevelAlloc flags.
+ explicit Arena(uint32_t flags_value);
+
+ base_internal::SpinLock mu;
+ // Head of free list, sorted by address
+ AllocList freelist GUARDED_BY(mu);
+ // Count of allocated blocks
+ int32_t allocation_count GUARDED_BY(mu);
+ // flags passed to NewArena
+ const uint32_t flags;
+ // Result of getpagesize()
+ const size_t pagesize;
+ // Lowest power of two >= max(16, sizeof(AllocList))
+ const size_t roundup;
+ // Smallest allocation block size
+ const size_t min_size;
+ // PRNG state
+ uint32_t random GUARDED_BY(mu);
};
-// The default arena, which is used when 0 is passed instead of an Arena
-// pointer.
-static struct LowLevelAlloc::Arena default_arena; // NOLINT
+namespace {
+using ArenaStorage = std::aligned_storage<sizeof(LowLevelAlloc::Arena),
+ alignof(LowLevelAlloc::Arena)>::type;
+
+// Static storage space for the lazily-constructed, default global arena
+// instances. We require this space because the whole point of LowLevelAlloc
+// is to avoid relying on malloc/new.
+ArenaStorage default_arena_storage;
+ArenaStorage unhooked_arena_storage;
+#ifndef ABSL_LOW_LEVEL_ALLOC_ASYNC_SIGNAL_SAFE_MISSING
+ArenaStorage unhooked_async_sig_safe_arena_storage;
+#endif
+
+// We must use LowLevelCallOnce here to construct the global arenas, rather than
+// using function-level statics, to avoid recursively invoking the scheduler.
+absl::once_flag create_globals_once;
+
+void CreateGlobalArenas() {
+ new (&default_arena_storage)
+ LowLevelAlloc::Arena(LowLevelAlloc::kCallMallocHook);
+ new (&unhooked_arena_storage) LowLevelAlloc::Arena(0);
+#ifndef ABSL_LOW_LEVEL_ALLOC_ASYNC_SIGNAL_SAFE_MISSING
+ new (&unhooked_async_sig_safe_arena_storage)
+ LowLevelAlloc::Arena(LowLevelAlloc::kAsyncSignalSafe);
+#endif
+}
-// Non-malloc-hooked arenas: used only to allocate metadata for arenas that
-// do not want malloc hook reporting, so that for them there's no malloc hook
-// reporting even during arena creation.
-static struct LowLevelAlloc::Arena unhooked_arena; // NOLINT
+// Returns a global arena that does not call into hooks. Used by NewArena()
+// when kCallMallocHook is not set.
+LowLevelAlloc::Arena* UnhookedArena() {
+ base_internal::LowLevelCallOnce(&create_globals_once, CreateGlobalArenas);
+ return reinterpret_cast<LowLevelAlloc::Arena*>(&unhooked_arena_storage);
+}
#ifndef ABSL_LOW_LEVEL_ALLOC_ASYNC_SIGNAL_SAFE_MISSING
-static struct LowLevelAlloc::Arena unhooked_async_sig_safe_arena; // NOLINT
+// Returns a global arena that is async-signal safe. Used by NewArena() when
+// kAsyncSignalSafe is set.
+LowLevelAlloc::Arena *UnhookedAsyncSigSafeArena() {
+ base_internal::LowLevelCallOnce(&create_globals_once, CreateGlobalArenas);
+ return reinterpret_cast<LowLevelAlloc::Arena *>(
+ &unhooked_async_sig_safe_arena_storage);
+}
#endif
+} // namespace
+
+// Returns the default arena, as used by LowLevelAlloc::Alloc() and friends.
+LowLevelAlloc::Arena *LowLevelAlloc::DefaultArena() {
+ base_internal::LowLevelCallOnce(&create_globals_once, CreateGlobalArenas);
+ return reinterpret_cast<LowLevelAlloc::Arena*>(&default_arena_storage);
+}
+
// magic numbers to identify allocated and unallocated blocks
static const uintptr_t kMagicAllocated = 0x4c833e95U;
static const uintptr_t kMagicUnallocated = ~kMagicAllocated;
@@ -242,9 +282,7 @@ class SCOPED_LOCKABLE ArenaLock {
EXCLUSIVE_LOCK_FUNCTION(arena->mu)
: arena_(arena) {
#ifndef ABSL_LOW_LEVEL_ALLOC_ASYNC_SIGNAL_SAFE_MISSING
- if (arena == &unhooked_async_sig_safe_arena ||
- (arena->flags.load(std::memory_order_relaxed) &
- LowLevelAlloc::kAsyncSignalSafe) != 0) {
+ if ((arena->flags & LowLevelAlloc::kAsyncSignalSafe) != 0) {
sigset_t all;
sigfillset(&all);
mask_valid_ = pthread_sigmask(SIG_BLOCK, &all, &mask_) == 0;
@@ -281,118 +319,107 @@ inline static uintptr_t Magic(uintptr_t magic, AllocList::Header *ptr) {
return magic ^ reinterpret_cast<uintptr_t>(ptr);
}
-// Initialize the fields of an Arena
-static void ArenaInit(LowLevelAlloc::Arena *arena) {
- if (arena->pagesize == 0) {
+namespace {
+size_t GetPageSize() {
#ifdef _WIN32
- SYSTEM_INFO system_info;
- GetSystemInfo(&system_info);
- arena->pagesize = std::max(system_info.dwPageSize,
- system_info.dwAllocationGranularity);
+ SYSTEM_INFO system_info;
+ GetSystemInfo(&system_info);
+ return std::max(system_info.dwPageSize, system_info.dwAllocationGranularity);
#else
- arena->pagesize = getpagesize();
-#endif
- // Round up block sizes to a power of two close to the header size.
- arena->roundup = 16;
- while (arena->roundup < sizeof (arena->freelist.header)) {
- arena->roundup += arena->roundup;
- }
- // Don't allocate blocks less than twice the roundup size to avoid tiny
- // free blocks.
- arena->min_size = 2 * arena->roundup;
- arena->freelist.header.size = 0;
- arena->freelist.header.magic =
- Magic(kMagicUnallocated, &arena->freelist.header);
- arena->freelist.header.arena = arena;
- arena->freelist.levels = 0;
- memset(arena->freelist.next, 0, sizeof (arena->freelist.next));
- arena->allocation_count = 0;
- if (arena == &default_arena) {
- // Default arena should be hooked, e.g. for heap-checker to trace
- // pointer chains through objects in the default arena.
- arena->flags.store(LowLevelAlloc::kCallMallocHook,
- std::memory_order_relaxed);
- }
-#ifndef ABSL_LOW_LEVEL_ALLOC_ASYNC_SIGNAL_SAFE_MISSING
- else if (arena == // NOLINT(readability/braces)
- &unhooked_async_sig_safe_arena) {
- arena->flags.store(LowLevelAlloc::kAsyncSignalSafe,
- std::memory_order_relaxed);
- }
+ return getpagesize();
#endif
- else { // NOLINT(readability/braces)
- // other arenas' flags may be overridden by client,
- // but unhooked_arena will have 0 in 'flags'.
- arena->flags.store(0, std::memory_order_relaxed);
- }
+}
+
+size_t RoundedUpBlockSize() {
+ // Round up block sizes to a power of two close to the header size.
+ size_t roundup = 16;
+ while (roundup < sizeof(AllocList::Header)) {
+ roundup += roundup;
}
+ return roundup;
+}
+
+} // namespace
+
+LowLevelAlloc::Arena::Arena(uint32_t flags_value)
+ : mu(base_internal::SCHEDULE_KERNEL_ONLY),
+ allocation_count(0),
+ flags(flags_value),
+ pagesize(GetPageSize()),
+ roundup(RoundedUpBlockSize()),
+ min_size(2 * roundup),
+ random(0) {
+ freelist.header.size = 0;
+ freelist.header.magic =
+ Magic(kMagicUnallocated, &freelist.header);
+ freelist.header.arena = this;
+ freelist.levels = 0;
+ memset(freelist.next, 0, sizeof(freelist.next));
}
// L < meta_data_arena->mu
LowLevelAlloc::Arena *LowLevelAlloc::NewArena(int32_t flags,
Arena *meta_data_arena) {
ABSL_RAW_CHECK(meta_data_arena != nullptr, "must pass a valid arena");
- if (meta_data_arena == &default_arena) {
+ if (meta_data_arena == DefaultArena()) {
#ifndef ABSL_LOW_LEVEL_ALLOC_ASYNC_SIGNAL_SAFE_MISSING
if ((flags & LowLevelAlloc::kAsyncSignalSafe) != 0) {
- meta_data_arena = &unhooked_async_sig_safe_arena;
+ meta_data_arena = UnhookedAsyncSigSafeArena();
} else // NOLINT(readability/braces)
#endif
if ((flags & LowLevelAlloc::kCallMallocHook) == 0) {
- meta_data_arena = &unhooked_arena;
+ meta_data_arena = UnhookedArena();
}
}
- // Arena(0) uses the constructor for non-static contexts
Arena *result =
- new (AllocWithArena(sizeof (*result), meta_data_arena)) Arena(0);
- ArenaInit(result);
- result->flags.store(flags, std::memory_order_relaxed);
+ new (AllocWithArena(sizeof (*result), meta_data_arena)) Arena(flags);
return result;
}
// L < arena->mu, L < arena->arena->mu
bool LowLevelAlloc::DeleteArena(Arena *arena) {
ABSL_RAW_CHECK(
- arena != nullptr && arena != &default_arena && arena != &unhooked_arena,
+ arena != nullptr && arena != DefaultArena() && arena != UnhookedArena(),
"may not delete default arena");
ArenaLock section(arena);
- bool empty = (arena->allocation_count == 0);
- section.Leave();
- if (empty) {
- while (arena->freelist.next[0] != nullptr) {
- AllocList *region = arena->freelist.next[0];
- size_t size = region->header.size;
- arena->freelist.next[0] = region->next[0];
- ABSL_RAW_CHECK(
- region->header.magic == Magic(kMagicUnallocated, &region->header),
- "bad magic number in DeleteArena()");
- ABSL_RAW_CHECK(region->header.arena == arena,
- "bad arena pointer in DeleteArena()");
- ABSL_RAW_CHECK(size % arena->pagesize == 0,
- "empty arena has non-page-aligned block size");
- ABSL_RAW_CHECK(reinterpret_cast<uintptr_t>(region) % arena->pagesize == 0,
- "empty arena has non-page-aligned block");
- int munmap_result;
+ if (arena->allocation_count != 0) {
+ section.Leave();
+ return false;
+ }
+ while (arena->freelist.next[0] != nullptr) {
+ AllocList *region = arena->freelist.next[0];
+ size_t size = region->header.size;
+ arena->freelist.next[0] = region->next[0];
+ ABSL_RAW_CHECK(
+ region->header.magic == Magic(kMagicUnallocated, &region->header),
+ "bad magic number in DeleteArena()");
+ ABSL_RAW_CHECK(region->header.arena == arena,
+ "bad arena pointer in DeleteArena()");
+ ABSL_RAW_CHECK(size % arena->pagesize == 0,
+ "empty arena has non-page-aligned block size");
+ ABSL_RAW_CHECK(reinterpret_cast<uintptr_t>(region) % arena->pagesize == 0,
+ "empty arena has non-page-aligned block");
+ int munmap_result;
#ifdef _WIN32
- munmap_result = VirtualFree(region, 0, MEM_RELEASE);
- ABSL_RAW_CHECK(munmap_result != 0,
- "LowLevelAlloc::DeleteArena: VitualFree failed");
+ munmap_result = VirtualFree(region, 0, MEM_RELEASE);
+ ABSL_RAW_CHECK(munmap_result != 0,
+ "LowLevelAlloc::DeleteArena: VitualFree failed");
#else
- if ((arena->flags.load(std::memory_order_relaxed) &
- LowLevelAlloc::kAsyncSignalSafe) == 0) {
- munmap_result = munmap(region, size);
- } else {
- munmap_result = MallocHook::UnhookedMUnmap(region, size);
- }
- if (munmap_result != 0) {
- ABSL_RAW_LOG(FATAL, "LowLevelAlloc::DeleteArena: munmap failed: %d",
- errno);
- }
-#endif
+ if ((arena->flags & LowLevelAlloc::kAsyncSignalSafe) == 0) {
+ munmap_result = munmap(region, size);
+ } else {
+ munmap_result = MallocHook::UnhookedMUnmap(region, size);
+ }
+ if (munmap_result != 0) {
+ ABSL_RAW_LOG(FATAL, "LowLevelAlloc::DeleteArena: munmap failed: %d",
+ errno);
}
- Free(arena);
+#endif
}
- return empty;
+ section.Leave();
+ arena->~Arena();
+ Free(arena);
+ return true;
}
// ---------------------------------------------------------------------------
@@ -479,7 +506,7 @@ void LowLevelAlloc::Free(void *v) {
ABSL_RAW_CHECK(f->header.magic == Magic(kMagicAllocated, &f->header),
"bad magic number in Free()");
LowLevelAlloc::Arena *arena = f->header.arena;
- if ((arena->flags.load(std::memory_order_relaxed) & kCallMallocHook) != 0) {
+ if ((arena->flags & kCallMallocHook) != 0) {
MallocHook::InvokeDeleteHook(v);
}
ArenaLock section(arena);
@@ -497,7 +524,6 @@ static void *DoAllocWithArena(size_t request, LowLevelAlloc::Arena *arena) {
if (request != 0) {
AllocList *s; // will point to region that satisfies request
ArenaLock section(arena);
- ArenaInit(arena);
// round up with header
size_t req_rnd = RoundUp(CheckedAdd(request, sizeof (s->header)),
arena->roundup);
@@ -526,8 +552,7 @@ static void *DoAllocWithArena(size_t request, LowLevelAlloc::Arena *arena) {
MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE);
ABSL_RAW_CHECK(new_pages != nullptr, "VirtualAlloc failed");
#else
- if ((arena->flags.load(std::memory_order_relaxed) &
- LowLevelAlloc::kAsyncSignalSafe) != 0) {
+ if ((arena->flags & LowLevelAlloc::kAsyncSignalSafe) != 0) {
new_pages = MallocHook::UnhookedMMap(nullptr, new_pages_size,
PROT_WRITE|PROT_READ, MAP_ANONYMOUS|MAP_PRIVATE, -1, 0);
} else {
@@ -570,20 +595,18 @@ static void *DoAllocWithArena(size_t request, LowLevelAlloc::Arena *arena) {
}
void *LowLevelAlloc::Alloc(size_t request) {
- void *result = DoAllocWithArena(request, &default_arena);
- if ((default_arena.flags.load(std::memory_order_relaxed) &
- kCallMallocHook) != 0) {
- // this call must be directly in the user-called allocator function
- // for MallocHook::GetCallerStackTrace to work properly
- MallocHook::InvokeNewHook(result, request);
- }
+ void *result = DoAllocWithArena(request, DefaultArena());
+ // The default arena always calls the malloc hook.
+ // This call must be directly in the user-called allocator function
+ // for MallocHook::GetCallerStackTrace to work properly
+ MallocHook::InvokeNewHook(result, request);
return result;
}
void *LowLevelAlloc::AllocWithArena(size_t request, Arena *arena) {
ABSL_RAW_CHECK(arena != nullptr, "must pass a valid arena");
void *result = DoAllocWithArena(request, arena);
- if ((arena->flags.load(std::memory_order_relaxed) & kCallMallocHook) != 0) {
+ if ((arena->flags & kCallMallocHook) != 0) {
// this call must be directly in the user-called allocator function
// for MallocHook::GetCallerStackTrace to work properly
MallocHook::InvokeNewHook(result, request);
@@ -591,10 +614,6 @@ void *LowLevelAlloc::AllocWithArena(size_t request, Arena *arena) {
return result;
}
-LowLevelAlloc::Arena *LowLevelAlloc::DefaultArena() {
- return &default_arena;
-}
-
} // namespace base_internal
} // namespace absl