From 92a7e778e7394386f413cec28d67a07630f784b1 Mon Sep 17 00:00:00 2001 From: Adam Cozzette Date: Fri, 1 Dec 2017 10:05:10 -0800 Subject: Integrated internal changes from Google --- src/google/protobuf/arena_impl.h | 158 +++++++++++++++++++++++++++++---------- 1 file changed, 117 insertions(+), 41 deletions(-) (limited to 'src/google/protobuf/arena_impl.h') diff --git a/src/google/protobuf/arena_impl.h b/src/google/protobuf/arena_impl.h index 6cc7096b..439b0969 100644 --- a/src/google/protobuf/arena_impl.h +++ b/src/google/protobuf/arena_impl.h @@ -40,10 +40,13 @@ #include #include #include -#include #include +#ifdef ADDRESS_SANITIZER +#include +#endif // ADDRESS_SANITIZER + namespace google { namespace protobuf { @@ -112,6 +115,10 @@ class LIBPROTOBUF_EXPORT ArenaImpl { void AddCleanup(void* elem, void (*cleanup)(void*)); private: + void* AllocateAlignedFallback(size_t n); + void* AllocateAlignedAndAddCleanupFallback(size_t n, void (*cleanup)(void*)); + void AddCleanupFallback(void* elem, void (*cleanup)(void*)); + // Node contains the ptr of the object to be cleaned up and the associated // cleanup function ptr. struct CleanupNode { @@ -124,34 +131,107 @@ class LIBPROTOBUF_EXPORT ArenaImpl { static size_t SizeOf(size_t i) { return sizeof(CleanupChunk) + (sizeof(CleanupNode) * (i - 1)); } - size_t len; // Number of elements currently present. size_t size; // Total elements in the list. CleanupChunk* next; // Next node in the list. CleanupNode nodes[1]; // True length is |size|. }; - struct Block; + class Block; + + // A thread-unsafe Arena that can only be used within its owning thread. + class LIBPROTOBUF_EXPORT SerialArena { + public: + // The allocate/free methods here are a little strange, since SerialArena is + // allocated inside a Block which it also manages. This is to avoid doing + // an extra allocation for the SerialArena itself. + + // Creates a new SerialArena inside Block* and returns it. + static SerialArena* New(Block* b, void* owner, ArenaImpl* arena); + + // Destroys this SerialArena, freeing all blocks with the given dealloc + // function, except any block equal to |initial_block|. + static uint64 Free(SerialArena* serial, Block* initial_block, + void (*block_dealloc)(void*, size_t)); + + void CleanupList(); + uint64 SpaceUsed() const; + + void* AllocateAligned(size_t n) { + GOOGLE_DCHECK_EQ(internal::AlignUpTo8(n), n); // Must be already aligned. + GOOGLE_DCHECK_GE(limit_, ptr_); + if (GOOGLE_PREDICT_FALSE(static_cast(limit_ - ptr_) < n)) { + return AllocateAlignedFallback(n); + } + void* ret = ptr_; + ptr_ += n; +#ifdef ADDRESS_SANITIZER + ASAN_UNPOISON_MEMORY_REGION(ret, n); +#endif // ADDRESS_SANITIZER + return ret; + } + + void AddCleanup(void* elem, void (*cleanup)(void*)) { + if (GOOGLE_PREDICT_FALSE(cleanup_ptr_ == cleanup_limit_)) { + AddCleanupFallback(elem, cleanup); + return; + } + cleanup_ptr_->elem = elem; + cleanup_ptr_->cleanup = cleanup; + cleanup_ptr_++; + } + + void* AllocateAlignedAndAddCleanup(size_t n, void (*cleanup)(void*)) { + void* ret = AllocateAligned(n); + AddCleanup(ret, cleanup); + return ret; + } - // Tracks per-thread info. ThreadInfos are kept in a linked list. - struct ThreadInfo { - void *owner; // &ThreadCache of this thread; - Block* head; // Head of linked list of blocks. - CleanupChunk* cleanup; // Head of cleanup list. - ThreadInfo* next; // Next ThreadInfo in this linked list. + void* owner() const { return owner_; } + SerialArena* next() const { return next_; } + void set_next(SerialArena* next) { next_ = next; } + + private: + void* AllocateAlignedFallback(size_t n); + void AddCleanupFallback(void* elem, void (*cleanup)(void*)); + void CleanupListFallback(); + + ArenaImpl* arena_; // Containing arena. + void* owner_; // &ThreadCache of this thread; + Block* head_; // Head of linked list of blocks. + CleanupChunk* cleanup_; // Head of cleanup list. + SerialArena* next_; // Next SerialArena in this linked list. + + // Next pointer to allocate from. Always 8-byte aligned. Points inside + // head_ (and head_->pos will always be non-canonical). We keep these + // here to reduce indirection. + char* ptr_; + char* limit_; + + // Next CleanupList members to append to. These point inside cleanup_. + CleanupNode* cleanup_ptr_; + CleanupNode* cleanup_limit_; }; // Blocks are variable length malloc-ed objects. The following structure // describes the common header for all blocks. - struct Block { - void* owner; // &ThreadCache of thread that owns this block. - ThreadInfo* thread_info; // ThreadInfo of thread that owns this block. - Block* next; // Next block in arena (may have different owner) - // ((char*) &block) + pos is next available byte. It is always - // aligned at a multiple of 8 bytes. - size_t pos; - size_t size; // total size of the block. - GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE - size_t avail() const { return size - pos; } + class LIBPROTOBUF_EXPORT Block { + public: + Block(size_t size, Block* next); + + char* Pointer(size_t n) { + GOOGLE_DCHECK(n <= size_); + return reinterpret_cast(this) + n; + } + + Block* next() const { return next_; } + size_t pos() const { return pos_; } + size_t size() const { return size_; } + void set_pos(size_t pos) { pos_ = pos; } + + private: + Block* next_; // Next block for this thread. + size_t pos_; + size_t size_; // data follows }; @@ -160,13 +240,13 @@ class LIBPROTOBUF_EXPORT ArenaImpl { // If we are using the ThreadLocalStorage class to store the ThreadCache, // then the ThreadCache's default constructor has to be responsible for // initializing it. - ThreadCache() : last_lifecycle_id_seen(-1), last_block_used_(NULL) {} + ThreadCache() : last_lifecycle_id_seen(-1), last_serial_arena(NULL) {} #endif // The ThreadCache is considered valid as long as this matches the // lifecycle_id of the arena being used. int64 last_lifecycle_id_seen; - Block* last_block_used_; + SerialArena* last_serial_arena; }; static google::protobuf::internal::SequenceNumber lifecycle_id_generator_; #if defined(GOOGLE_PROTOBUF_NO_THREADLOCAL) @@ -188,38 +268,30 @@ class LIBPROTOBUF_EXPORT ArenaImpl { // Free all blocks and return the total space used which is the sums of sizes // of the all the allocated blocks. uint64 FreeBlocks(); - - void AddCleanupInBlock(Block* b, void* elem, void (*func)(void*)); - CleanupChunk* ExpandCleanupList(CleanupChunk* cleanup, Block* b); // Delete or Destruct all objects owned by the arena. void CleanupList(); - inline void CacheBlock(Block* block) { - thread_cache().last_block_used_ = block; + inline void CacheSerialArena(SerialArena* serial) { + thread_cache().last_serial_arena = serial; thread_cache().last_lifecycle_id_seen = lifecycle_id_; // TODO(haberman): evaluate whether we would gain efficiency by getting rid // of hint_. It's the only write we do to ArenaImpl in the allocation path, // which will dirty the cache line. - google::protobuf::internal::Release_Store(&hint_, reinterpret_cast(block)); + google::protobuf::internal::Release_Store(&hint_, reinterpret_cast(serial)); } - google::protobuf::internal::AtomicWord threads_; // Pointer to a linked list of ThreadInfo. + google::protobuf::internal::AtomicWord threads_; // Pointer to a linked list of SerialArena. google::protobuf::internal::AtomicWord hint_; // Fast thread-local block access google::protobuf::internal::AtomicWord space_allocated_; // Sum of sizes of all allocated blocks. Block *initial_block_; // If non-NULL, points to the block that came from // user data. - // Returns a block owned by this thread. - Block* GetBlock(size_t n); - Block* GetBlockSlow(void* me, Block* my_full_block, size_t n); - Block* NewBlock(void* me, Block* my_last_block, size_t min_bytes); - void InitBlock(Block* b, void *me, size_t size); - static void* AllocFromBlock(Block* b, size_t n); - ThreadInfo* NewThreadInfo(Block* b); - ThreadInfo* FindThreadInfo(void* me); - ThreadInfo* GetThreadInfo(void* me, size_t n); + Block* NewBlock(Block* last_block, size_t min_bytes); + SerialArena* GetSerialArena(); + bool GetSerialArenaFast(SerialArena** arena); + SerialArena* GetSerialArenaFallback(void* me); int64 lifecycle_id_; // Unique for each arena. Changes on Reset(). Options options_; @@ -227,11 +299,15 @@ class LIBPROTOBUF_EXPORT ArenaImpl { GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(ArenaImpl); public: - // kHeaderSize is sizeof(Block), aligned up to the nearest multiple of 8 to - // protect the invariant that pos is always at a multiple of 8. - static const size_t kHeaderSize = (sizeof(Block) + 7) & -8; + // kBlockHeaderSize is sizeof(Block), aligned up to the nearest multiple of 8 + // to protect the invariant that pos is always at a multiple of 8. + static const size_t kBlockHeaderSize = (sizeof(Block) + 7) & -8; + static const size_t kSerialArenaSize = (sizeof(SerialArena) + 7) & -8; #if LANG_CXX11 - static_assert(kHeaderSize % 8 == 0, "kHeaderSize must be a multiple of 8."); + static_assert(kBlockHeaderSize % 8 == 0, + "kBlockHeaderSize must be a multiple of 8."); + static_assert(kSerialArenaSize % 8 == 0, + "kSerialArenaSize must be a multiple of 8."); #endif }; -- cgit v1.2.3