From 1feab4fff90f904518e66cf80971063486fbc984 Mon Sep 17 00:00:00 2001 From: Connal de Souza Date: Fri, 9 Jun 2023 12:38:09 -0700 Subject: Optimize Cord Refcount decrement. Introduce kHighRefcountMask which masks off flags and the LSb of the refcount value. In the cases where this mask is used, we don't need to check the LSb because we can assume the refcount is 1 when the rest of the masked RefcountAndFlags is empty, and the LSb doesn't matter if the masked value is not empty (either it's immortal or refcount > 1). This saves an instruction and a cycle (and + cmp -> tst) https://godbolt.org/z/Kz69eqfhq PiperOrigin-RevId: 539151659 Change-Id: I2ec7d72918f052c4b0938edd746af9d5b3052c7e --- absl/strings/internal/cord_internal.h | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) (limited to 'absl/strings/internal') diff --git a/absl/strings/internal/cord_internal.h b/absl/strings/internal/cord_internal.h index 8d9836ba..20dd008c 100644 --- a/absl/strings/internal/cord_internal.h +++ b/absl/strings/internal/cord_internal.h @@ -157,20 +157,19 @@ class RefcountAndFlags { // false will be visible to a thread that just observed this method returning // false. Always returns false when the immortal bit is set. inline bool Decrement() { - int32_t refcount = count_.load(std::memory_order_acquire) & kRefcountMask; - assert(refcount > 0 || refcount & kImmortalFlag); + int32_t refcount = count_.load(std::memory_order_acquire); + assert((refcount & kRefcountMask) > 0 || refcount & kImmortalFlag); return refcount != kRefIncrement && (count_.fetch_sub(kRefIncrement, std::memory_order_acq_rel) & - kRefcountMask) != kRefIncrement; + kHighRefcountMask) != 0; } // Same as Decrement but expect that refcount is greater than 1. inline bool DecrementExpectHighRefcount() { int32_t refcount = - count_.fetch_sub(kRefIncrement, std::memory_order_acq_rel) & - kRefcountMask; - assert(refcount > 0 || refcount & kImmortalFlag); - return refcount != kRefIncrement; + count_.fetch_sub(kRefIncrement, std::memory_order_acq_rel); + assert((refcount & kRefcountMask) > 0 || refcount & kImmortalFlag); + return (refcount & kHighRefcountMask) != 0; } // Returns the current reference count using acquire semantics. @@ -214,6 +213,15 @@ class RefcountAndFlags { // purposes of equality. (A refcount of 0 or 1 does not count as 0 or 1 // if the immortal bit is set.) kRefcountMask = ~kReservedFlag, + + // Bitmask to use when checking if refcount is equal to 1 and not + // immortal when decrementing the refcount. This masks out kRefIncrement and + // all flags except kImmortalFlag. If the masked RefcountAndFlags is 0, we + // assume the refcount is equal to 1, since we know it's not immortal and + // not greater than 1. If the masked RefcountAndFlags is not 0, we can + // assume the refcount is not equal to 1 since either a higher bit in the + // refcount is set, or kImmortal is set. + kHighRefcountMask = kRefcountMask & ~kRefIncrement, }; std::atomic count_; -- cgit v1.2.3