diff --git a/fb303/ThreadLocalStatsMap-inl.h b/fb303/ThreadLocalStatsMap-inl.h index ec6264e6b..66b517b9a 100644 --- a/fb303/ThreadLocalStatsMap-inl.h +++ b/fb303/ThreadLocalStatsMap-inl.h @@ -55,8 +55,7 @@ void ThreadLocalStatsMapT::clearStat( ExportType exportType) { auto state = state_.lock(); if (auto ptr = folly::get_ptr(state->namedTimeseries_, name)) { - auto mask = uintptr_t(1) << size_t(exportType); - ptr->exports = ptr->exports & ~mask; + ptr->type(exportType, false); } this->getServiceData()->addStatExportType(name, exportType, nullptr); } @@ -85,10 +84,10 @@ std::shared_ptr::TLTimeseries> ThreadLocalStatsMapT::getTimeseriesSafe(folly::StringPiece name) { auto state = state_.lock(); auto& entry = state->namedTimeseries_[name]; - if (!entry.ptr) { - entry.ptr = std::make_shared(this, name); + if (!entry) { + entry.ptr(std::make_shared(this, name)); } - return entry.ptr; + return entry.ptr(); } template @@ -100,21 +99,21 @@ ThreadLocalStatsMapT::getTimeseriesSafe( const int levelDurations[]) { auto state = state_.lock(); auto& entry = state->namedTimeseries_[name]; - if (!entry.ptr) { - entry.ptr = std::make_shared( - this, name, numBuckets, numLevels, levelDurations); + if (!entry) { + entry.ptr(std::make_shared( + this, name, numBuckets, numLevels, levelDurations)); } - return entry.ptr; + return entry.ptr(); } template typename ThreadLocalStatsMapT::TLTimeseries* ThreadLocalStatsMapT< LockTraits>::getTimeseriesLocked(State& state, folly::StringPiece name) { auto& entry = state.namedTimeseries_[name]; - if (!entry.ptr) { - entry.ptr = std::make_shared(this, name); + if (!entry) { + entry.ptr(std::make_shared(this, name)); } - return entry.ptr.get(); + return entry.raw(); } template @@ -124,15 +123,14 @@ ThreadLocalStatsMapT::getTimeseriesLocked( folly::StringPiece name, ExportType exportType) { auto& entry = state.namedTimeseries_[name]; - if (!entry.ptr) { - entry.ptr = std::make_shared(this, name); + if (!entry) { + entry.ptr(std::make_shared(this, name)); } - auto mask = uintptr_t(1) << size_t(exportType); - if (!(entry.exports & mask)) { + if (!entry.type(exportType)) { this->getServiceData()->addStatExportType(name, exportType); - entry.exports = entry.exports | mask; + entry.type(exportType, true); } - return entry.ptr.get(); + return entry.raw(); } template @@ -152,11 +150,11 @@ template typename ThreadLocalStatsMapT::TLHistogram* ThreadLocalStatsMapT< LockTraits>::getHistogramLockedPtr(State& state, folly::StringPiece name) { auto& entry = state.namedHistograms_[name]; - if (!entry.ptr) { - entry.ptr = this->createHistogramLocked(state, name); + if (!entry) { + entry.ptr(this->createHistogramLocked(state, name)); } - return entry.ptr.get(); + return entry.raw(); } template @@ -185,11 +183,11 @@ ThreadLocalStatsMapT::getHistogramLocked( State& state, folly::StringPiece name) { auto& entry = state.namedHistograms_[name]; - if (!entry.ptr) { - entry.ptr = this->createHistogramLocked(state, name); + if (!entry) { + entry.ptr(this->createHistogramLocked(state, name)); } - return entry.ptr; + return entry.ptr(); } template @@ -197,20 +195,20 @@ std::shared_ptr::TLCounter> ThreadLocalStatsMapT::getCounterSafe(folly::StringPiece name) { auto state = state_.lock(); auto& entry = state->namedCounters_[name]; - if (!entry.ptr) { - entry.ptr = std::make_shared(this, name); + if (!entry) { + entry.ptr(std::make_shared(this, name)); } - return entry.ptr; + return entry.ptr(); } template typename ThreadLocalStatsMapT::TLCounter* ThreadLocalStatsMapT< LockTraits>::getCounterLocked(State& state, folly::StringPiece name) { auto& entry = state.namedCounters_[name]; - if (!entry.ptr) { - entry.ptr = std::make_shared(this, name); + if (!entry) { + entry.ptr(std::make_shared(this, name)); } - return entry.ptr.get(); + return entry.raw(); } template diff --git a/fb303/ThreadLocalStatsMap.h b/fb303/ThreadLocalStatsMap.h index 80dabb69e..520f1269a 100644 --- a/fb303/ThreadLocalStatsMap.h +++ b/fb303/ThreadLocalStatsMap.h @@ -128,11 +128,115 @@ class ThreadLocalStatsMapT : public ThreadLocalStatsT { using NamedMapLock = typename TLStatsNoLocking::RegistryLock; template - struct StatPtr { - std::shared_ptr ptr; - uintptr_t exports{}; + class StatPtrBase { + protected: + static inline constexpr size_t nbits = sizeof(uintptr_t) * 8; + static inline constexpr size_t ntypes = ExportTypeMeta::kNumExportTypes; + static inline constexpr uintptr_t types_mask = ~uintptr_t(0) + << (nbits - ntypes); + + static constexpr uintptr_t mask_(ExportType key) noexcept { + assert(size_t(key) < ntypes); + return uintptr_t(1) << (nbits - ntypes + size_t(key)); + } + }; + + /// Stores the export-type list alongside the stat-ptr. + template + class StatPtrFallback : private StatPtrBase { + private: + std::shared_ptr ptr_; + uintptr_t exports_{}; + + using StatPtrBase::mask_; + + public: + explicit operator bool() const noexcept { + return !!ptr_; + } + std::shared_ptr ptr() const noexcept { + return ptr_; + } + void ptr(std::shared_ptr&& val) noexcept { + ptr_ = std::move(val); + } + StatType* raw() const noexcept { + return ptr_.get(); + } + bool type(ExportType key) const noexcept { + return exports_ & mask_(key); + } + void type(ExportType key, bool val) noexcept { + exports_ = val ? exports_ | mask_(key) : exports_ & ~mask_(key); + } }; + /// Embeds the export-type list into the stat-ptr control-block-pointer. + /// + /// The export-type list uses the high 5 bits of the control-block pointer. + /// This assumption is valid on most platforms under most configurations. + template + class StatPtrCompress : private StatPtrBase { + private: + using Sp = std::shared_ptr; + /// The layout of std::shared_ptr in all major library implementations. + struct SpLayout { + StatType* raw{}; + uintptr_t ctl{}; + }; + + SpLayout rep_; + + using StatPtrBase::mask_; + using StatPtrBase::types_mask; + + static Sp& cast(SpLayout& rep) noexcept { + FOLLY_PUSH_WARNING + FOLLY_GCC_DISABLE_WARNING("-Wstrict-aliasing") + return reinterpret_cast(rep); + FOLLY_POP_WARNING + } + + public: + StatPtrCompress() = default; + ~StatPtrCompress() { + rep_.ctl &= ~types_mask; + cast(rep_).~Sp(); + } + StatPtrCompress(StatPtrCompress&& that) noexcept + : rep_{std::exchange(that.rep_, {})} {} + StatPtrCompress(StatPtrCompress const& that) = delete; + void operator=(StatPtrCompress&& that) = delete; + void operator=(StatPtrCompress const& that) = delete; + explicit operator bool() const noexcept { + return !!rep_.raw; + } + std::shared_ptr ptr() const noexcept { + auto rep = rep_; + rep.ctl &= ~types_mask; + return cast(rep); + } + void ptr(std::shared_ptr&& val) noexcept { + rep_.ctl &= ~types_mask; + cast(rep_) = std::move(val); + } + StatType* raw() const noexcept { + return rep_.raw; + } + bool type(ExportType key) const noexcept { + return rep_.ctl & mask_(key); + } + void type(ExportType key, bool val) noexcept { + rep_.ctl = val ? rep_.ctl | mask_(key) : rep_.ctl & ~mask_(key); + } + }; + + template + using StatPtr = folly::conditional_t< + folly::kIsArchAmd64 || folly::kIsArchAArch64, + StatPtrCompress, + StatPtrFallback>; + template using StatMap = folly::F14FastMap>;