Skip to content

Commit

Permalink
shrink ThreadLocalStatsMap mapped items
Browse files Browse the repository at this point in the history 
Summary:
{D68659181} incorporated a cache of export-types into ThreadLocalStatsMap by embedding the list of per-stat export-types into the mapped items as a separate pointer-sized value.

But we can embed the list into pre-existing words in the mapped items. Here, we embed them into the high bits of the shared-ptr control-block pointer.

To do this, we use the well-known cross-library layout of `std::shared_ptr`. As an alternative, we could embed the export-type list into the high bits of the shared-ptr object pointer. If done this way, it is possible to make the whole thing work with only the `std::shared_ptr` aliasing constructor and without taking advantage of the well-known cross-library layout.

Differential Revision: D68665021

fbshipit-source-id: 0c6999758c1928703b4d10485d3db12765066033
  • Loading branch information
@yfeldblum @facebook-github-bot
yfeldblum authored and facebook-github-bot committed Jan 28, 2025
1 parent 057f91f commit 3a66ed4
Show file tree
Hide file tree
Showing 2 changed files with 135 additions and 33 deletions.
58 changes: 28 additions & 30 deletions fb303/ThreadLocalStatsMap-inl.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,8 +55,7 @@ void ThreadLocalStatsMapT<LockTraits>::clearStat(
ExportType exportType) {
auto state = state_.lock();
if (auto ptr = folly::get_ptr(state->namedTimeseries_, name)) {
auto mask = uintptr_t(1) << size_t(exportType);
ptr->exports = ptr->exports & ~mask;
ptr->type(exportType, false);
}
this->getServiceData()->addStatExportType(name, exportType, nullptr);
}
Expand Down Expand Up @@ -85,10 +84,10 @@ std::shared_ptr<typename ThreadLocalStatsMapT<LockTraits>::TLTimeseries>
ThreadLocalStatsMapT<LockTraits>::getTimeseriesSafe(folly::StringPiece name) {
auto state = state_.lock();
auto& entry = state->namedTimeseries_[name];
if (!entry.ptr) {
entry.ptr = std::make_shared<TLTimeseries>(this, name);
if (!entry) {
entry.ptr(std::make_shared<TLTimeseries>(this, name));
}
return entry.ptr;
return entry.ptr();
}

template <class LockTraits>
Expand All @@ -100,21 +99,21 @@ ThreadLocalStatsMapT<LockTraits>::getTimeseriesSafe(
const int levelDurations[]) {
auto state = state_.lock();
auto& entry = state->namedTimeseries_[name];
if (!entry.ptr) {
entry.ptr = std::make_shared<TLTimeseries>(
this, name, numBuckets, numLevels, levelDurations);
if (!entry) {
entry.ptr(std::make_shared<TLTimeseries>(
this, name, numBuckets, numLevels, levelDurations));
}
return entry.ptr;
return entry.ptr();
}

template <class LockTraits>
typename ThreadLocalStatsMapT<LockTraits>::TLTimeseries* ThreadLocalStatsMapT<
LockTraits>::getTimeseriesLocked(State& state, folly::StringPiece name) {
auto& entry = state.namedTimeseries_[name];
if (!entry.ptr) {
entry.ptr = std::make_shared<TLTimeseries>(this, name);
if (!entry) {
entry.ptr(std::make_shared<TLTimeseries>(this, name));
}
return entry.ptr.get();
return entry.raw();
}

template <class LockTraits>
Expand All @@ -124,15 +123,14 @@ ThreadLocalStatsMapT<LockTraits>::getTimeseriesLocked(
folly::StringPiece name,
ExportType exportType) {
auto& entry = state.namedTimeseries_[name];
if (!entry.ptr) {
entry.ptr = std::make_shared<TLTimeseries>(this, name);
if (!entry) {
entry.ptr(std::make_shared<TLTimeseries>(this, name));
}
auto mask = uintptr_t(1) << size_t(exportType);
if (!(entry.exports & mask)) {
if (!entry.type(exportType)) {
this->getServiceData()->addStatExportType(name, exportType);
entry.exports = entry.exports | mask;
entry.type(exportType, true);
}
return entry.ptr.get();
return entry.raw();
}

template <class LockTraits>
Expand All @@ -152,11 +150,11 @@ template <class LockTraits>
typename ThreadLocalStatsMapT<LockTraits>::TLHistogram* ThreadLocalStatsMapT<
LockTraits>::getHistogramLockedPtr(State& state, folly::StringPiece name) {
auto& entry = state.namedHistograms_[name];
if (!entry.ptr) {
entry.ptr = this->createHistogramLocked(state, name);
if (!entry) {
entry.ptr(this->createHistogramLocked(state, name));
}

return entry.ptr.get();
return entry.raw();
}

template <class LockTraits>
Expand Down Expand Up @@ -185,32 +183,32 @@ ThreadLocalStatsMapT<LockTraits>::getHistogramLocked(
State& state,
folly::StringPiece name) {
auto& entry = state.namedHistograms_[name];
if (!entry.ptr) {
entry.ptr = this->createHistogramLocked(state, name);
if (!entry) {
entry.ptr(this->createHistogramLocked(state, name));
}

return entry.ptr;
return entry.ptr();
}

template <class LockTraits>
std::shared_ptr<typename ThreadLocalStatsMapT<LockTraits>::TLCounter>
ThreadLocalStatsMapT<LockTraits>::getCounterSafe(folly::StringPiece name) {
auto state = state_.lock();
auto& entry = state->namedCounters_[name];
if (!entry.ptr) {
entry.ptr = std::make_shared<TLCounter>(this, name);
if (!entry) {
entry.ptr(std::make_shared<TLCounter>(this, name));
}
return entry.ptr;
return entry.ptr();
}

template <class LockTraits>
typename ThreadLocalStatsMapT<LockTraits>::TLCounter* ThreadLocalStatsMapT<
LockTraits>::getCounterLocked(State& state, folly::StringPiece name) {
auto& entry = state.namedCounters_[name];
if (!entry.ptr) {
entry.ptr = std::make_shared<TLCounter>(this, name);
if (!entry) {
entry.ptr(std::make_shared<TLCounter>(this, name));
}
return entry.ptr.get();
return entry.raw();
}

template <class LockTraits>
Expand Down
110 changes: 107 additions & 3 deletions fb303/ThreadLocalStatsMap.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -128,11 +128,115 @@ class ThreadLocalStatsMapT : public ThreadLocalStatsT<LockTraits> {
using NamedMapLock = typename TLStatsNoLocking::RegistryLock;

template <class StatType>
struct StatPtr {
std::shared_ptr<StatType> ptr;
uintptr_t exports{};
class StatPtrBase {
protected:
static inline constexpr size_t nbits = sizeof(uintptr_t) * 8;
static inline constexpr size_t ntypes = ExportTypeMeta::kNumExportTypes;
static inline constexpr uintptr_t types_mask = ~uintptr_t(0)
<< (nbits - ntypes);

static constexpr uintptr_t mask_(ExportType key) noexcept {
assert(size_t(key) < ntypes);
return uintptr_t(1) << (nbits - ntypes + size_t(key));
}
};

/// Stores the export-type list alongside the stat-ptr.
template <class StatType>
class StatPtrFallback : private StatPtrBase<StatType> {
private:
std::shared_ptr<StatType> ptr_;
uintptr_t exports_{};

using StatPtrBase<StatType>::mask_;

public:
explicit operator bool() const noexcept {
return !!ptr_;
}
std::shared_ptr<StatType> ptr() const noexcept {
return ptr_;
}
void ptr(std::shared_ptr<StatType>&& val) noexcept {
ptr_ = std::move(val);
}
StatType* raw() const noexcept {
return ptr_.get();
}
bool type(ExportType key) const noexcept {
return exports_ & mask_(key);
}
void type(ExportType key, bool val) noexcept {
exports_ = val ? exports_ | mask_(key) : exports_ & ~mask_(key);
}
};

/// Embeds the export-type list into the stat-ptr control-block-pointer.
///
/// The export-type list uses the high 5 bits of the control-block pointer.
/// This assumption is valid on most platforms under most configurations.
template <class StatType>
class StatPtrCompress : private StatPtrBase<StatType> {
private:
using Sp = std::shared_ptr<StatType>;
/// The layout of std::shared_ptr in all major library implementations.
struct SpLayout {
StatType* raw{};
uintptr_t ctl{};
};

SpLayout rep_;

using StatPtrBase<StatType>::mask_;
using StatPtrBase<StatType>::types_mask;

static Sp& cast(SpLayout& rep) noexcept {
FOLLY_PUSH_WARNING
FOLLY_GCC_DISABLE_WARNING("-Wstrict-aliasing")
return reinterpret_cast<Sp&>(rep);
FOLLY_POP_WARNING
}

public:
StatPtrCompress() = default;
~StatPtrCompress() {
rep_.ctl &= ~types_mask;
cast(rep_).~Sp();
}
StatPtrCompress(StatPtrCompress&& that) noexcept
: rep_{std::exchange(that.rep_, {})} {}
StatPtrCompress(StatPtrCompress const& that) = delete;
void operator=(StatPtrCompress&& that) = delete;
void operator=(StatPtrCompress const& that) = delete;
explicit operator bool() const noexcept {
return !!rep_.raw;
}
std::shared_ptr<StatType> ptr() const noexcept {
auto rep = rep_;
rep.ctl &= ~types_mask;
return cast(rep);
}
void ptr(std::shared_ptr<StatType>&& val) noexcept {
rep_.ctl &= ~types_mask;
cast(rep_) = std::move(val);
}
StatType* raw() const noexcept {
return rep_.raw;
}
bool type(ExportType key) const noexcept {
return rep_.ctl & mask_(key);
}
void type(ExportType key, bool val) noexcept {
rep_.ctl = val ? rep_.ctl | mask_(key) : rep_.ctl & ~mask_(key);
}
};

template <class StatType>
using StatPtr = folly::conditional_t<
folly::kIsArchAmd64 || folly::kIsArchAArch64,
StatPtrCompress<StatType>,
StatPtrFallback<StatType>>;

template <class StatType>
using StatMap = folly::F14FastMap<std::string, StatPtr<StatType>>;

Expand Down

0 comments on commit 3a66ed4

Please sign in to comment.