Sanitize the secondary_cache option in TieredCacheOptions (#12137)

Summary:
Sanitize the `secondary_cache` field in the `cache_opts` option of `TieredCacheOptions` to `nullptr` if set by the user. The nvm secondary cache should be directly set in `TieredCacheOptions`.

Pull Request resolved: #12137

Reviewed By: akankshamahajan15

Differential Revision: D52063817

Pulled By: anand1976

fbshipit-source-id: 255116c665a9b908c8f44109a2d331d4b73e7591

cache/secondary_cache_adapter.cc

@@ -683,12 +683,14 @@ std::shared_ptr<Cache> NewTieredCache(const TieredCacheOptions& _opts) {
         *(static_cast_with_check<LRUCacheOptions, ShardedCacheOptions>(
             opts.cache_opts));
     cache_opts.capacity = opts.total_capacity;
+    cache_opts.secondary_cache = nullptr;
     cache = cache_opts.MakeSharedCache();
   } else if (opts.cache_type == PrimaryCacheType::kCacheTypeHCC) {
     HyperClockCacheOptions cache_opts =
         *(static_cast_with_check<HyperClockCacheOptions, ShardedCacheOptions>(
             opts.cache_opts));
     cache_opts.capacity = opts.total_capacity;
+    cache_opts.secondary_cache = nullptr;
     cache = cache_opts.MakeSharedCache();
   } else {
     return nullptr;

include/rocksdb/cache.h

@@ -499,6 +499,10 @@ enum TieredAdmissionPolicy {
 // allocations costed to the block cache, will be distributed
 // proportionally across both the primary and secondary.
 struct TieredCacheOptions {
+  // This should point to an instance of either LRUCacheOptions or
+  // HyperClockCacheOptions, depending on the cache_type. In either
+  // case, the capacity and secondary_cache fields in those options
+  // should not be set. If set, they will be ignored by NewTieredCache.
   ShardedCacheOptions* cache_opts = nullptr;
   PrimaryCacheType cache_type = PrimaryCacheType::kCacheTypeLRU;
   TieredAdmissionPolicy adm_policy = TieredAdmissionPolicy::kAdmPolicyAuto;