feat(github): GraphQL cache for datasources (#19059)

Co-authored-by: Rhys Arkins <[email protected]>

lib/util/github/graphql/cache-strategies/abstract-cache-strategy.ts

@@ -0,0 +1,152 @@
+import { DateTime } from 'luxon';
+import type {
+  GithubDatasourceItem,
+  GithubGraphqlCacheRecord,
+  GithubGraphqlCacheStrategy,
+} from '../types';
+import { isDateExpired } from '../util';
+
+/**
+ * Cache strategy handles the caching Github GraphQL items
+ * and reconciling them with newly obtained ones from paginated queries.
+ */
+export abstract class AbstractGithubGraphqlCacheStrategy<
+  GithubItem extends GithubDatasourceItem
+> implements GithubGraphqlCacheStrategy<GithubItem>
+{
+  /**
+   * Time period after which a cache record is considered expired.
+   */
+  protected static readonly cacheTTLDays = 30;
+
+  /**
+   * The time which is used during single cache access cycle.
+   */
+  protected readonly now = DateTime.now();
+
+  /**
+   * Set of all versions which were reconciled
+   * during the current cache access cycle.
+   */
+  private readonly reconciledVersions = new Set<string>();
+
+  /**
+   * These fields will be persisted.
+   */
+  private items: Record<string, GithubItem> | undefined;
+  protected createdAt = this.now;
+  protected updatedAt = this.now;
+
+  constructor(
+    protected readonly cacheNs: string,
+    protected readonly cacheKey: string
+  ) {}
+
+  /**
+   * Load data previously persisted by this strategy
+   * for given `cacheNs` and `cacheKey`.
+   */
+  private async getItems(): Promise<Record<string, GithubItem>> {
+    if (this.items) {
+      return this.items;
+    }
+
+    let result: GithubGraphqlCacheRecord<GithubItem> = {
+      items: {},
+      createdAt: this.createdAt.toISO(),
+      updatedAt: this.updatedAt.toISO(),
+    };
+
+    const storedData = await this.load();
+    if (storedData) {
+      const cacheTTLDuration = {
+        days: AbstractGithubGraphqlCacheStrategy.cacheTTLDays,
+      };
+      if (!isDateExpired(this.now, storedData.createdAt, cacheTTLDuration)) {
+        result = storedData;
+      }
+    }
+
+    this.createdAt = DateTime.fromISO(result.createdAt);
+    this.updatedAt = DateTime.fromISO(result.updatedAt);
+    this.items = result.items;
+    return this.items;
+  }
+
+  /**
+   * If package release exists longer than this cache can exist,
+   * we assume it won't updated/removed on the Github side.
+   */
+  private isStabilized(item: GithubItem): boolean {
+    const unstableDuration = {
+      days: AbstractGithubGraphqlCacheStrategy.cacheTTLDays,
+    };
+    return isDateExpired(this.now, item.releaseTimestamp, unstableDuration);
+  }
+
+  /**
+   * Process items received from GraphQL page
+   * ordered by `releaseTimestamp` in descending order
+   * (fresh versions go first).
+   */
+  async reconcile(items: GithubItem[]): Promise<boolean> {
+    const cachedItems = await this.getItems();
+
+    let isPaginationDone = false;
+    for (const item of items) {
+      const { version } = item;
+
+      // If we reached previously stored item that is stabilized,
+      // we assume the further pagination will not yield any new items.
+      const oldItem = cachedItems[version];
+      if (oldItem && this.isStabilized(oldItem)) {
+        isPaginationDone = true;
+        break;
+      }
+
+      cachedItems[version] = item;
+      this.reconciledVersions.add(version);
+    }
+
+    this.items = cachedItems;
+    return isPaginationDone;
+  }
+
+  /**
+   * Handle removed items for packages that are not stabilized
+   * and return the list of all items.
+   */
+  async finalize(): Promise<GithubItem[]> {
+    const cachedItems = await this.getItems();
+    const resultItems: Record<string, GithubItem> = {};
+
+    for (const [version, item] of Object.entries(cachedItems)) {
+      if (this.isStabilized(item) || this.reconciledVersions.has(version)) {
+        resultItems[version] = item;
+      }
+    }
+
+    await this.store(resultItems);
+    return Object.values(resultItems);
+  }
+
+  /**
+   * Update `updatedAt` field and persist the data.
+   */
+  private async store(cachedItems: Record<string, GithubItem>): Promise<void> {
+    const cacheRecord: GithubGraphqlCacheRecord<GithubItem> = {
+      items: cachedItems,
+      createdAt: this.createdAt.toISO(),
+      updatedAt: this.now.toISO(),
+    };
+    await this.persist(cacheRecord);
+  }
+
+  /**
+   * Loading and persisting data is delegated to the concrete strategy.
+   */
+  abstract load(): Promise<GithubGraphqlCacheRecord<GithubItem> | undefined>;
+  abstract persist(
+    cacheRecord: GithubGraphqlCacheRecord<GithubItem>
+  ): Promise<void>;
+}

lib/util/github/graphql/cache-strategies/memory-cache-strategy.spec.ts

@@ -0,0 +1,188 @@
+import { DateTime, Settings } from 'luxon';
+import * as memCache from '../../../cache/memory';
+import { clone } from '../../../clone';
+import type { GithubDatasourceItem, GithubGraphqlCacheRecord } from '../types';
+import { GithubGraphqlMemoryCacheStrategy } from './memory-cache-strategy';
+
+const isoTs = (t: string) => DateTime.fromJSDate(new Date(t)).toISO();
+
+const mockTime = (input: string): void => {
+  const now = DateTime.fromISO(isoTs(input)).valueOf();
+  Settings.now = () => now;
+};
+
+type CacheRecord = GithubGraphqlCacheRecord<GithubDatasourceItem>;
+
+describe('util/github/graphql/cache-strategies/memory-cache-strategy', () => {
+  beforeEach(() => {
+    jest.resetAllMocks();
+    memCache.init();
+  });
+
+  it('resets old cache', async () => {
+    const items = {
+      '1': { version: '1', releaseTimestamp: isoTs('2020-01-01 10:00') },
+    };
+    const cacheRecord: CacheRecord = {
+      items,
+      createdAt: isoTs('2022-10-01 15:30'),
+      updatedAt: isoTs('2022-10-30 12:35'),
+    };
+    memCache.set('github-graphql-cache:foo:bar', clone(cacheRecord));
+
+    // At this moment, cache is valid
+    let now = '2022-10-31 15:29:59';
+    mockTime(now);
+
+    let strategy = new GithubGraphqlMemoryCacheStrategy('foo', 'bar');
+    let isPaginationDone = await strategy.reconcile([items['1']]);
+    let res = await strategy.finalize();
+
+    expect(res).toEqual(Object.values(items));
+    expect(isPaginationDone).toBe(true);
+    expect(memCache.get('github-graphql-cache:foo:bar')).toEqual({
+      ...cacheRecord,
+      updatedAt: isoTs(now),
+    });
+
+    // One second later, the cache is invalid
+    now = '2022-10-31 15:30:00';
+    mockTime(now);
+
+    strategy = new GithubGraphqlMemoryCacheStrategy('foo', 'bar');
+    isPaginationDone = await strategy.reconcile([]);
+    res = await strategy.finalize();
+
+    expect(res).toEqual([]);
+    expect(isPaginationDone).toBe(false);
+    expect(memCache.get('github-graphql-cache:foo:bar')).toEqual({
+      items: {},
+      createdAt: isoTs(now),
+      updatedAt: isoTs(now),
+    });
+  });
+
+  it('reconciles old cache record with new items', async () => {
+    const oldItems = {
+      '1': { version: '1', releaseTimestamp: isoTs('2020-01-01 10:00') },
+      '2': { version: '2', releaseTimestamp: isoTs('2020-01-01 11:00') },
+      '3': { version: '3', releaseTimestamp: isoTs('2020-01-01 12:00') },
+    };
+    const cacheRecord: CacheRecord = {
+      items: oldItems,
+      createdAt: isoTs('2022-10-30 12:00'),
+      updatedAt: isoTs('2022-10-30 12:00'),
+    };
+    memCache.set('github-graphql-cache:foo:bar', clone(cacheRecord));
+
+    const now = '2022-10-31 15:30';
+    mockTime(now);
+
+    const newItem = {
+      version: '4',
+      releaseTimestamp: isoTs('2022-10-15 18:00'),
+    };
+    const page = [newItem];
+
+    const strategy = new GithubGraphqlMemoryCacheStrategy('foo', 'bar');
+    const isPaginationDone = await strategy.reconcile(page);
+    const res = await strategy.finalize();
+
+    expect(res).toEqual([...Object.values(oldItems), newItem]);
+    expect(isPaginationDone).toBe(false);
+    expect(memCache.get('github-graphql-cache:foo:bar')).toEqual({
+      items: {
+        ...oldItems,
+        '4': newItem,
+      },
+      createdAt: isoTs('2022-10-30 12:00'),
+      updatedAt: isoTs(now),
+    });
+  });
+
+  it('signals to stop pagination', async () => {
+    const oldItems = {
+      '1': { releaseTimestamp: isoTs('2020-01-01 10:00'), version: '1' },
+      '2': { releaseTimestamp: isoTs('2020-01-01 11:00'), version: '2' },
+      '3': { releaseTimestamp: isoTs('2020-01-01 12:00'), version: '3' },
+    };
+    const cacheRecord: CacheRecord = {
+      items: oldItems,
+      createdAt: isoTs('2022-10-30 12:00'),
+      updatedAt: isoTs('2022-10-30 12:00'),
+    };
+    memCache.set('github-graphql-cache:foo:bar', clone(cacheRecord));
+
+    const now = '2022-10-31 15:30';
+    mockTime(now);
+
+    const page = [
+      ...Object.values(oldItems),
+      { version: '4', releaseTimestamp: isoTs('2022-10-15 18:00') },
+    ].reverse();
+
+    const strategy = new GithubGraphqlMemoryCacheStrategy('foo', 'bar');
+    const isPaginationDone = await strategy.reconcile(page);
+
+    expect(isPaginationDone).toBe(true);
+  });
+
+  it('detects removed packages', async () => {
+    const items = {
+      // stabilized
+      '0': { version: '0', releaseTimestamp: isoTs('2022-09-30 10:00') }, // to be preserved
+      '1': { version: '1', releaseTimestamp: isoTs('2022-10-01 10:00') }, // to be preserved
+      // not stabilized
+      '2': { version: '2', releaseTimestamp: isoTs('2022-10-02 10:00') },
+      '3': { version: '3', releaseTimestamp: isoTs('2022-10-03 10:00') }, // to be deleted
+      '4': { version: '4', releaseTimestamp: isoTs('2022-10-04 10:00') },
+      '5': { version: '5', releaseTimestamp: isoTs('2022-10-05 10:00') }, // to be deleted
+      '6': { version: '6', releaseTimestamp: isoTs('2022-10-06 10:00') },
+      '7': { version: '7', releaseTimestamp: isoTs('2022-10-07 10:00') }, // to be deleted
+      '8': { version: '8', releaseTimestamp: isoTs('2022-10-08 10:00') },
+    };
+    const cacheRecord: CacheRecord = {
+      items,
+      createdAt: isoTs('2022-10-30 12:00'),
+      updatedAt: isoTs('2022-10-30 12:00'),
+    };
+    memCache.set('github-graphql-cache:foo:bar', clone(cacheRecord));
+
+    const now = '2022-10-31 15:30';
+    mockTime(now);
+
+    const page = [
+      items['1'],
+      items['2'],
+      items['4'],
+      items['6'],
+      items['8'],
+    ].reverse();
+
+    const strategy = new GithubGraphqlMemoryCacheStrategy('foo', 'bar');
+    const isPaginationDone = await strategy.reconcile(page);
+    const res = await strategy.finalize();
+
+    expect(res).toEqual([
+      { version: '0', releaseTimestamp: isoTs('2022-09-30 10:00') },
+      { version: '1', releaseTimestamp: isoTs('2022-10-01 10:00') },
+      { version: '2', releaseTimestamp: isoTs('2022-10-02 10:00') },
+      { version: '4', releaseTimestamp: isoTs('2022-10-04 10:00') },
+      { version: '6', releaseTimestamp: isoTs('2022-10-06 10:00') },
+      { version: '8', releaseTimestamp: isoTs('2022-10-08 10:00') },
+    ]);
+    expect(isPaginationDone).toBe(true);
+    expect(memCache.get('github-graphql-cache:foo:bar')).toEqual({
+      items: {
+        '0': { version: '0', releaseTimestamp: isoTs('2022-09-30 10:00') },
+        '1': { version: '1', releaseTimestamp: isoTs('2022-10-01 10:00') },
+        '2': { version: '2', releaseTimestamp: isoTs('2022-10-02 10:00') },
+        '4': { version: '4', releaseTimestamp: isoTs('2022-10-04 10:00') },
+        '6': { version: '6', releaseTimestamp: isoTs('2022-10-06 10:00') },
+        '8': { version: '8', releaseTimestamp: isoTs('2022-10-08 10:00') },
+      },
+      createdAt: isoTs('2022-10-30 12:00'),
+      updatedAt: isoTs('2022-10-31 15:30'),
+    });
+  });
+});

lib/util/github/graphql/cache-strategies/memory-cache-strategy.ts

@@ -0,0 +1,27 @@
+import * as memCache from '../../../cache/memory';
+import type { GithubDatasourceItem, GithubGraphqlCacheRecord } from '../types';
+import { AbstractGithubGraphqlCacheStrategy } from './abstract-cache-strategy';
+
+/**
+ * In-memory strategy meant to be used for private packages
+ * and for testing purposes.
+ */
+export class GithubGraphqlMemoryCacheStrategy<
+  GithubItem extends GithubDatasourceItem
+> extends AbstractGithubGraphqlCacheStrategy<GithubItem> {
+  private fullKey(): string {
+    return `github-graphql-cache:${this.cacheNs}:${this.cacheKey}`;
+  }
+
+  load(): Promise<GithubGraphqlCacheRecord<GithubItem> | undefined> {
+    const key = this.fullKey();
+    const res = memCache.get(key);
+    return Promise.resolve(res);
+  }
+
+  persist(cacheRecord: GithubGraphqlCacheRecord<GithubItem>): Promise<void> {
+    const key = this.fullKey();
+    memCache.set(key, cacheRecord);
+    return Promise.resolve();
+  }
+}