feat(github): Don't store updatedAt field in GraphQL cache record (#…

…21024)

lib/util/github/graphql/cache-strategies/abstract-cache-strategy.ts

@@ -1,3 +1,4 @@
+import { dequal } from 'dequal';
 import { DateTime } from 'luxon';
 import type {
   GithubDatasourceItem,
@@ -35,7 +36,20 @@ export abstract class AbstractGithubGraphqlCacheStrategy<
    */
   private items: Record<string, GithubItem> | undefined;
   protected createdAt = this.now;
-  protected updatedAt = this.now;
+
+  /**
+   * This flag helps to indicate whether the cache record
+   * should be persisted after the current cache access cycle.
+   */
+  protected hasUpdatedItems = false;
+
+  /**
+   * Loading and persisting data is delegated to the concrete strategy.
+   */
+  abstract load(): Promise<GithubGraphqlCacheRecord<GithubItem> | undefined>;
+  abstract persist(
+    cacheRecord: GithubGraphqlCacheRecord<GithubItem>
+  ): Promise<void>;
 
   constructor(
     protected readonly cacheNs: string,
@@ -54,7 +68,6 @@ export abstract class AbstractGithubGraphqlCacheStrategy<
     let result: GithubGraphqlCacheRecord<GithubItem> = {
       items: {},
       createdAt: this.createdAt.toISO(),
-      updatedAt: this.updatedAt.toISO(),
     };
 
     const storedData = await this.load();
@@ -68,7 +81,6 @@ export abstract class AbstractGithubGraphqlCacheStrategy<
     }
 
     this.createdAt = DateTime.fromISO(result.createdAt);
-    this.updatedAt = DateTime.fromISO(result.updatedAt);
     this.items = result.items;
     return this.items;
   }
@@ -99,8 +111,14 @@ export abstract class AbstractGithubGraphqlCacheStrategy<
       // If we reached previously stored item that is stabilized,
       // we assume the further pagination will not yield any new items.
       const oldItem = cachedItems[version];
-      if (oldItem && this.isStabilized(oldItem)) {
-        isPaginationDone = true;
+      if (oldItem) {
+        if (this.isStabilized(oldItem)) {
+          isPaginationDone = true;
+        }
+
+        if (!dequal(oldItem, item)) {
+          this.hasUpdatedItems = true;
+        }
       }
 
       cachedItems[version] = item;
@@ -129,23 +147,11 @@ export abstract class AbstractGithubGraphqlCacheStrategy<
     return Object.values(resultItems);
   }
 
-  /**
-   * Update `updatedAt` field and persist the data.
-   */
   private async store(cachedItems: Record<string, GithubItem>): Promise<void> {
     const cacheRecord: GithubGraphqlCacheRecord<GithubItem> = {
       items: cachedItems,
       createdAt: this.createdAt.toISO(),
-      updatedAt: this.now.toISO(),
     };
     await this.persist(cacheRecord);
   }
-
-  /**
-   * Loading and persisting data is delegated to the concrete strategy.
-   */
-  abstract load(): Promise<GithubGraphqlCacheRecord<GithubItem> | undefined>;
-  abstract persist(
-    cacheRecord: GithubGraphqlCacheRecord<GithubItem>
-  ): Promise<void>;
 }

lib/util/github/graphql/cache-strategies/memory-cache-strategy.spec.ts

@@ -26,7 +26,6 @@ describe('util/github/graphql/cache-strategies/memory-cache-strategy', () => {
     const cacheRecord: CacheRecord = {
       items,
       createdAt: isoTs('2022-10-01 15:30'),
-      updatedAt: isoTs('2022-10-30 12:35'),
     };
     memCache.set('github-graphql-cache:foo:bar', clone(cacheRecord));
 
@@ -40,10 +39,7 @@ describe('util/github/graphql/cache-strategies/memory-cache-strategy', () => {
 
     expect(res).toEqual(Object.values(items));
     expect(isPaginationDone).toBe(true);
-    expect(memCache.get('github-graphql-cache:foo:bar')).toEqual({
-      ...cacheRecord,
-      updatedAt: isoTs(now),
-    });
+    expect(memCache.get('github-graphql-cache:foo:bar')).toEqual(cacheRecord);
 
     // One second later, the cache is invalid
     now = '2022-10-31 15:30:00';
@@ -58,7 +54,6 @@ describe('util/github/graphql/cache-strategies/memory-cache-strategy', () => {
     expect(memCache.get('github-graphql-cache:foo:bar')).toEqual({
       items: {},
       createdAt: isoTs(now),
-      updatedAt: isoTs(now),
     });
   });
 
@@ -71,7 +66,6 @@ describe('util/github/graphql/cache-strategies/memory-cache-strategy', () => {
     const cacheRecord: CacheRecord = {
       items: oldItems,
       createdAt: isoTs('2022-10-30 12:00'),
-      updatedAt: isoTs('2022-10-30 12:00'),
     };
     memCache.set('github-graphql-cache:foo:bar', clone(cacheRecord));
 
@@ -96,7 +90,6 @@ describe('util/github/graphql/cache-strategies/memory-cache-strategy', () => {
         '4': newItem,
       },
       createdAt: isoTs('2022-10-30 12:00'),
-      updatedAt: isoTs(now),
     });
   });
 
@@ -109,7 +102,6 @@ describe('util/github/graphql/cache-strategies/memory-cache-strategy', () => {
     const cacheRecord: CacheRecord = {
       items: oldItems,
       createdAt: isoTs('2022-10-30 12:00'),
-      updatedAt: isoTs('2022-10-30 12:00'),
     };
     memCache.set('github-graphql-cache:foo:bar', clone(cacheRecord));
 
@@ -136,7 +128,6 @@ describe('util/github/graphql/cache-strategies/memory-cache-strategy', () => {
     const cacheRecord: CacheRecord = {
       items: oldItems,
       createdAt: isoTs('2022-12-31 12:00'),
-      updatedAt: isoTs('2022-12-31 12:00'),
     };
     memCache.set('github-graphql-cache:foo:bar', clone(cacheRecord));
 
@@ -181,7 +172,6 @@ describe('util/github/graphql/cache-strategies/memory-cache-strategy', () => {
     const cacheRecord: CacheRecord = {
       items,
       createdAt: isoTs('2022-10-30 12:00'),
-      updatedAt: isoTs('2022-10-30 12:00'),
     };
     memCache.set('github-graphql-cache:foo:bar', clone(cacheRecord));
 
@@ -219,7 +209,6 @@ describe('util/github/graphql/cache-strategies/memory-cache-strategy', () => {
         '8': { version: '8', releaseTimestamp: isoTs('2022-10-08 10:00') },
       },
       createdAt: isoTs('2022-10-30 12:00'),
-      updatedAt: isoTs('2022-10-31 15:30'),
     });
   });
 });

lib/util/github/graphql/cache-strategies/package-cache-strategy.spec.ts

@@ -18,46 +18,52 @@ describe('util/github/graphql/cache-strategies/package-cache-strategy', () => {
   const cacheSet = jest.spyOn(packageCache, 'set');
 
   it('reconciles old cache record with new items', async () => {
+    const item1 = { version: '1', releaseTimestamp: isoTs('2020-01-01 10:00') };
+    const item2 = { version: '2', releaseTimestamp: isoTs('2020-01-01 11:00') };
+    const item3 = { version: '3', releaseTimestamp: isoTs('2020-01-01 12:00') };
+
     const oldItems = {
-      '1': { version: '1', releaseTimestamp: isoTs('2020-01-01 10:00') },
-      '2': { version: '2', releaseTimestamp: isoTs('2020-01-01 11:00') },
-      '3': { version: '3', releaseTimestamp: isoTs('2020-01-01 12:00') },
+      '1': item1,
+      '2': item2,
+      '3': item3,
     };
     const cacheRecord: CacheRecord = {
       items: oldItems,
       createdAt: isoTs('2022-10-15 12:00'),
-      updatedAt: isoTs('2022-10-15 12:00'),
     };
     cacheGet.mockResolvedValueOnce(clone(cacheRecord));
 
     const now = '2022-10-30 12:00';
     mockTime(now);
 
+    const updatedItem = {
+      ...item3,
+      releaseTimestamp: isoTs('2020-01-01 12:30'),
+    };
     const newItem = {
       version: '4',
       releaseTimestamp: isoTs('2022-10-15 18:00'),
     };
-    const page = [newItem];
+    const page = [newItem, updatedItem];
 
     const strategy = new GithubGraphqlPackageCacheStrategy('foo', 'bar');
     const isPaginationDone = await strategy.reconcile(page);
     const res = await strategy.finalize();
 
-    expect(res).toEqual([...Object.values(oldItems), newItem]);
-    expect(isPaginationDone).toBe(false);
+    expect(res).toEqual([item1, item2, updatedItem, newItem]);
+    expect(isPaginationDone).toBe(true);
     expect(cacheSet.mock.calls).toEqual([
       [
         'foo',
         'bar',
         {
           items: {
-            '1': { version: '1', releaseTimestamp: isoTs('2020-01-01 10:00') },
-            '2': { version: '2', releaseTimestamp: isoTs('2020-01-01 11:00') },
-            '3': { version: '3', releaseTimestamp: isoTs('2020-01-01 12:00') },
-            '4': { version: '4', releaseTimestamp: isoTs('2022-10-15 18:00') },
+            '1': item1,
+            '2': item2,
+            '3': updatedItem,
+            '4': newItem,
           },
           createdAt: isoTs('2022-10-15 12:00'),
-          updatedAt: isoTs('2022-10-30 12:00'),
         },
         15 * 24 * 60,
       ],

lib/util/github/graphql/cache-strategies/package-cache-strategy.ts

@@ -15,17 +15,19 @@ export class GithubGraphqlPackageCacheStrategy<
   async persist(
     cacheRecord: GithubGraphqlCacheRecord<GithubItem>
   ): Promise<void> {
-    const expiry = this.createdAt.plus({
-      days: AbstractGithubGraphqlCacheStrategy.cacheTTLDays,
-    });
-    const ttlMinutes = expiry.diff(this.now, ['minutes']).as('minutes');
-    if (ttlMinutes && ttlMinutes > 0) {
-      await packageCache.set(
-        this.cacheNs,
-        this.cacheKey,
-        cacheRecord,
-        ttlMinutes
-      );
+    if (this.hasUpdatedItems) {
+      const expiry = this.createdAt.plus({
+        days: AbstractGithubGraphqlCacheStrategy.cacheTTLDays,
+      });
+      const ttlMinutes = expiry.diff(this.now, ['minutes']).as('minutes');
+      if (ttlMinutes && ttlMinutes > 0) {
+        await packageCache.set(
+          this.cacheNs,
+          this.cacheKey,
+          cacheRecord,
+          ttlMinutes
+        );
+      }
     }
   }
 }

lib/util/github/graphql/readme.md

@@ -99,7 +99,6 @@ As we retrieve items during the fetch process, we gradually construct a data str
     "2.0.0": { "version": "2.0.0", "releaseTimestamp": "2022-09-01" },
   },
   "createdAt": "2022-12-20",
-  "updatedAt": "2022-12-20",
 }
 ```
 
@@ -159,7 +158,6 @@ Given we performed fetch at the day of latest release, new cache looks like:
     "2.0.0": { "version": "2.0.0", "releaseTimestamp": "2022-09-01" },
   },
   "createdAt": "2022-12-20",
-  "updatedAt": "2022-12-30",
 }
 ```
 

lib/util/github/graphql/types.ts

@@ -92,7 +92,6 @@ export interface GithubGraphqlCacheRecord<
 > {
   items: Record<string, GithubItem>;
   createdAt: string;
-  updatedAt: string;
 }
 
 export interface GithubGraphqlCacheStrategy<