feat(cache): file cache cleanup (#20061)

Checks file cache for expired items at the end of a run. Non-breaking change but it may result in some long cleanup jobs for any bots which have been left to populate their package cache for a long time.

Closes #13732

lib/util/cache/package/file.spec.ts

@@ -1,5 +1,6 @@
 import os from 'os';
-import { get, init, set } from './file';
+import cacache from 'cacache';
+import { cleanup, get, init, set } from './file';
 
 describe('util/cache/package/file', () => {
   it('returns if uninitiated', async () => {
@@ -23,4 +24,14 @@ describe('util/cache/package/file', () => {
     await set('test', 'key', 1234, -5);
     expect(await get('test', 'key')).toBeUndefined();
   });
+
+  it('cleans up', async () => {
+    const cacheFileName = init(os.tmpdir());
+    await set('test', 'valid', 1234);
+    await set('test', 'expired', 1234, -5);
+    await cacache.put(cacheFileName, 'invalid', 'not json');
+    await cleanup();
+    const entries = await cacache.ls(cacheFileName);
+    expect(Object.keys(entries)).toEqual(['test-valid']);
+  });
 });

lib/util/cache/package/file.ts

@@ -64,7 +64,42 @@ export async function set(
   );
 }
 
-export function init(cacheDir: string): void {
+export function init(cacheDir: string): string {
   cacheFileName = upath.join(cacheDir, '/renovate/renovate-cache-v1');
   logger.debug('Initializing Renovate internal cache into ' + cacheFileName);
+  return cacheFileName;
+}
+
+export async function cleanup(): Promise<void> {
+  logger.debug('Checking file package cache for expired items');
+  try {
+    let totalCount = 0;
+    let deletedCount = 0;
+    const startTime = Date.now();
+    for await (const item of cacache.ls.stream(cacheFileName)) {
+      totalCount += 1;
+      const cachedItem = item as unknown as cacache.CacheObject;
+      const res = await cacache.get(cacheFileName, cachedItem.key);
+      let cachedValue: any;
+      try {
+        cachedValue = JSON.parse(res.data.toString());
+      } catch (err) {
+        logger.debug('Error parsing cached value - deleting');
+      }
+      if (
+        !cachedValue ||
+        (cachedValue?.expiry &&
+          DateTime.local() > DateTime.fromISO(cachedValue.expiry))
+      ) {
+        await cacache.rm.entry(cacheFileName, cachedItem.key);
+        deletedCount += 1;
+      }
+    }
+    const durationMs = Math.round(Date.now() - startTime);
+    logger.debug(
+      `Deleted ${deletedCount} of ${totalCount} file cached entries in ${durationMs}ms`
+    );
+  } catch (err) /* istanbul ignore next */ {
+    logger.warn({ err }, 'Error cleaning up expired file cache');
+  }
 }

lib/util/cache/package/index.ts

@@ -65,6 +65,7 @@ export async function init(config: AllConfig): Promise<void> {
     cacheProxy = {
       get: fileCache.get,
       set: fileCache.set,
+      cleanup: fileCache.cleanup,
     };
   }
 }
@@ -73,4 +74,7 @@ export async function cleanup(config: AllConfig): Promise<void> {
   if (config?.redisUrl) {
     await redisCache.end();
   }
+  if (cacheProxy.cleanup) {
+    await cacheProxy.cleanup();
+  }
 }

lib/util/cache/package/types.ts

@@ -7,4 +7,6 @@ export interface PackageCache {
     value: T,
     ttlMinutes?: number
   ): Promise<void>;
+
+  cleanup?(): Promise<void>;
 }