feat(cache): Fallback to older decorator results on error

docs/usage/self-hosted-experimental.md

@@ -14,6 +14,10 @@ We do not follow Semantic Versioning for any experimental variables.
 These variables may be removed or have their behavior changed in **any** version.
 We will try to keep breakage to a minimum, but make no guarantees that an experimental variable will keep working.
 
+## `RENOVATE_CACHE_DECORATOR_MINUTES`
+
+If set to any integer, Renovate will use this integer instead of the time explicitly defined via `ttlMinutes` in decorators.
+
 ## `RENOVATE_CACHE_NPM_MINUTES`
 
 If set to any integer, Renovate will use this integer instead of the default npm cache time (15 minutes) for the npm datasource.

lib/util/cache/package/decorator.spec.ts

@@ -1,4 +1,5 @@
 import os from 'os';
+import { GlobalConfig } from '../../../config/global';
 import * as memCache from '../memory';
 import { cache } from './decorator';
 import * as packageCache from '.';
@@ -37,8 +38,8 @@ describe('util/cache/package/decorator', () => {
     expect(getValue).toHaveBeenCalledTimes(1);
     expect(setCache).toHaveBeenCalledOnceWith(
       'some-namespace',
-      'some-key',
-      '111',
+      'cache-decorator:some-key',
+      { cachedAt: expect.any(String), value: '111' },
       30
     );
   });
@@ -75,7 +76,12 @@ describe('util/cache/package/decorator', () => {
     expect(await obj.fn(null)).toBeNull();
 
     expect(getValue).toHaveBeenCalledTimes(1);
-    expect(setCache).toHaveBeenCalledOnceWith('namespace', 'key', null, 30);
+    expect(setCache).toHaveBeenCalledOnceWith(
+      'namespace',
+      'cache-decorator:key',
+      { cachedAt: expect.any(String), value: null },
+      30
+    );
   });
 
   it('does not cache undefined', async () => {
@@ -120,8 +126,8 @@ describe('util/cache/package/decorator', () => {
     expect(getValue).toHaveBeenCalledTimes(1);
     expect(setCache).toHaveBeenCalledOnceWith(
       'some-namespace',
-      'some-key',
-      '111',
+      'cache-decorator:some-key',
+      { cachedAt: expect.any(String), value: '111' },
       30
     );
   });
@@ -140,6 +146,103 @@ describe('util/cache/package/decorator', () => {
     expect(await fn.value?.()).toBe('111');
 
     expect(getValue).toHaveBeenCalledTimes(1);
-    expect(setCache).toHaveBeenCalledOnceWith('namespace', 'key', '111', 30);
+    expect(setCache).toHaveBeenCalledOnceWith(
+      'namespace',
+      'cache-decorator:key',
+      { cachedAt: expect.any(String), value: '111' },
+      30
+    );
+  });
+
+  describe('Fallbacks with hard TTL', () => {
+    class Class {
+      @cache({
+        namespace: 'namespace',
+        key: 'key',
+        ttlMinutes: 1,
+      })
+
+      // Hard TTL is enabled only for `getReleases` and `getDigest` methods
+      public getReleases(): Promise<string> {
+        return getValue();
+      }
+    }
+
+    beforeEach(() => {
+      jest.useFakeTimers({ advanceTimers: false });
+      GlobalConfig.set({ cacheHardTtlMinutes: 2 });
+    });
+
+    afterEach(() => {
+      jest.useRealTimers();
+      GlobalConfig.reset();
+    });
+
+    it('updates cached result', async () => {
+      const obj = new Class();
+
+      expect(await obj.getReleases()).toBe('111');
+      expect(getValue).toHaveBeenCalledTimes(1);
+
+      jest.advanceTimersByTime(60 * 1000 - 1);
+      expect(await obj.getReleases()).toBe('111');
+      expect(getValue).toHaveBeenCalledTimes(1);
+      expect(setCache).toHaveBeenLastCalledWith(
+        'namespace',
+        'cache-decorator:key',
+        { cachedAt: expect.any(String), value: '111' },
+        2
+      );
+
+      jest.advanceTimersByTime(1);
+      expect(await obj.getReleases()).toBe('222');
+      expect(getValue).toHaveBeenCalledTimes(2);
+      expect(setCache).toHaveBeenLastCalledWith(
+        'namespace',
+        'cache-decorator:key',
+        { cachedAt: expect.any(String), value: '222' },
+        2
+      );
+    });
+
+    it('returns obsolete result on error', async () => {
+      const obj = new Class();
+
+      expect(await obj.getReleases()).toBe('111');
+      expect(getValue).toHaveBeenCalledTimes(1);
+      expect(setCache).toHaveBeenLastCalledWith(
+        'namespace',
+        'cache-decorator:key',
+        { cachedAt: expect.any(String), value: '111' },
+        2
+      );
+
+      jest.advanceTimersByTime(60 * 1000);
+      getValue.mockRejectedValueOnce(new Error('test'));
+      expect(await obj.getReleases()).toBe('111');
+      expect(getValue).toHaveBeenCalledTimes(2);
+      expect(setCache).toHaveBeenCalledTimes(1);
+    });
+
+    it('drops obsolete value after hard TTL is out', async () => {
+      const obj = new Class();
+
+      expect(await obj.getReleases()).toBe('111');
+      expect(getValue).toHaveBeenCalledTimes(1);
+      expect(setCache).toHaveBeenLastCalledWith(
+        'namespace',
+        'cache-decorator:key',
+        { cachedAt: expect.any(String), value: '111' },
+        2
+      );
+
+      jest.advanceTimersByTime(2 * 60 * 1000 - 1);
+      getValue.mockRejectedValueOnce(new Error('test'));
+      expect(await obj.getReleases()).toBe('111');
+
+      jest.advanceTimersByTime(100);
+      getValue.mockRejectedValueOnce(new Error('test'));
+      await expect(obj.getReleases()).rejects.toThrow('test');
+    });
   });
 });

lib/util/cache/package/decorator.ts

@@ -1,5 +1,9 @@
 import is from '@sindresorhus/is';
+import { DateTime } from 'luxon';
+import { GlobalConfig } from '../../../config/global';
+import { logger } from '../../../logger';
 import { Decorator, decorate } from '../../decorator';
+import type { DecoratorCachedRecord } from './types';
 import * as packageCache from '.';
 
 type HashFunction<T extends any[] = any[]> = (...args: T) => string;
@@ -42,7 +46,7 @@ export function cache<T>({
   cacheable = () => true,
   ttlMinutes = 30,
 }: CacheParameters): Decorator<T> {
-  return decorate(async ({ args, instance, callback }) => {
+  return decorate(async ({ args, instance, callback, methodName }) => {
     if (!cacheable.apply(instance, args)) {
       return callback();
     }
@@ -66,21 +70,59 @@ export function cache<T>({
       return callback();
     }
 
-    const cachedResult = await packageCache.get<unknown>(
+    finalKey = `cache-decorator:${finalKey}`;
+    const oldRecord = await packageCache.get<DecoratorCachedRecord>(
       finalNamespace,
       finalKey
     );
 
-    if (cachedResult !== undefined) {
-      return cachedResult;
+    const softTTL = ttlMinutes;
+    const cacheHardTtlMinutes = GlobalConfig.get().cacheHardTtlMinutes ?? 0;
+    let hardTTL =
+      methodName === 'getReleases' || methodName === 'getDigest'
+        ? cacheHardTtlMinutes
+        : 0;
+    hardTTL = Math.max(hardTTL, softTTL);
+
+    let oldData: unknown;
+    if (oldRecord) {
+      const now = DateTime.local();
+      const cachedAt = DateTime.fromISO(oldRecord.cachedAt);
+
+      const softDeadline = cachedAt.plus({ minutes: softTTL });
+      if (now < softDeadline) {
+        return oldRecord.value;
+      }
+
+      const hardDeadline = cachedAt.plus({ minutes: hardTTL });
+      if (now < hardDeadline) {
+        oldData = oldRecord.value;
+      }
     }
 
-    const result = await callback();
+    let newData: unknown;
+    if (oldData) {
+      try {
+        newData = (await callback()) as T | undefined;
+      } catch (err) {
+        logger.debug(
+          { err },
+          'Package cache decorator: callback error, returning old data'
+        );
+        return oldData;
+      }
+    } else {
+      newData = (await callback()) as T | undefined;
+    }
 
-    // only cache if we got a valid result
-    if (result !== undefined) {
-      await packageCache.set(finalNamespace, finalKey, result, ttlMinutes);
+    if (!is.undefined(newData)) {
+      const newRecord: DecoratorCachedRecord = {
+        cachedAt: DateTime.local().toISO(),
+        value: newData,
+      };
+      await packageCache.set(finalNamespace, finalKey, newRecord, hardTTL);
     }
-    return result;
+
+    return newData;
   });
 }

lib/util/cache/package/types.ts

@@ -10,3 +10,8 @@ export interface PackageCache {
 
   cleanup?(): Promise<void>;
 }
+
+export interface DecoratorCachedRecord {
+  value: unknown;
+  cachedAt: string;
+}

lib/util/decorator/index.ts

@@ -23,6 +23,11 @@ export interface DecoratorParameters<T, U extends any[] = any[]> {
    * Current call context.
    */
   instance: T;
+
+  /**
+   * The decorated method name.
+   */
+  methodName?: string;
 }
 
 /**
@@ -48,6 +53,7 @@ export function decorate<T>(fn: Handler<T>): Decorator<T> {
           args,
           instance: this,
           callback: () => value?.apply(this, args),
+          methodName: value?.name,
         });
       },
     });