diff --git a/lib/modules/datasource/github-releases/cache/cache-base.spec.ts b/lib/modules/datasource/github-releases/cache/cache-base.spec.ts deleted file mode 100644 index c39f3d92cfbc54..00000000000000 --- a/lib/modules/datasource/github-releases/cache/cache-base.spec.ts +++ /dev/null @@ -1,515 +0,0 @@ -import { DateTime } from 'luxon'; -import { mocked } from '../../../../../test/util'; -import * as _packageCache from '../../../../util/cache/package'; -import type { - GithubCachedItem, - GithubGraphqlRepoResponse, -} from '../../../../util/github/types'; -import { - GithubGraphqlResponse, - GithubHttp, -} from '../../../../util/http/github'; -import { AbstractGithubDatasourceCache } from './cache-base'; - -jest.mock('../../../../util/cache/package'); -const packageCache = mocked(_packageCache); - -interface FetchedItem { - name: string; - createdAt: string; - foo: string; -} - -interface StoredItem extends GithubCachedItem { - bar: string; -} - -type GraphqlDataResponse = { - statusCode: 200; - headers: Record; - body: GithubGraphqlResponse>; -}; - -type GraphqlResponse = GraphqlDataResponse | Error; - -class TestCache extends AbstractGithubDatasourceCache { - cacheNs = 'test-cache'; - graphqlQuery = `query { ... }`; - - coerceFetched({ - name: version, - createdAt: releaseTimestamp, - foo: bar, - }: FetchedItem): StoredItem | null { - return version === 'invalid' ? null : { version, releaseTimestamp, bar }; - } - - isEquivalent({ bar: x }: StoredItem, { bar: y }: StoredItem): boolean { - return x === y; - } -} - -function resp(items: FetchedItem[], hasNextPage = false): GraphqlDataResponse { - return { - statusCode: 200, - headers: {}, - body: { - data: { - repository: { - payload: { - nodes: items, - pageInfo: { - hasNextPage, - endCursor: 'abc', - }, - }, - }, - }, - }, - }; -} - -const sortItems = (items: StoredItem[]) => - items.sort(({ releaseTimestamp: x }, { releaseTimestamp: y }) => - x.localeCompare(y) - ); - -describe('modules/datasource/github-releases/cache/cache-base', () => { - const http = new GithubHttp(); - const httpPostJson = jest.spyOn(GithubHttp.prototype, 'postJson'); - - const now = DateTime.local(2022, 6, 15, 18, 30, 30); - const t1 = now.minus({ days: 3 }).toISO(); - const t2 = now.minus({ days: 2 }).toISO(); - const t3 = now.minus({ days: 1 }).toISO(); - - let responses: GraphqlResponse[] = []; - - beforeEach(() => { - responses = []; - jest.resetAllMocks(); - jest.spyOn(DateTime, 'now').mockReturnValue(now); - httpPostJson.mockImplementation(() => { - const resp = responses.shift()!; - return resp instanceof Error - ? Promise.reject(resp) - : Promise.resolve(resp); - }); - }); - - it('performs pre-fetch', async () => { - responses = [ - resp([{ name: 'v3', createdAt: t3, foo: 'ccc' }], true), - resp([{ name: 'v2', createdAt: t2, foo: 'bbb' }], true), - resp([{ name: 'v1', createdAt: t1, foo: 'aaa' }]), - ]; - const cache = new TestCache(http, { resetDeltaMinutes: 0 }); - - const res = await cache.getItems({ packageName: 'foo/bar' }); - - expect(sortItems(res)).toMatchObject([ - { version: 'v1', bar: 'aaa' }, - { version: 'v2', bar: 'bbb' }, - { version: 'v3', bar: 'ccc' }, - ]); - expect(packageCache.set).toHaveBeenCalledWith( - 'test-cache', - 'https://api.github.com/:foo:bar', - { - createdAt: now.toISO(), - updatedAt: now.toISO(), - lastReleasedAt: t3, - items: { - v1: { bar: 'aaa', releaseTimestamp: t1, version: 'v1' }, - v2: { bar: 'bbb', releaseTimestamp: t2, version: 'v2' }, - v3: { bar: 'ccc', releaseTimestamp: t3, version: 'v3' }, - }, - }, - 7 * 24 * 60 - ); - }); - - it('filters out items being coerced to null', async () => { - responses = [ - resp([{ name: 'v3', createdAt: t3, foo: 'ccc' }], true), - resp([{ name: 'invalid', createdAt: t3, foo: 'xxx' }], true), - resp([{ name: 'v2', createdAt: t2, foo: 'bbb' }], true), - resp([{ name: 'v1', createdAt: t1, foo: 'aaa' }]), - ]; - const cache = new TestCache(http, { resetDeltaMinutes: 0 }); - - const res = await cache.getItems({ packageName: 'foo/bar' }); - - expect(sortItems(res)).toMatchObject([ - { version: 'v1' }, - { version: 'v2' }, - { version: 'v3' }, - ]); - }); - - it('updates items', async () => { - packageCache.get.mockResolvedValueOnce({ - items: { - v1: { version: 'v1', releaseTimestamp: t1, bar: 'aaa' }, - v2: { version: 'v2', releaseTimestamp: t2, bar: 'bbb' }, - v3: { version: 'v3', releaseTimestamp: t3, bar: 'ccc' }, - }, - createdAt: t3, - updatedAt: t3, - }); - - responses = [ - resp([{ name: 'v3', createdAt: t3, foo: 'xxx' }], true), - resp([{ name: 'v2', createdAt: t2, foo: 'bbb' }], true), - resp([{ name: 'v1', createdAt: t1, foo: 'aaa' }]), - ]; - const cache = new TestCache(http, { resetDeltaMinutes: 0 }); - - const res = await cache.getItems({ packageName: 'foo/bar' }); - - expect(sortItems(res)).toMatchObject([ - { version: 'v1', bar: 'aaa' }, - { version: 'v2', bar: 'bbb' }, - { version: 'v3', bar: 'xxx' }, - ]); - expect(packageCache.set).toHaveBeenCalledWith( - 'test-cache', - 'https://api.github.com/:foo:bar', - { - createdAt: t3, - updatedAt: now.toISO(), - lastReleasedAt: t3, - items: { - v1: { bar: 'aaa', releaseTimestamp: t1, version: 'v1' }, - v2: { bar: 'bbb', releaseTimestamp: t2, version: 'v2' }, - v3: { bar: 'xxx', releaseTimestamp: t3, version: 'v3' }, - }, - }, - 6 * 24 * 60 - ); - }); - - it('does not update non-fresh packages earlier than 120 minutes ago', async () => { - const releaseTimestamp = now.minus({ days: 7 }).toISO(); - const createdAt = now.minus({ minutes: 119 }).toISO(); - packageCache.get.mockResolvedValueOnce({ - items: { v1: { version: 'v1', releaseTimestamp, bar: 'aaa' } }, - createdAt, - updatedAt: createdAt, - }); - responses = [ - resp([ - { name: 'v1', createdAt: releaseTimestamp, foo: 'aaa' }, - { name: 'v2', createdAt: now.toISO(), foo: 'bbb' }, - ]), - ]; - const cache = new TestCache(http, { resetDeltaMinutes: 0 }); - - const res = await cache.getItems({ packageName: 'foo/bar' }); - - expect(sortItems(res)).toMatchObject([ - { version: 'v1', releaseTimestamp, bar: 'aaa' }, - ]); - expect(httpPostJson).not.toHaveBeenCalled(); - }); - - it('updates non-fresh packages after 120 minutes', async () => { - const releaseTimestamp = now.minus({ days: 7 }).toISO(); - const recentTimestamp = now.toISO(); - const createdAt = now.minus({ minutes: 120 }).toISO(); - packageCache.get.mockResolvedValueOnce({ - items: { v1: { version: 'v1', releaseTimestamp, bar: 'aaa' } }, - createdAt, - updatedAt: createdAt, - }); - responses = [ - resp([ - { name: 'v1', createdAt: releaseTimestamp, foo: 'aaa' }, - { name: 'v2', createdAt: recentTimestamp, foo: 'bbb' }, - ]), - ]; - const cache = new TestCache(http, { resetDeltaMinutes: 0 }); - - const res = await cache.getItems({ packageName: 'foo/bar' }); - - expect(sortItems(res)).toMatchObject([ - { version: 'v1', releaseTimestamp, bar: 'aaa' }, - { version: 'v2', releaseTimestamp: recentTimestamp, bar: 'bbb' }, - ]); - expect(packageCache.set).toHaveBeenCalledWith( - 'test-cache', - 'https://api.github.com/:foo:bar', - { - createdAt, - items: { - v1: { bar: 'aaa', releaseTimestamp, version: 'v1' }, - v2: { bar: 'bbb', releaseTimestamp: recentTimestamp, version: 'v2' }, - }, - lastReleasedAt: recentTimestamp, - updatedAt: recentTimestamp, - }, - 60 * 24 * 7 - 120 - ); - }); - - it('stops updating once stability period have passed', async () => { - packageCache.get.mockResolvedValueOnce({ - items: { - v1: { version: 'v1', releaseTimestamp: t1, bar: 'aaa' }, - v2: { version: 'v2', releaseTimestamp: t2, bar: 'bbb' }, - v3: { version: 'v3', releaseTimestamp: t3, bar: 'ccc' }, - }, - createdAt: t3, - updatedAt: t3, - }); - responses = [ - resp([{ name: 'v3', createdAt: t3, foo: 'zzz' }], true), - resp([{ name: 'v2', createdAt: t2, foo: 'yyy' }], true), - resp([{ name: 'v1', createdAt: t1, foo: 'xxx' }]), - ]; - const cache = new TestCache(http, { unstableDays: 1.5 }); - - const res = await cache.getItems({ packageName: 'foo/bar' }); - - expect(sortItems(res)).toMatchObject([ - { version: 'v1', bar: 'aaa' }, - { version: 'v2', bar: 'yyy' }, - { version: 'v3', bar: 'zzz' }, - ]); - }); - - it('removes deleted items from cache', async () => { - packageCache.get.mockResolvedValueOnce({ - items: { - v1: { version: 'v1', releaseTimestamp: t1, bar: 'aaa' }, - v2: { version: 'v2', releaseTimestamp: t2, bar: 'bbb' }, - v3: { version: 'v3', releaseTimestamp: t3, bar: 'ccc' }, - }, - createdAt: t3, - updatedAt: t3, - }); - responses = [ - resp([{ name: 'v3', createdAt: t3, foo: 'ccc' }], true), - resp([{ name: 'v1', createdAt: t1, foo: 'aaa' }]), - ]; - const cache = new TestCache(http, { resetDeltaMinutes: 0 }); - - const res = await cache.getItems({ packageName: 'foo/bar' }); - - expect(sortItems(res)).toMatchObject([ - { version: 'v1', bar: 'aaa' }, - { version: 'v3', bar: 'ccc' }, - ]); - }); - - it('throws for http errors', async () => { - packageCache.get.mockResolvedValueOnce({ - items: { - v1: { version: 'v1', releaseTimestamp: t1, bar: 'aaa' }, - v2: { version: 'v2', releaseTimestamp: t2, bar: 'bbb' }, - v3: { version: 'v3', releaseTimestamp: t3, bar: 'ccc' }, - }, - createdAt: t3, - updatedAt: t3, - }); - responses = [ - resp([{ name: 'v3', createdAt: t3, foo: 'zzz' }], true), - new Error('Unknown error'), - resp([{ name: 'v1', createdAt: t1, foo: 'xxx' }]), - ]; - const cache = new TestCache(http, { resetDeltaMinutes: 0 }); - - await expect(cache.getItems({ packageName: 'foo/bar' })).rejects.toThrow( - 'Unknown error' - ); - expect(packageCache.get).toHaveBeenCalled(); - expect(packageCache.set).not.toHaveBeenCalled(); - }); - - it('throws for graphql errors', async () => { - packageCache.get.mockResolvedValueOnce({ - items: {}, - createdAt: t3, - updatedAt: t3, - }); - responses = [ - { - statusCode: 200, - headers: {}, - body: { errors: [{} as never, { message: 'Ooops' }] }, - }, - ]; - const cache = new TestCache(http, { resetDeltaMinutes: 0 }); - - await expect(cache.getItems({ packageName: 'foo/bar' })).rejects.toThrow( - 'Ooops' - ); - expect(packageCache.get).toHaveBeenCalled(); - expect(packageCache.set).not.toHaveBeenCalled(); - }); - - it('throws for unknown graphql errors', async () => { - packageCache.get.mockResolvedValueOnce({ - items: {}, - createdAt: t3, - updatedAt: t3, - }); - responses = [ - { - statusCode: 200, - headers: {}, - body: { errors: [] }, - }, - ]; - const cache = new TestCache(http, { resetDeltaMinutes: 0 }); - - await expect(cache.getItems({ packageName: 'foo/bar' })).rejects.toThrow( - 'GitHub datasource cache: unknown GraphQL error' - ); - expect(packageCache.get).toHaveBeenCalled(); - expect(packageCache.set).not.toHaveBeenCalled(); - }); - - it('throws for empty payload', async () => { - packageCache.get.mockResolvedValueOnce({ - items: {}, - createdAt: t3, - updatedAt: t3, - }); - responses = [ - { - statusCode: 200, - headers: {}, - body: { data: { repository: { payload: null as never } } }, - }, - ]; - const cache = new TestCache(http, { resetDeltaMinutes: 0 }); - - await expect(cache.getItems({ packageName: 'foo/bar' })).rejects.toThrow( - 'GitHub datasource cache: failed to obtain payload data' - ); - expect(packageCache.get).toHaveBeenCalled(); - expect(packageCache.set).not.toHaveBeenCalled(); - }); - - it('shrinks for some of graphql errors', async () => { - packageCache.get.mockResolvedValueOnce({ - items: {}, - createdAt: t3, - updatedAt: t3, - }); - responses = [ - { - statusCode: 200, - headers: {}, - body: { - errors: [ - { message: 'Something went wrong while executing your query.' }, - ], - }, - }, - resp([{ name: 'v3', createdAt: t3, foo: 'ccc' }], true), - resp([{ name: 'v2', createdAt: t2, foo: 'bbb' }], true), - resp([{ name: 'v1', createdAt: t1, foo: 'aaa' }]), - ]; - const cache = new TestCache(http, { resetDeltaMinutes: 0 }); - - const res = await cache.getItems({ packageName: 'foo/bar' }); - - expect(sortItems(res)).toMatchObject([ - { version: 'v1', bar: 'aaa' }, - { version: 'v2', bar: 'bbb' }, - { version: 'v3', bar: 'ccc' }, - ]); - expect(packageCache.set).toHaveBeenCalled(); - }); - - it('finds latest release timestamp correctly', () => { - const cache = new TestCache(http); - const ts = cache.getLastReleaseTimestamp({ - v2: { bar: 'bbb', releaseTimestamp: t2, version: 'v2' }, - v3: { bar: 'ccc', releaseTimestamp: t3, version: 'v3' }, - v1: { bar: 'aaa', releaseTimestamp: t1, version: 'v1' }, - }); - expect(ts).toEqual(t3); - }); - - describe('Changelog-based cache busting', () => { - describe('newChangelogReleaseDetected', () => { - const cache = new TestCache(http, { resetDeltaMinutes: 0 }); - - it('returns false for undefined release argument', () => { - expect( - cache.newChangelogReleaseDetected(undefined, now, {}, {}) - ).toBeFalse(); - }); - - it('returns false if version is present in cache', () => { - expect( - cache.newChangelogReleaseDetected( - { date: now.minus({ minutes: 10 }).toISO(), version: '1.2.3' }, - now, - { minutes: 20 }, - { - '1.2.3': { - bar: '1', - version: '1.2.3', - releaseTimestamp: now.toISO(), - }, - } - ) - ).toBeFalse(); - }); - - it('returns false if changelog release is not fresh', () => { - expect( - cache.newChangelogReleaseDetected( - { date: now.minus({ minutes: 20 }).toISO(), version: '1.2.3' }, - now, - { minutes: 10 }, - {} - ) - ).toBeFalse(); - }); - - it('returns true for fresh changelog release', () => { - expect( - cache.newChangelogReleaseDetected( - { date: now.minus({ minutes: 10 }).toISO(), version: '1.2.3' }, - now, - { minutes: 20 }, - {} - ) - ).toBeTrue(); - }); - }); - - it('forces cache update', async () => { - const lastUpdateTime = now.minus({ minutes: 15 }).toISO(); - const githubTime = now.minus({ minutes: 10 }).toISO(); - const changelogTime = now.minus({ minutes: 5 }).toISO(); - packageCache.get.mockResolvedValueOnce({ - items: {}, - createdAt: lastUpdateTime, - updatedAt: lastUpdateTime, - }); - responses = [ - resp([{ name: '1.0.0', createdAt: githubTime, foo: 'aaa' }]), - ]; - const cache = new TestCache(http, { resetDeltaMinutes: 0 }); - - const res = await cache.getItems({ packageName: 'foo/bar' }, { - version: '1.0.0', - date: changelogTime, - } as never); - - expect(sortItems(res)).toEqual([ - { - bar: 'aaa', - releaseTimestamp: githubTime, - version: '1.0.0', - }, - ]); - }); - }); -}); diff --git a/lib/modules/datasource/github-releases/cache/cache-base.ts b/lib/modules/datasource/github-releases/cache/cache-base.ts deleted file mode 100644 index a105abd8a9975e..00000000000000 --- a/lib/modules/datasource/github-releases/cache/cache-base.ts +++ /dev/null @@ -1,479 +0,0 @@ -import is from '@sindresorhus/is'; -import { DateTime, DurationLikeObject } from 'luxon'; -import { logger } from '../../../../logger'; -import * as memCache from '../../../../util/cache/memory'; -import * as packageCache from '../../../../util/cache/package'; -import type { - CacheOptions, - ChangelogRelease, - GithubCachedItem, - GithubDatasourceCache, - GithubGraphqlRepoParams, - GithubGraphqlRepoResponse, -} from '../../../../util/github/types'; -import { getApiBaseUrl } from '../../../../util/github/url'; -import type { - GithubGraphqlResponse, - GithubHttp, - GithubHttpOptions, -} from '../../../../util/http/github'; -import type { GetReleasesConfig } from '../../types'; - -/** - * The options that are meant to be used in production. - */ -const cacheDefaults: Required = { - /** - * How many minutes to wait until next cache update - */ - updateAfterMinutes: 120, - - /** - * If package was released recently, we assume higher - * probability of having one more release soon. - * - * In this case, we use `updateAfterMinutesFresh` option. - */ - packageFreshDays: 7, - - /** - * If package was released recently, we assume higher - * probability of having one more release soon. - * - * In this case, this option will be used - * instead of `updateAfterMinutes`. - * - * Fresh period is configured via `freshDays` option. - */ - updateAfterMinutesFresh: 30, - - /** - * How many days to wait until full cache reset (for single package). - */ - resetAfterDays: 7, - - /** - * Delays cache reset by some random amount of minutes, - * in order to stabilize load during mass cache reset. - */ - resetDeltaMinutes: 3 * 60, - - /** - * How many days ago the package should be published to be considered as stable. - * Since this period is expired, it won't be refreshed via soft updates anymore. - */ - unstableDays: 30, - - /** - * How many items per page to obtain per page during initial fetch (i.e. pre-fetch) - */ - itemsPerPrefetchPage: 100, - - /** - * How many pages to fetch (at most) during the initial fetch (i.e. pre-fetch) - */ - maxPrefetchPages: 100, - - /** - * How many items per page to obtain per page during the soft update - */ - itemsPerUpdatePage: 100, - - /** - * How many pages to fetch (at most) during the soft update - */ - maxUpdatePages: 100, -}; - -/** - * Tells whether the time `duration` is expired starting - * from the `date` (ISO date format) at the moment of `now`. - */ -function isExpired( - now: DateTime, - date: string, - duration: DurationLikeObject -): boolean { - const then = DateTime.fromISO(date); - const expiry = then.plus(duration); - return now >= expiry; -} - -export abstract class AbstractGithubDatasourceCache< - CachedItem extends GithubCachedItem, - FetchedItem = unknown -> { - private updateDuration: DurationLikeObject; - private packageFreshDaysDuration: DurationLikeObject; - private updateDurationFresh: DurationLikeObject; - private resetDuration: DurationLikeObject; - private stabilityDuration: DurationLikeObject; - - private maxPrefetchPages: number; - private itemsPerPrefetchPage: number; - - private maxUpdatePages: number; - private itemsPerUpdatePage: number; - - private resetDeltaMinutes: number; - - constructor(private http: GithubHttp, opts: CacheOptions = {}) { - const { - updateAfterMinutes, - packageFreshDays, - updateAfterMinutesFresh, - resetAfterDays, - unstableDays, - maxPrefetchPages, - itemsPerPrefetchPage, - maxUpdatePages, - itemsPerUpdatePage, - resetDeltaMinutes, - } = { - ...cacheDefaults, - ...opts, - }; - - this.updateDuration = { minutes: updateAfterMinutes }; - this.packageFreshDaysDuration = { days: packageFreshDays }; - this.updateDurationFresh = { minutes: updateAfterMinutesFresh }; - this.resetDuration = { days: resetAfterDays }; - this.stabilityDuration = { days: unstableDays }; - - this.maxPrefetchPages = maxPrefetchPages; - this.itemsPerPrefetchPage = itemsPerPrefetchPage; - this.maxUpdatePages = maxUpdatePages; - this.itemsPerUpdatePage = itemsPerUpdatePage; - - this.resetDeltaMinutes = resetDeltaMinutes; - } - - /** - * The key at which data is stored in the package cache. - */ - abstract readonly cacheNs: string; - - /** - * The query string. - * For parameters, see `GithubQueryParams`. - */ - abstract readonly graphqlQuery: string; - - /** - * Transform `fetchedItem` for storing in the package cache. - * @param fetchedItem Node obtained from GraphQL response - */ - abstract coerceFetched(fetchedItem: FetchedItem): CachedItem | null; - - private async queryPayload( - baseUrl: string, - variables: GithubGraphqlRepoParams, - options: GithubHttpOptions - ): Promise< - GithubGraphqlRepoResponse['repository']['payload'] | Error - > { - try { - const graphqlRes = await this.http.postJson< - GithubGraphqlResponse> - >('/graphql', { - ...options, - baseUrl, - body: { query: this.graphqlQuery, variables }, - }); - const { body } = graphqlRes; - const { data, errors } = body; - - if (errors) { - let [errorMessage] = errors - .map(({ message }) => message) - .filter(is.string); - errorMessage ??= 'GitHub datasource cache: unknown GraphQL error'; - return new Error(errorMessage); - } - - if (!data?.repository?.payload) { - return new Error( - 'GitHub datasource cache: failed to obtain payload data' - ); - } - - return data.repository.payload; - } catch (err) { - return err; - } - } - - private getBaseUrl(registryUrl: string | undefined): string { - const baseUrl = getApiBaseUrl(registryUrl).replace(/\/v3\/$/, '/'); // Replace for GHE - return baseUrl; - } - - private getCacheKey( - registryUrl: string | undefined, - packageName: string - ): string { - const baseUrl = this.getBaseUrl(registryUrl); - const [owner, name] = packageName.split('/'); - const cacheKey = `${baseUrl}:${owner}:${name}`; - return cacheKey; - } - - /** - * Pre-fetch, update, or just return the package cache items. - */ - async getItemsImpl( - releasesConfig: GetReleasesConfig, - changelogRelease?: ChangelogRelease - ): Promise { - const { packageName, registryUrl } = releasesConfig; - - // The time meant to be used across the function - const now = DateTime.now(); - - // Initialize items and timestamps for the new cache - let cacheItems: Record = {}; - - // Add random minutes to the creation date in order to - // provide back-off time during mass cache invalidation. - const randomDelta = this.getRandomDeltaMinutes(); - let cacheCreatedAt = now.plus(randomDelta).toISO(); - - // We have to initialize `updatedAt` value as already expired, - // so that soft update mechanics is immediately starting. - let cacheUpdatedAt = now.minus(this.updateDuration).toISO(); - - const [owner, name] = packageName.split('/'); - if (owner && name) { - const baseUrl = this.getBaseUrl(registryUrl); - const cacheKey = this.getCacheKey(registryUrl, packageName); - const cache = await packageCache.get>( - this.cacheNs, - cacheKey - ); - - const cacheDoesExist = - cache && !isExpired(now, cache.createdAt, this.resetDuration); - let lastReleasedAt: string | null = null; - let updateDuration = this.updateDuration; - if (cacheDoesExist) { - // Keeping the the original `cache` value intact - // in order to be used in exception handler - cacheItems = { ...cache.items }; - cacheCreatedAt = cache.createdAt; - cacheUpdatedAt = cache.updatedAt; - lastReleasedAt = - cache.lastReleasedAt ?? this.getLastReleaseTimestamp(cacheItems); - - // Release is considered fresh, so we'll check it earlier - if ( - lastReleasedAt && - !isExpired(now, lastReleasedAt, this.packageFreshDaysDuration) - ) { - updateDuration = this.updateDurationFresh; - } - } - - if ( - isExpired(now, cacheUpdatedAt, updateDuration) || - this.newChangelogReleaseDetected( - changelogRelease, - now, - updateDuration, - cacheItems - ) - ) { - const variables: GithubGraphqlRepoParams = { - owner, - name, - cursor: null, - count: cacheDoesExist - ? this.itemsPerUpdatePage - : this.itemsPerPrefetchPage, - }; - - // Collect version values to determine deleted items - const checkedVersions = new Set(); - - // Page-by-page update loop - let pagesRemained = cacheDoesExist - ? this.maxUpdatePages - : this.maxPrefetchPages; - let stopIteration = false; - while (pagesRemained > 0 && !stopIteration) { - const queryResult = await this.queryPayload(baseUrl, variables, { - repository: packageName, - }); - if (queryResult instanceof Error) { - if ( - queryResult.message.startsWith( - 'Something went wrong while executing your query.' // #16343 - ) && - variables.count > 30 - ) { - logger.warn( - `GitHub datasource cache: shrinking GraphQL page size due to error` - ); - pagesRemained *= 2; - variables.count = Math.floor(variables.count / 2); - continue; - } - throw queryResult; - } - - pagesRemained -= 1; - - const { - nodes: fetchedItems, - pageInfo: { hasNextPage, endCursor }, - } = queryResult; - - if (hasNextPage) { - variables.cursor = endCursor; - } else { - stopIteration = true; - } - - for (const item of fetchedItems) { - const newStoredItem = this.coerceFetched(item); - if (newStoredItem) { - const { version, releaseTimestamp } = newStoredItem; - - // Stop earlier if the stored item have reached stability, - // which means `unstableDays` period have passed - const oldStoredItem = cacheItems[version]; - if ( - oldStoredItem && - isExpired( - now, - oldStoredItem.releaseTimestamp, - this.stabilityDuration - ) - ) { - stopIteration = true; - } - - cacheItems[version] = newStoredItem; - checkedVersions.add(version); - - lastReleasedAt ??= releaseTimestamp; - // It may be tempting to optimize the code and - // remove the check, as we're fetching fresh releases here. - // That's wrong, because some items are already cached, - // and they obviously aren't latest. - if ( - DateTime.fromISO(releaseTimestamp) > - DateTime.fromISO(lastReleasedAt) - ) { - lastReleasedAt = releaseTimestamp; - } - } - } - } - - // Detect removed items - for (const [version, item] of Object.entries(cacheItems)) { - if ( - !isExpired(now, item.releaseTimestamp, this.stabilityDuration) && - !checkedVersions.has(version) - ) { - delete cacheItems[version]; - } - } - - // Store cache - const expiry = DateTime.fromISO(cacheCreatedAt).plus( - this.resetDuration - ); - const { minutes: ttlMinutes } = expiry - .diff(now, ['minutes']) - .toObject(); - if (ttlMinutes && ttlMinutes > 0) { - const cacheValue: GithubDatasourceCache = { - items: cacheItems, - createdAt: cacheCreatedAt, - updatedAt: now.toISO(), - }; - - if (lastReleasedAt) { - cacheValue.lastReleasedAt = lastReleasedAt; - } - - await packageCache.set( - this.cacheNs, - cacheKey, - cacheValue, - ttlMinutes - ); - } - } - } - - const items = Object.values(cacheItems); - return items; - } - - getItems( - releasesConfig: GetReleasesConfig, - changelogRelease?: ChangelogRelease - ): Promise { - const { packageName, registryUrl } = releasesConfig; - const cacheKey = this.getCacheKey(registryUrl, packageName); - const promiseKey = `github-datasource-cache:${this.cacheNs}:${cacheKey}`; - const res = - memCache.get>(promiseKey) ?? - this.getItemsImpl(releasesConfig, changelogRelease); - memCache.set(promiseKey, res); - return res; - } - - getRandomDeltaMinutes(): number { - const rnd = Math.random(); - return Math.floor(rnd * this.resetDeltaMinutes); - } - - public getLastReleaseTimestamp( - items: Record - ): string | null { - let result: string | null = null; - let latest: DateTime | null = null; - - for (const { releaseTimestamp } of Object.values(items)) { - const timestamp = DateTime.fromISO(releaseTimestamp); - - result ??= releaseTimestamp; - latest ??= timestamp; - - if (timestamp > latest) { - result = releaseTimestamp; - latest = timestamp; - } - } - - return result; - } - - newChangelogReleaseDetected( - changelogRelease: ChangelogRelease | undefined, - now: DateTime, - updateDuration: DurationLikeObject, - cacheItems: Record - ): boolean { - if (!changelogRelease?.date) { - return false; - } - - const releaseTime = changelogRelease.date.toString(); - const isVersionPresentInCache = !!cacheItems[changelogRelease.version]; - const isChangelogReleaseFresh = !isExpired( - now, - releaseTime, - updateDuration - ); - - if (isVersionPresentInCache || !isChangelogReleaseFresh) { - return false; - } - - return true; - } -} diff --git a/lib/modules/datasource/github-releases/cache/index.spec.ts b/lib/modules/datasource/github-releases/cache/index.spec.ts deleted file mode 100644 index 2e5c70cf29c52b..00000000000000 --- a/lib/modules/datasource/github-releases/cache/index.spec.ts +++ /dev/null @@ -1,44 +0,0 @@ -import type { GithubGraphqlRelease } from '../../../../util/github/types'; -import { GithubHttp } from '../../../../util/http/github'; -import { CacheableGithubReleases } from '.'; - -describe('modules/datasource/github-releases/cache/index', () => { - const http = new GithubHttp(); - const cache = new CacheableGithubReleases(http, { resetDeltaMinutes: 0 }); - - const fetchedItem: GithubGraphqlRelease = { - version: '1.2.3', - releaseTimestamp: '2020-04-09T10:00:00.000Z', - isDraft: false, - isPrerelease: false, - url: 'https://example.com/', - id: 123, - name: 'Some name', - description: 'Some description', - }; - - describe('coerceFetched', () => { - it('transforms GraphQL item', () => { - expect(cache.coerceFetched(fetchedItem)).toEqual({ - description: 'Some description', - id: 123, - name: 'Some name', - releaseTimestamp: '2020-04-09T10:00:00.000Z', - url: 'https://example.com/', - version: '1.2.3', - }); - }); - - it('marks pre-release as unstable', () => { - expect( - cache.coerceFetched({ ...fetchedItem, isPrerelease: true }) - ).toMatchObject({ - isStable: false, - }); - }); - - it('filters out drafts', () => { - expect(cache.coerceFetched({ ...fetchedItem, isDraft: true })).toBeNull(); - }); - }); -}); diff --git a/lib/modules/datasource/github-releases/cache/index.ts b/lib/modules/datasource/github-releases/cache/index.ts deleted file mode 100644 index 1c63e040a9d760..00000000000000 --- a/lib/modules/datasource/github-releases/cache/index.ts +++ /dev/null @@ -1,78 +0,0 @@ -import type { - CacheOptions, - GithubCachedRelease, - GithubGraphqlRelease, -} from '../../../../util/github/types'; -import type { GithubHttp } from '../../../../util/http/github'; -import { AbstractGithubDatasourceCache } from './cache-base'; - -export const query = ` -query ($owner: String!, $name: String!, $cursor: String, $count: Int!) { - repository(owner: $owner, name: $name) { - payload: releases( - first: $count - after: $cursor - orderBy: {field: CREATED_AT, direction: DESC} - ) { - nodes { - version: tagName - releaseTimestamp: publishedAt - isDraft - isPrerelease - url - id: databaseId - name - description - } - pageInfo { - hasNextPage - endCursor - } - } - } -} -`; - -export class CacheableGithubReleases extends AbstractGithubDatasourceCache< - GithubCachedRelease, - GithubGraphqlRelease -> { - cacheNs = 'github-datasource-graphql-releases'; - graphqlQuery = query; - - constructor(http: GithubHttp, opts: CacheOptions = {}) { - super(http, opts); - } - - coerceFetched(item: GithubGraphqlRelease): GithubCachedRelease | null { - const { - version, - releaseTimestamp, - isDraft, - isPrerelease, - url, - id, - name, - description, - } = item; - - if (isDraft) { - return null; - } - - const result: GithubCachedRelease = { - version, - releaseTimestamp, - url, - id, - name, - description, - }; - - if (isPrerelease) { - result.isStable = false; - } - - return result; - } -} diff --git a/lib/modules/datasource/github-tags/cache.spec.ts b/lib/modules/datasource/github-tags/cache.spec.ts deleted file mode 100644 index 18f61aecdf667b..00000000000000 --- a/lib/modules/datasource/github-tags/cache.spec.ts +++ /dev/null @@ -1,54 +0,0 @@ -import type { GithubGraphqlTag } from '../../../util/github/types'; -import { GithubHttp } from '../../../util/http/github'; -import { CacheableGithubTags } from './cache'; - -describe('modules/datasource/github-tags/cache', () => { - const http = new GithubHttp(); - const cache = new CacheableGithubTags(http, { resetDeltaMinutes: 0 }); - - const fetchedItem: GithubGraphqlTag = { - version: '1.2.3', - target: { - type: 'Commit', - hash: 'abc', - releaseTimestamp: '2020-04-09T10:00:00.000Z', - }, - }; - - describe('coerceFetched', () => { - it('transforms GraphQL items', () => { - expect(cache.coerceFetched(fetchedItem)).toEqual({ - version: '1.2.3', - hash: 'abc', - releaseTimestamp: '2020-04-09T10:00:00.000Z', - }); - expect( - cache.coerceFetched({ - version: '1.2.3', - target: { - type: 'Tag', - target: { - hash: 'abc', - }, - tagger: { - releaseTimestamp: '2020-04-09T10:00:00.000Z', - }, - }, - }) - ).toEqual({ - version: '1.2.3', - hash: 'abc', - releaseTimestamp: '2020-04-09T10:00:00.000Z', - }); - }); - - it('returns null for tags we can not process', () => { - expect( - cache.coerceFetched({ - version: '1.2.3', - target: { type: 'Blob' } as never, - }) - ).toBeNull(); - }); - }); -}); diff --git a/lib/modules/datasource/github-tags/cache.ts b/lib/modules/datasource/github-tags/cache.ts deleted file mode 100644 index 6f7469e4c6a5dd..00000000000000 --- a/lib/modules/datasource/github-tags/cache.ts +++ /dev/null @@ -1,70 +0,0 @@ -import type { - CacheOptions, - GithubCachedTag, - GithubGraphqlTag, -} from '../../../util/github/types'; -import type { GithubHttp } from '../../../util/http/github'; -import { AbstractGithubDatasourceCache } from '../github-releases/cache/cache-base'; - -const query = ` -query ($owner: String!, $name: String!, $cursor: String, $count: Int!) { - repository(owner: $owner, name: $name) { - payload: refs( - first: $count - after: $cursor - orderBy: {field: TAG_COMMIT_DATE, direction: DESC} - refPrefix: "refs/tags/" - ) { - nodes { - version: name - target { - type: __typename - ... on Commit { - hash: oid - releaseTimestamp: committedDate - } - ... on Tag { - target { - ... on Commit { - hash: oid - } - } - tagger { - releaseTimestamp: date - } - } - } - } - pageInfo { - hasNextPage - endCursor - } - } - } -} -`; - -export class CacheableGithubTags extends AbstractGithubDatasourceCache< - GithubCachedTag, - GithubGraphqlTag -> { - readonly cacheNs = 'github-datasource-graphql-tags-v2'; - readonly graphqlQuery = query; - - constructor(http: GithubHttp, opts: CacheOptions = {}) { - super(http, opts); - } - - coerceFetched(item: GithubGraphqlTag): GithubCachedTag | null { - const { version, target } = item; - if (target.type === 'Commit') { - const { hash, releaseTimestamp } = target; - return { version, hash, releaseTimestamp }; - } else if (target.type === 'Tag') { - const { hash } = target.target; - const { releaseTimestamp } = target.tagger; - return { version, hash, releaseTimestamp }; - } - return null; - } -}