From 52ad01d585b82702b514b1f491262f73abfeede1 Mon Sep 17 00:00:00 2001 From: Sam <687928+samrogerson@users.noreply.github.com> Date: Wed, 5 Apr 2023 13:30:35 +0100 Subject: [PATCH] feat(datasource/crate): add sparse registry support (#21187) Co-authored-by: Sam Co-authored-by: Rhys Arkins --- lib/modules/datasource/crate/index.spec.ts | 18 ++++++ lib/modules/datasource/crate/index.ts | 66 ++++++++++++++-------- lib/modules/datasource/crate/types.ts | 3 + 3 files changed, 65 insertions(+), 22 deletions(-) diff --git a/lib/modules/datasource/crate/index.spec.ts b/lib/modules/datasource/crate/index.spec.ts index 4e4e76afbb3d55..ae2c9920ae5264 100644 --- a/lib/modules/datasource/crate/index.spec.ts +++ b/lib/modules/datasource/crate/index.spec.ts @@ -354,6 +354,23 @@ describe('modules/datasource/crate/index', () => { expect(result).toBeNull(); expect(result2).toBeNull(); }); + + it('does not clone for sparse registries', async () => { + GlobalConfig.set({ ...adminConfig, allowCustomCrateRegistries: true }); + const { mockClone } = setupGitMocks(); + + const url = 'https://github.com/mcorbin/othertestregistry'; + const sparseUrl = `sparse+${url}`; + httpMock.scope(url).get('/my/pk/mypkg').reply(200, {}); + + const res = await getPkgReleases({ + datasource, + packageName: 'mypkg', + registryUrls: [sparseUrl], + }); + expect(mockClone).toHaveBeenCalledTimes(0); + expect(res).toBeNull(); + }); }); describe('fetchCrateRecordsPayload', () => { @@ -362,6 +379,7 @@ describe('modules/datasource/crate/index', () => { rawUrl: 'https://example.com', url: new URL('https://example.com'), flavor: 'cloudsmith', + isSparse: false, }; const crateDatasource = new CrateDatasource(); await expect( diff --git a/lib/modules/datasource/crate/index.ts b/lib/modules/datasource/crate/index.ts index 3c6e67a52067f2..28535884774f86 100644 --- a/lib/modules/datasource/crate/index.ts +++ b/lib/modules/datasource/crate/index.ts @@ -8,7 +8,7 @@ import { cache } from '../../../util/cache/package/decorator'; import { privateCacheDir, readCacheFile } from '../../../util/fs'; import { simpleGitConfig } from '../../../util/git/config'; import { newlineRegex, regEx } from '../../../util/regex'; -import { parseUrl } from '../../../util/url'; +import { joinUrlParts, parseUrl } from '../../../util/url'; import * as cargoVersioning from '../../versioning/cargo'; import { Datasource } from '../datasource'; import type { GetReleasesConfig, Release, ReleaseResult } from '../types'; @@ -165,10 +165,16 @@ export class CrateDatasource extends Datasource { return readCacheFile(path, 'utf8'); } - if (info.flavor === 'crates.io') { - const crateUrl = - CrateDatasource.CRATES_IO_BASE_URL + - CrateDatasource.getIndexSuffix(packageName.toLowerCase()).join('/'); + const baseUrl = + info.flavor === 'crates.io' + ? CrateDatasource.CRATES_IO_BASE_URL + : info.rawUrl; + + if (info.flavor === 'crates.io' || info.isSparse) { + const packageSuffix = CrateDatasource.getIndexSuffix( + packageName.toLowerCase() + ); + const crateUrl = joinUrlParts(baseUrl, ...packageSuffix); try { return (await this.http.get(crateUrl)).body; } catch (err) { @@ -215,6 +221,14 @@ export class CrateDatasource extends Datasource { return `crate-registry-${proto}-${host}-${hash}`; } + private static isSparseRegistry(url: string): boolean { + const parsed = parseUrl(url); + if (!parsed) { + return false; + } + return parsed.protocol.startsWith('sparse+'); + } + /** * Fetches information about a registry, by url. * If no url is given, assumes crates.io. @@ -230,9 +244,14 @@ export class CrateDatasource extends Datasource { return null; } - const url = parseUrl(registryUrl); + const isSparseRegistry = CrateDatasource.isSparseRegistry(registryUrl); + const registryFetchUrl = isSparseRegistry + ? registryUrl.replace(/^sparse\+/, '') + : registryUrl; + + const url = parseUrl(registryFetchUrl); if (!url) { - logger.debug(`Could not parse registry URL ${registryUrl}`); + logger.debug(`Could not parse registry URL ${registryFetchUrl}`); return null; } @@ -247,20 +266,23 @@ export class CrateDatasource extends Datasource { const registry: RegistryInfo = { flavor, - rawUrl: registryUrl, + rawUrl: registryFetchUrl, url, + isSparse: isSparseRegistry, }; - if (flavor !== 'crates.io') { - if (!GlobalConfig.get('allowCustomCrateRegistries')) { - logger.warn( - 'crate datasource: allowCustomCrateRegistries=true is required for registries other than crates.io, bailing out' - ); - return null; - } - - const cacheKey = `crate-datasource/registry-clone-path/${registryUrl}`; - const cacheKeyForError = `crate-datasource/registry-clone-path/${registryUrl}/error`; + if ( + registry.flavor !== 'crates.io' && + !GlobalConfig.get('allowCustomCrateRegistries') + ) { + logger.warn( + 'crate datasource: allowCustomCrateRegistries=true is required for registries other than crates.io, bailing out' + ); + return null; + } + if (registry.flavor !== 'crates.io' && !registry.isSparse) { + const cacheKey = `crate-datasource/registry-clone-path/${registryFetchUrl}`; + const cacheKeyForError = `crate-datasource/registry-clone-path/${registryFetchUrl}/error`; // We need to ensure we don't run `git clone` in parallel. Therefore we store // a promise of the running operation in the mem cache, which in the end resolves @@ -277,12 +299,12 @@ export class CrateDatasource extends Datasource { CrateDatasource.cacheDirFromUrl(url) ); logger.info( - { clonePath, registryUrl }, + { clonePath, registryFetchUrl }, `Cloning private cargo registry` ); const git = Git({ ...simpleGitConfig(), maxConcurrentProcesses: 1 }); - const clonePromise = git.clone(registryUrl, clonePath, { + const clonePromise = git.clone(registryFetchUrl, clonePath, { '--depth': 1, }); @@ -295,7 +317,7 @@ export class CrateDatasource extends Datasource { await clonePromise; } catch (err) { logger.warn( - { err, packageName, registryUrl }, + { err, packageName, registryFetchUrl }, 'failed cloning git registry' ); memCache.set(cacheKeyForError, err); @@ -307,7 +329,7 @@ export class CrateDatasource extends Datasource { if (!clonePath) { const err = memCache.get(cacheKeyForError); logger.warn( - { err, packageName, registryUrl }, + { err, packageName, registryFetchUrl }, 'Previous git clone failed, bailing out.' ); diff --git a/lib/modules/datasource/crate/types.ts b/lib/modules/datasource/crate/types.ts index b8a99d1440cf67..a7b59efbcb1145 100644 --- a/lib/modules/datasource/crate/types.ts +++ b/lib/modules/datasource/crate/types.ts @@ -17,6 +17,9 @@ export interface RegistryInfo { /** parsed URL of the registry */ url: URL; + /** whether the registry uses sparse indexing (rfc-2789) */ + isSparse: boolean; + /** path where the registry is cloned */ clonePath?: string; }