From dd24032b584a957e1137318344d0ca5362e417ab Mon Sep 17 00:00:00 2001 From: Sam Rogerson Date: Fri, 24 Mar 2023 09:36:24 +0000 Subject: [PATCH 1/7] base impl --- lib/modules/datasource/crate/index.ts | 45 ++++++++++++++++++--------- lib/modules/datasource/crate/types.ts | 3 ++ 2 files changed, 34 insertions(+), 14 deletions(-) diff --git a/lib/modules/datasource/crate/index.ts b/lib/modules/datasource/crate/index.ts index 3c6e67a52067f2..d3fe414ebb341d 100644 --- a/lib/modules/datasource/crate/index.ts +++ b/lib/modules/datasource/crate/index.ts @@ -165,10 +165,16 @@ export class CrateDatasource extends Datasource { return readCacheFile(path, 'utf8'); } - if (info.flavor === 'crates.io') { - const crateUrl = - CrateDatasource.CRATES_IO_BASE_URL + - CrateDatasource.getIndexSuffix(packageName.toLowerCase()).join('/'); + const baseUrl = + info.flavor === 'crates.io' + ? CrateDatasource.CRATES_IO_BASE_URL + : info.rawUrl; + + if (info.flavor === 'crates.io' || info.isSparse) { + const packageSuffix = CrateDatasource.getIndexSuffix( + packageName.toLowerCase() + ).join('/'); + const crateUrl = `${baseUrl}/${packageSuffix}`; try { return (await this.http.get(crateUrl)).body; } catch (err) { @@ -215,6 +221,11 @@ export class CrateDatasource extends Datasource { return `crate-registry-${proto}-${host}-${hash}`; } + private static isSparseRegistry(url: string): boolean { + const parsed = new URL(url); + return parsed.protocol.startsWith('sparse+'); + } + /** * Fetches information about a registry, by url. * If no url is given, assumes crates.io. @@ -230,9 +241,14 @@ export class CrateDatasource extends Datasource { return null; } - const url = parseUrl(registryUrl); + const isSparseRegistry = CrateDatasource.isSparseRegistry(registryUrl); + const registryFetchUrl = isSparseRegistry + ? registryUrl.replace(/^sparse\+/, '') + : registryUrl; + + const url = parseUrl(registryFetchUrl); if (!url) { - logger.debug(`Could not parse registry URL ${registryUrl}`); + logger.debug(`Could not parse registry URL ${registryFetchUrl}`); return null; } @@ -247,11 +263,12 @@ export class CrateDatasource extends Datasource { const registry: RegistryInfo = { flavor, - rawUrl: registryUrl, + rawUrl: registryFetchUrl, url, + isSparse: isSparseRegistry, }; - if (flavor !== 'crates.io') { + if (registry.flavor !== 'crates.io' && !registry.isSparse) { if (!GlobalConfig.get('allowCustomCrateRegistries')) { logger.warn( 'crate datasource: allowCustomCrateRegistries=true is required for registries other than crates.io, bailing out' @@ -259,8 +276,8 @@ export class CrateDatasource extends Datasource { return null; } - const cacheKey = `crate-datasource/registry-clone-path/${registryUrl}`; - const cacheKeyForError = `crate-datasource/registry-clone-path/${registryUrl}/error`; + const cacheKey = `crate-datasource/registry-clone-path/${registryFetchUrl}`; + const cacheKeyForError = `crate-datasource/registry-clone-path/${registryFetchUrl}/error`; // We need to ensure we don't run `git clone` in parallel. Therefore we store // a promise of the running operation in the mem cache, which in the end resolves @@ -277,12 +294,12 @@ export class CrateDatasource extends Datasource { CrateDatasource.cacheDirFromUrl(url) ); logger.info( - { clonePath, registryUrl }, + { clonePath, registryFetchUrl }, `Cloning private cargo registry` ); const git = Git({ ...simpleGitConfig(), maxConcurrentProcesses: 1 }); - const clonePromise = git.clone(registryUrl, clonePath, { + const clonePromise = git.clone(registryFetchUrl, clonePath, { '--depth': 1, }); @@ -295,7 +312,7 @@ export class CrateDatasource extends Datasource { await clonePromise; } catch (err) { logger.warn( - { err, packageName, registryUrl }, + { err, packageName, registryFetchUrl }, 'failed cloning git registry' ); memCache.set(cacheKeyForError, err); @@ -307,7 +324,7 @@ export class CrateDatasource extends Datasource { if (!clonePath) { const err = memCache.get(cacheKeyForError); logger.warn( - { err, packageName, registryUrl }, + { err, packageName, registryFetchUrl }, 'Previous git clone failed, bailing out.' ); diff --git a/lib/modules/datasource/crate/types.ts b/lib/modules/datasource/crate/types.ts index b8a99d1440cf67..a7b59efbcb1145 100644 --- a/lib/modules/datasource/crate/types.ts +++ b/lib/modules/datasource/crate/types.ts @@ -17,6 +17,9 @@ export interface RegistryInfo { /** parsed URL of the registry */ url: URL; + /** whether the registry uses sparse indexing (rfc-2789) */ + isSparse: boolean; + /** path where the registry is cloned */ clonePath?: string; } From e31a4beab555763427ec925ed2232a0895d44ee9 Mon Sep 17 00:00:00 2001 From: Sam Rogerson Date: Mon, 27 Mar 2023 09:56:58 +0000 Subject: [PATCH 2/7] clean up logic --- lib/modules/datasource/crate/index.ts | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/lib/modules/datasource/crate/index.ts b/lib/modules/datasource/crate/index.ts index d3fe414ebb341d..5253cc9512b1fd 100644 --- a/lib/modules/datasource/crate/index.ts +++ b/lib/modules/datasource/crate/index.ts @@ -268,14 +268,16 @@ export class CrateDatasource extends Datasource { isSparse: isSparseRegistry, }; + if ( + registry.flavor !== 'crates.io' && + !GlobalConfig.get('allowCustomCrateRegistries') + ) { + logger.warn( + 'crate datasource: allowCustomCrateRegistries=true is required for registries other than crates.io, bailing out' + ); + return null; + } if (registry.flavor !== 'crates.io' && !registry.isSparse) { - if (!GlobalConfig.get('allowCustomCrateRegistries')) { - logger.warn( - 'crate datasource: allowCustomCrateRegistries=true is required for registries other than crates.io, bailing out' - ); - return null; - } - const cacheKey = `crate-datasource/registry-clone-path/${registryFetchUrl}`; const cacheKeyForError = `crate-datasource/registry-clone-path/${registryFetchUrl}/error`; From 30d82fcde4e4a143027cdfbc773b16ee2b940c3b Mon Sep 17 00:00:00 2001 From: Sam Rogerson Date: Mon, 27 Mar 2023 10:01:56 +0000 Subject: [PATCH 3/7] unit test --- lib/modules/datasource/crate/index.spec.ts | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/lib/modules/datasource/crate/index.spec.ts b/lib/modules/datasource/crate/index.spec.ts index 4e4e76afbb3d55..d272351be8a3f4 100644 --- a/lib/modules/datasource/crate/index.spec.ts +++ b/lib/modules/datasource/crate/index.spec.ts @@ -354,6 +354,20 @@ describe('modules/datasource/crate/index', () => { expect(result).toBeNull(); expect(result2).toBeNull(); }); + + it('does not clone for sparse registries', async () => { + GlobalConfig.set({ ...adminConfig, allowCustomCrateRegistries: true }); + const { mockClone } = setupGitMocks(); + + const url = 'sparse+https://github.com/mcorbin/othertestregistry'; + const res = await getPkgReleases({ + datasource, + packageName: 'mypkg', + registryUrls: [url], + }); + expect(mockClone).toHaveBeenCalledTimes(0); + expect(res).toBeNull(); + }); }); describe('fetchCrateRecordsPayload', () => { From a72b85029799caa7887d545faec5ef8d75785153 Mon Sep 17 00:00:00 2001 From: Sam Rogerson Date: Wed, 29 Mar 2023 17:39:13 +0000 Subject: [PATCH 4/7] fix test registryinfo declaration --- lib/modules/datasource/crate/index.spec.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/modules/datasource/crate/index.spec.ts b/lib/modules/datasource/crate/index.spec.ts index d272351be8a3f4..8c94700dee7c75 100644 --- a/lib/modules/datasource/crate/index.spec.ts +++ b/lib/modules/datasource/crate/index.spec.ts @@ -376,6 +376,7 @@ describe('modules/datasource/crate/index', () => { rawUrl: 'https://example.com', url: new URL('https://example.com'), flavor: 'cloudsmith', + isSparse: false, }; const crateDatasource = new CrateDatasource(); await expect( From 36ce56196042bdaa5ca17a02e3c39214ca21acfa Mon Sep 17 00:00:00 2001 From: Sam Rogerson Date: Thu, 30 Mar 2023 10:14:55 +0000 Subject: [PATCH 5/7] less janky url join --- lib/modules/datasource/crate/index.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/modules/datasource/crate/index.ts b/lib/modules/datasource/crate/index.ts index 5253cc9512b1fd..b215f4d58829a6 100644 --- a/lib/modules/datasource/crate/index.ts +++ b/lib/modules/datasource/crate/index.ts @@ -8,7 +8,7 @@ import { cache } from '../../../util/cache/package/decorator'; import { privateCacheDir, readCacheFile } from '../../../util/fs'; import { simpleGitConfig } from '../../../util/git/config'; import { newlineRegex, regEx } from '../../../util/regex'; -import { parseUrl } from '../../../util/url'; +import { joinUrlParts, parseUrl } from '../../../util/url'; import * as cargoVersioning from '../../versioning/cargo'; import { Datasource } from '../datasource'; import type { GetReleasesConfig, Release, ReleaseResult } from '../types'; @@ -173,8 +173,8 @@ export class CrateDatasource extends Datasource { if (info.flavor === 'crates.io' || info.isSparse) { const packageSuffix = CrateDatasource.getIndexSuffix( packageName.toLowerCase() - ).join('/'); - const crateUrl = `${baseUrl}/${packageSuffix}`; + ); + const crateUrl = joinUrlParts(baseUrl, ...packageSuffix); try { return (await this.http.get(crateUrl)).body; } catch (err) { From 30b6ff4d58d8068f6f4e975e19bd7600989cf683 Mon Sep 17 00:00:00 2001 From: Sam Rogerson Date: Thu, 30 Mar 2023 12:38:52 +0000 Subject: [PATCH 6/7] learn how to mock in ts --- lib/modules/datasource/crate/index.spec.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/modules/datasource/crate/index.spec.ts b/lib/modules/datasource/crate/index.spec.ts index 8c94700dee7c75..ae2c9920ae5264 100644 --- a/lib/modules/datasource/crate/index.spec.ts +++ b/lib/modules/datasource/crate/index.spec.ts @@ -359,11 +359,14 @@ describe('modules/datasource/crate/index', () => { GlobalConfig.set({ ...adminConfig, allowCustomCrateRegistries: true }); const { mockClone } = setupGitMocks(); - const url = 'sparse+https://github.com/mcorbin/othertestregistry'; + const url = 'https://github.com/mcorbin/othertestregistry'; + const sparseUrl = `sparse+${url}`; + httpMock.scope(url).get('/my/pk/mypkg').reply(200, {}); + const res = await getPkgReleases({ datasource, packageName: 'mypkg', - registryUrls: [url], + registryUrls: [sparseUrl], }); expect(mockClone).toHaveBeenCalledTimes(0); expect(res).toBeNull(); From 698ff45b90f9d0cbf58c712a074f003090c81738 Mon Sep 17 00:00:00 2001 From: Sam Rogerson Date: Thu, 30 Mar 2023 16:40:45 +0000 Subject: [PATCH 7/7] prevent shortcircuit --- lib/modules/datasource/crate/index.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/modules/datasource/crate/index.ts b/lib/modules/datasource/crate/index.ts index b215f4d58829a6..28535884774f86 100644 --- a/lib/modules/datasource/crate/index.ts +++ b/lib/modules/datasource/crate/index.ts @@ -222,7 +222,10 @@ export class CrateDatasource extends Datasource { } private static isSparseRegistry(url: string): boolean { - const parsed = new URL(url); + const parsed = parseUrl(url); + if (!parsed) { + return false; + } return parsed.protocol.startsWith('sparse+'); }