From 83edceef307c77f946b7e3efddca6e4512f09dbc Mon Sep 17 00:00:00 2001 From: Tom Fay Date: Mon, 13 Feb 2023 14:28:22 +0000 Subject: [PATCH] feat(cargo): support sparse registry indices --- docs/usage/rust.md | 11 +- .../datasource/crate/__fixtures__/config.json | 4 + .../crate/__snapshots__/index.spec.ts.snap | 273 ++++++++++++++++++ lib/modules/datasource/crate/index.spec.ts | 58 +++- lib/modules/datasource/crate/index.ts | 56 +++- lib/modules/datasource/crate/types.ts | 10 + 6 files changed, 400 insertions(+), 12 deletions(-) create mode 100644 lib/modules/datasource/crate/__fixtures__/config.json diff --git a/docs/usage/rust.md b/docs/usage/rust.md index cf7494e0b412933..3ceaecb8f561284 100644 --- a/docs/usage/rust.md +++ b/docs/usage/rust.md @@ -19,11 +19,18 @@ Renovate supports upgrading dependencies in `Cargo.toml` files and their accompa Renovate updates Rust crates by default. -## Private crate registries and private Git dependencies +## Private Git crate registries and private Git dependencies If any dependencies are hosted in private Git repositories, [Git Authentication for cargo](https://doc.rust-lang.org/cargo/appendix/git-authentication.html) must be set up. -If any dependencies are hosted on private crate registries (i.e., not on `crates.io`), then credentials should be set up in such a way that the Git command-line is able to clone the registry index. +If any dependencies are hosted on private Git crate registries (i.e., not on `crates.io`), then credentials should be set up in such a way that the Git command-line is able to clone the registry index. Third-party crate registries usually provide instructions to achieve this. Both of these are currently only possible when running Renovate self-hosted. + +## Private sparse crate registries + +Renovate can update dependencies hosted on a private sparse crate registry (). +Since sparse registries are HTTP based, authentication for Renovate can be configured via additional hostRules. + +Renovate can only update Cargo lockfiles for projects using dependencies from sparse registries if the local Rust toolchain has the sparse-registry feature enabled (i.e beta/nightly/or 1.68 or later - 1.68 is due for release on March 9th 2023). diff --git a/lib/modules/datasource/crate/__fixtures__/config.json b/lib/modules/datasource/crate/__fixtures__/config.json new file mode 100644 index 000000000000000..b453be65ff94b3a --- /dev/null +++ b/lib/modules/datasource/crate/__fixtures__/config.json @@ -0,0 +1,4 @@ +{ + "dl": "https://crates.io/api/v1/crates", + "api": "https://crates.io" +} diff --git a/lib/modules/datasource/crate/__snapshots__/index.spec.ts.snap b/lib/modules/datasource/crate/__snapshots__/index.spec.ts.snap index 0470b485064e0e4..990dd6dd7a3e18b 100644 --- a/lib/modules/datasource/crate/__snapshots__/index.spec.ts.snap +++ b/lib/modules/datasource/crate/__snapshots__/index.spec.ts.snap @@ -30,6 +30,279 @@ exports[`modules/datasource/crate/index getReleases clones other private registr } `; +exports[`modules/datasource/crate/index getReleases processes real data for sparse registry: amethyst 1`] = ` +{ + "dependencyUrl": "https://crates.io/api/v1/crates/amethyst", + "registryUrl": "sparse+https://index.crates.io", + "releases": [ + { + "version": "0.1.0", + }, + { + "version": "0.1.1", + }, + { + "version": "0.1.3", + }, + { + "version": "0.1.4", + }, + { + "version": "0.2.1", + }, + { + "version": "0.3.0", + }, + { + "version": "0.3.1", + }, + { + "version": "0.4.0", + }, + { + "version": "0.4.1", + }, + { + "version": "0.4.2", + }, + { + "version": "0.4.3", + }, + { + "version": "0.5.0", + }, + { + "version": "0.5.1", + }, + { + "version": "0.6.0", + }, + { + "version": "0.7.0", + }, + { + "version": "0.8.0", + }, + { + "version": "0.9.0", + }, + { + "version": "0.10.0", + }, + { + "isDeprecated": true, + "version": "0.10.1", + }, + ], +} +`; + +exports[`modules/datasource/crate/index getReleases processes real data for sparse registry: libc 1`] = ` +{ + "dependencyUrl": "https://crates.io/api/v1/crates/libc", + "registryUrl": "sparse+https://index.crates.io", + "releases": [ + { + "version": "0.1.0", + }, + { + "version": "0.1.1", + }, + { + "version": "0.1.2", + }, + { + "version": "0.1.3", + }, + { + "version": "0.1.4", + }, + { + "version": "0.1.5", + }, + { + "version": "0.1.6", + }, + { + "version": "0.1.7", + }, + { + "version": "0.1.8", + }, + { + "isDeprecated": true, + "version": "0.1.9", + }, + { + "version": "0.1.10", + }, + { + "isDeprecated": true, + "version": "0.1.11", + }, + { + "version": "0.1.12", + }, + { + "version": "0.2.0", + }, + { + "version": "0.2.1", + }, + { + "version": "0.2.2", + }, + { + "version": "0.2.3", + }, + { + "version": "0.2.4", + }, + { + "version": "0.2.5", + }, + { + "version": "0.2.6", + }, + { + "version": "0.2.7", + }, + { + "version": "0.2.8", + }, + { + "version": "0.2.9", + }, + { + "version": "0.2.10", + }, + { + "version": "0.2.11", + }, + { + "version": "0.2.12", + }, + { + "version": "0.2.13", + }, + { + "version": "0.2.14", + }, + { + "version": "0.2.15", + }, + { + "version": "0.2.16", + }, + { + "version": "0.2.17", + }, + { + "version": "0.2.18", + }, + { + "version": "0.2.19", + }, + { + "version": "0.2.20", + }, + { + "version": "0.2.21", + }, + { + "version": "0.2.22", + }, + { + "version": "0.2.23", + }, + { + "version": "0.2.24", + }, + { + "version": "0.2.25", + }, + { + "version": "0.2.26", + }, + { + "version": "0.2.27", + }, + { + "version": "0.2.28", + }, + { + "version": "0.2.29", + }, + { + "version": "0.2.30", + }, + { + "version": "0.2.31", + }, + { + "version": "0.2.32", + }, + { + "version": "0.2.33", + }, + { + "version": "0.2.34", + }, + { + "version": "0.2.35", + }, + { + "version": "0.2.36", + }, + { + "version": "0.2.37", + }, + { + "version": "0.2.38", + }, + { + "version": "0.2.39", + }, + { + "version": "0.2.40", + }, + { + "version": "0.2.41", + }, + { + "version": "0.2.42", + }, + { + "version": "0.2.43", + }, + { + "version": "0.2.44", + }, + { + "version": "0.2.45", + }, + { + "version": "0.2.46", + }, + { + "version": "0.2.47", + }, + { + "version": "0.2.48", + }, + { + "version": "0.2.49", + }, + { + "version": "0.2.50", + }, + { + "version": "0.2.51", + }, + ], +} +`; + exports[`modules/datasource/crate/index getReleases processes real data: amethyst 1`] = ` { "dependencyUrl": "https://crates.io/crates/amethyst", diff --git a/lib/modules/datasource/crate/index.spec.ts b/lib/modules/datasource/crate/index.spec.ts index c8d0637391734f5..fbb833a4c2624c9 100644 --- a/lib/modules/datasource/crate/index.spec.ts +++ b/lib/modules/datasource/crate/index.spec.ts @@ -21,6 +21,8 @@ const API_BASE_URL = CrateDatasource.CRATES_IO_API_BASE_URL; const baseUrl = 'https://raw.githubusercontent.com/rust-lang/crates.io-index/master/'; +const sparseBaseUrl = 'https://index.crates.io'; + const datasource = CrateDatasource.id; function setupGitMocks(delayMs?: number): { mockClone: jest.Mock } { @@ -68,6 +70,13 @@ function mockCratesApiCallFor(crateName: string, response?: httpMock.Body) { .reply(response ? 200 : 404, response); } +function mockSparseConfig() { + httpMock + .scope(sparseBaseUrl) + .get('/config.json') + .reply(200, Fixtures.get('config.json')); +} + describe('modules/datasource/crate/index', () => { describe('getIndexSuffix', () => { it('returns correct suffixes', () => { @@ -248,6 +257,40 @@ describe('modules/datasource/crate/index', () => { expect(res).toBeDefined(); }); + it('processes real data for sparse registry: libc', async () => { + mockSparseConfig(); + + httpMock + .scope(sparseBaseUrl) + .get('/li/bc/libc') + .reply(200, Fixtures.get('libc')); + const res = await getPkgReleases({ + datasource, + depName: 'libc', + registryUrls: ['sparse+https://index.crates.io'], + }); + expect(res).toMatchSnapshot(); + expect(res).not.toBeNull(); + expect(res).toBeDefined(); + }); + + it('processes real data for sparse registry: amethyst', async () => { + mockSparseConfig(); + + httpMock + .scope(sparseBaseUrl) + .get('/am/et/amethyst') + .reply(200, Fixtures.get('amethyst')); + const res = await getPkgReleases({ + datasource, + depName: 'amethyst', + registryUrls: ['sparse+https://index.crates.io'], + }); + expect(res).toMatchSnapshot(); + expect(res).not.toBeNull(); + expect(res).toBeDefined(); + }); + it('refuses to clone if allowCustomCrateRegistries is not true', async () => { const { mockClone } = setupGitMocks(); @@ -354,10 +397,23 @@ describe('modules/datasource/crate/index', () => { expect(result).toBeNull(); expect(result2).toBeNull(); }); + + it('does not clone for sparse registries', async () => { + const { mockClone } = setupGitMocks(); + mockSparseConfig(); + + const res = await getPkgReleases({ + datasource, + depName: 'mypkg', + registryUrls: ['sparse+https://index.crates.io'], + }); + expect(mockClone).toHaveBeenCalledTimes(0); + expect(res).toBeNull(); + }); }); describe('fetchCrateRecordsPayload', () => { - it('rejects if it has neither clonePath nor crates.io flavor', async () => { + it('rejects if it has neither clonePath nor crates.io/sparse flavor', async () => { const info: RegistryInfo = { rawUrl: 'https://example.com', url: new URL('https://example.com'), diff --git a/lib/modules/datasource/crate/index.ts b/lib/modules/datasource/crate/index.ts index 3c6e67a52067f2e..11d4117f75cfb90 100644 --- a/lib/modules/datasource/crate/index.ts +++ b/lib/modules/datasource/crate/index.ts @@ -7,6 +7,7 @@ import * as memCache from '../../../util/cache/memory'; import { cache } from '../../../util/cache/package/decorator'; import { privateCacheDir, readCacheFile } from '../../../util/fs'; import { simpleGitConfig } from '../../../util/git/config'; +import { Http } from '../../../util/http'; import { newlineRegex, regEx } from '../../../util/regex'; import { parseUrl } from '../../../util/url'; import * as cargoVersioning from '../../versioning/cargo'; @@ -15,6 +16,7 @@ import type { GetReleasesConfig, Release, ReleaseResult } from '../types'; import type { CrateMetadata, CrateRecord, + RegistryConfig, RegistryFlavor, RegistryInfo, } from './types'; @@ -24,6 +26,7 @@ export class CrateDatasource extends Datasource { constructor() { super(CrateDatasource.id); + this.http = new Http('crate'); } override defaultRegistryUrls = ['https://crates.io']; @@ -56,7 +59,7 @@ export class CrateDatasource extends Datasource { return null; } - const registryInfo = await CrateDatasource.fetchRegistryInfo({ + const registryInfo = await this.fetchRegistryInfo({ packageName, registryUrl, }); @@ -165,9 +168,16 @@ export class CrateDatasource extends Datasource { return readCacheFile(path, 'utf8'); } - if (info.flavor === 'crates.io') { + if (info.flavor === 'crates.io' || info.flavor === 'sparse') { + let baseUrl: string; + if (info.flavor === 'sparse') { + baseUrl = info.url.toString(); + } else { + baseUrl = CrateDatasource.CRATES_IO_BASE_URL; + } + const crateUrl = - CrateDatasource.CRATES_IO_BASE_URL + + baseUrl + CrateDatasource.getIndexSuffix(packageName.toLowerCase()).join('/'); try { return (await this.http.get(crateUrl)).body; @@ -196,6 +206,8 @@ export class CrateDatasource extends Datasource { const repo = tokens[3]; return `https://cloudsmith.io/~${org}/repos/${repo}/packages/detail/cargo/${packageName}`; } + case 'sparse': + return `${info.dl ?? info.rawUrl}/${packageName}`; default: return `${info.rawUrl}/${packageName}`; } @@ -221,7 +233,7 @@ export class CrateDatasource extends Datasource { * If an url is given, assumes it's a valid Git repository * url and clones it to cache. */ - private static async fetchRegistryInfo({ + private async fetchRegistryInfo({ packageName, registryUrl, }: GetReleasesConfig): Promise { @@ -230,14 +242,28 @@ export class CrateDatasource extends Datasource { return null; } - const url = parseUrl(registryUrl); + let sparse = false; + let url: URL | null; + let rawUrl: string; + // Sparse registries use the sparse+ protocol + if (registryUrl.startsWith('sparse+')) { + sparse = true; + rawUrl = registryUrl.substring(7); + url = parseUrl(rawUrl); + } else { + rawUrl = registryUrl; + url = parseUrl(registryUrl); + } + if (!url) { logger.debug(`Could not parse registry URL ${registryUrl}`); return null; } let flavor: RegistryFlavor; - if (url.hostname === 'crates.io') { + if (sparse === true) { + flavor = 'sparse'; + } else if (url.hostname === 'crates.io') { flavor = 'crates.io'; } else if (url.hostname === 'dl.cloudsmith.io') { flavor = 'cloudsmith'; @@ -247,14 +273,26 @@ export class CrateDatasource extends Datasource { const registry: RegistryInfo = { flavor, - rawUrl: registryUrl, + rawUrl, url, }; - if (flavor !== 'crates.io') { + if (flavor === 'sparse') { + // Extract the index's config.json file in order to determine the + // crate download location + // https://doc.rust-lang.org/cargo/reference/registries.html#index-format + const res = await this.http.getJson( + `${rawUrl}/config.json` + ); + if (res?.body.dl) { + registry.dl = res?.body.dl; + } + } + + if (flavor !== 'crates.io' && flavor !== 'sparse') { if (!GlobalConfig.get('allowCustomCrateRegistries')) { logger.warn( - 'crate datasource: allowCustomCrateRegistries=true is required for registries other than crates.io, bailing out' + 'crate datasource: allowCustomCrateRegistries=true is required for registries other than crates.io or sparse registries, bailing out' ); return null; } diff --git a/lib/modules/datasource/crate/types.ts b/lib/modules/datasource/crate/types.ts index b8a99d1440cf673..b3163569d49a9d0 100644 --- a/lib/modules/datasource/crate/types.ts +++ b/lib/modules/datasource/crate/types.ts @@ -5,6 +5,8 @@ export type RegistryFlavor = /** https://cloudsmith.io, needs git clone */ | 'cloudsmith' + /** A sparse registry accessed via HTTP, not git */ + | 'sparse' /** unknown, assuming private git repository */ | 'other'; @@ -17,6 +19,9 @@ export interface RegistryInfo { /** parsed URL of the registry */ url: URL; + /** download location of crates, as specified in the registry's config.json */ + dl?: string; + /** path where the registry is cloned */ clonePath?: string; } @@ -32,3 +37,8 @@ export interface CrateMetadata { homepage: string | null; repository: string | null; } + +export interface RegistryConfig { + dl: string; + api: string; +}