From 34b5401812986226a236f67ecfd7b52ce63597d9 Mon Sep 17 00:00:00 2001 From: Michael Kriese Date: Wed, 29 Mar 2023 12:41:05 +0200 Subject: [PATCH] feat(manager/bitbucket-pipelines): support docker image object (#21102) --- lib/modules/datasource/types.ts | 3 - .../__fixtures__/bitbucket-pipelines.yaml | 44 +++++++++ .../bitbucket-pipelines/extract.spec.ts | 92 ++++++++++++------- .../manager/bitbucket-pipelines/extract.ts | 72 ++++++++------- .../manager/bitbucket-pipelines/index.ts | 4 + .../manager/bitbucket-pipelines/util.ts | 66 +++++++++++++ lib/types/base.ts | 3 + tools/docs/manager.ts | 12 ++- 8 files changed, 227 insertions(+), 69 deletions(-) create mode 100644 lib/modules/manager/bitbucket-pipelines/util.ts diff --git a/lib/modules/datasource/types.ts b/lib/modules/datasource/types.ts index 095d9f2ef1fcf1..f8023d3d924996 100644 --- a/lib/modules/datasource/types.ts +++ b/lib/modules/datasource/types.ts @@ -103,7 +103,4 @@ export interface DatasourceApi extends ModuleApi { * false: caching is not performed, or performed within the datasource implementation */ caching?: boolean | undefined; - - /** optional URLs to add to docs as references */ - urls?: string[]; } diff --git a/lib/modules/manager/bitbucket-pipelines/__fixtures__/bitbucket-pipelines.yaml b/lib/modules/manager/bitbucket-pipelines/__fixtures__/bitbucket-pipelines.yaml index c56064f849f004..d50138be948103 100644 --- a/lib/modules/manager/bitbucket-pipelines/__fixtures__/bitbucket-pipelines.yaml +++ b/lib/modules/manager/bitbucket-pipelines/__fixtures__/bitbucket-pipelines.yaml @@ -1,11 +1,55 @@ image: node:10.15.1 +definitions: + steps: + - step: &build-test + name: Build and test + image: + # comment + name: node:18.15.0 + script: + - mvn package + artifacts: + - target/** + + - step: &build-test1 + image: + username: xxxx + name: node:18.15.1 + + - step: &build-test2 + image: + username: xxx + password: xxx + + name: node:18.15.2 + + - step: + image: + test: + name: malformed + + - step: + image: + username: xxx + test: + name: malformed + + - step: + image: + username: xxx + password: xxx + test: + name: malformed + + pipelines: default: - step: name: Build and Test image: node:10.15.2 script: + - step: *build-test - pipe: docker://jfrogecosystem/jfrog-setup-cli:2.0.2 - npm install - npm test diff --git a/lib/modules/manager/bitbucket-pipelines/extract.spec.ts b/lib/modules/manager/bitbucket-pipelines/extract.spec.ts index cec1fdd230ccf4..e2157795251bd9 100644 --- a/lib/modules/manager/bitbucket-pipelines/extract.spec.ts +++ b/lib/modules/manager/bitbucket-pipelines/extract.spec.ts @@ -4,49 +4,77 @@ import { extractPackageFile } from '.'; describe('modules/manager/bitbucket-pipelines/extract', () => { describe('extractPackageFile()', () => { it('returns null for empty', () => { - expect(extractPackageFile('nothing here')).toBeNull(); + expect( + extractPackageFile('nothing here', 'bitbucket-pipelines.yaml') + ).toBeNull(); + }); + + it('returns null for malformed', () => { + expect( + extractPackageFile( + 'image:\n username: ccc', + 'bitbucket-pipelines.yaml' + ) + ).toBeNull(); }); it('extracts dependencies', () => { - const res = extractPackageFile(Fixtures.get('bitbucket-pipelines.yaml')); - expect(res?.deps).toMatchInlineSnapshot(` - [ + const res = extractPackageFile( + Fixtures.get('bitbucket-pipelines.yaml'), + 'bitbucket-pipelines.yaml' + ); + expect(res).toMatchObject({ + deps: [ + { + currentDigest: undefined, + currentValue: '10.15.1', + datasource: 'docker', + depName: 'node', + depType: 'docker', + }, + { + currentDigest: undefined, + currentValue: '18.15.0', + datasource: 'docker', + depName: 'node', + depType: 'docker', + }, + { + currentDigest: undefined, + currentValue: '18.15.1', + datasource: 'docker', + depName: 'node', + depType: 'docker', + }, { - "autoReplaceStringTemplate": "{{depName}}{{#if newValue}}:{{newValue}}{{/if}}{{#if newDigest}}@{{newDigest}}{{/if}}", - "currentDigest": undefined, - "currentValue": "10.15.1", - "datasource": "docker", - "depName": "node", - "depType": "docker", - "replaceString": "node:10.15.1", + currentDigest: undefined, + currentValue: '18.15.2', + datasource: 'docker', + depName: 'node', + depType: 'docker', }, { - "autoReplaceStringTemplate": "{{depName}}{{#if newValue}}:{{newValue}}{{/if}}{{#if newDigest}}@{{newDigest}}{{/if}}", - "currentDigest": undefined, - "currentValue": "10.15.2", - "datasource": "docker", - "depName": "node", - "depType": "docker", - "replaceString": "node:10.15.2", + currentDigest: undefined, + currentValue: '10.15.2', + datasource: 'docker', + depName: 'node', + depType: 'docker', }, { - "autoReplaceStringTemplate": "{{depName}}{{#if newValue}}:{{newValue}}{{/if}}{{#if newDigest}}@{{newDigest}}{{/if}}", - "currentDigest": undefined, - "currentValue": "2.0.2", - "datasource": "docker", - "depName": "jfrogecosystem/jfrog-setup-cli", - "depType": "docker", - "replaceString": "jfrogecosystem/jfrog-setup-cli:2.0.2", + currentDigest: undefined, + currentValue: '2.0.2', + datasource: 'docker', + depName: 'jfrogecosystem/jfrog-setup-cli', + depType: 'docker', }, { - "currentValue": "0.2.1", - "datasource": "bitbucket-tags", - "depName": "atlassian/aws-s3-deploy", - "depType": "bitbucket-tags", + currentValue: '0.2.1', + datasource: 'bitbucket-tags', + depName: 'atlassian/aws-s3-deploy', + depType: 'bitbucket-tags', }, - ] - `); - expect(res?.deps).toHaveLength(4); + ], + }); }); }); }); diff --git a/lib/modules/manager/bitbucket-pipelines/extract.ts b/lib/modules/manager/bitbucket-pipelines/extract.ts index f033e89c2a2c6a..4348eee72a0a11 100644 --- a/lib/modules/manager/bitbucket-pipelines/extract.ts +++ b/lib/modules/manager/bitbucket-pipelines/extract.ts @@ -1,18 +1,44 @@ import { logger } from '../../../logger'; -import { newlineRegex, regEx } from '../../../util/regex'; -import { BitbucketTagsDatasource } from '../../datasource/bitbucket-tags'; -import { getDep } from '../dockerfile/extract'; +import { newlineRegex } from '../../../util/regex'; import type { PackageDependency, PackageFileContent } from '../types'; +import { + addDepAsBitbucketTag, + addDepAsDockerImage, + addDepFromObject, + dockerImageObjectRegex, + dockerImageRegex, + pipeRegex, +} from './util'; -const pipeRegex = regEx(`^\\s*-\\s?pipe:\\s*'?"?([^\\s'"]+)'?"?\\s*$`); -const dockerImageRegex = regEx(`^\\s*-?\\s?image:\\s*'?"?([^\\s'"]+)'?"?\\s*$`); - -export function extractPackageFile(content: string): PackageFileContent | null { +export function extractPackageFile( + content: string, + filename: string +): PackageFileContent | null { const deps: PackageDependency[] = []; try { - const lines = content.split(newlineRegex); - for (const line of lines) { + const lines = content + .replaceAll(/^\s*\r?\n/gm, '') // replace empty lines + .replaceAll(/^\s*#.*\r?\n/gm, '') // replace comment lines + .split(newlineRegex); + const len = lines.length; + for (let lineIdx = 0; lineIdx < len; lineIdx++) { + const line = lines[lineIdx]; + + const dockerImageObjectGroups = dockerImageObjectRegex.exec(line)?.groups; + if (dockerImageObjectGroups) { + // image object + // https://support.atlassian.com/bitbucket-cloud/docs/docker-image-options/ + lineIdx = addDepFromObject( + deps, + lines, + lineIdx, + len, + dockerImageObjectGroups.spaces + ); + continue; + } + const pipeMatch = pipeRegex.exec(line); if (pipeMatch) { const pipe = pipeMatch[1]; @@ -23,6 +49,7 @@ export function extractPackageFile(content: string): PackageFileContent | null { } else { addDepAsBitbucketTag(deps, pipe); } + continue; } const dockerImageMatch = dockerImageRegex.exec(line); @@ -32,32 +59,13 @@ export function extractPackageFile(content: string): PackageFileContent | null { } } } catch (err) /* istanbul ignore next */ { - logger.warn({ err }, 'Error extracting Bitbucket Pipes dependencies'); + logger.debug( + { err, filename }, + 'Error extracting Bitbucket Pipes dependencies' + ); } if (!deps.length) { return null; } return { deps }; } -function addDepAsBitbucketTag( - deps: PackageDependency>[], - pipe: string -): void { - const [depName, currentValue] = pipe.split(':'); - const dep: PackageDependency = { - depName, - currentValue, - datasource: BitbucketTagsDatasource.id, - }; - dep.depType = 'bitbucket-tags'; - deps.push(dep); -} - -function addDepAsDockerImage( - deps: PackageDependency>[], - currentDockerImage: string -): void { - const dep = getDep(currentDockerImage); - dep.depType = 'docker'; - deps.push(dep); -} diff --git a/lib/modules/manager/bitbucket-pipelines/index.ts b/lib/modules/manager/bitbucket-pipelines/index.ts index 930c4a868f90d6..86d39fd3e9f2e3 100644 --- a/lib/modules/manager/bitbucket-pipelines/index.ts +++ b/lib/modules/manager/bitbucket-pipelines/index.ts @@ -8,3 +8,7 @@ export const defaultConfig = { }; export const supportedDatasources = [DockerDatasource.id]; + +export const urls = [ + 'https://support.atlassian.com/bitbucket-cloud/docs/bitbucket-pipelines-configuration-reference/', +]; diff --git a/lib/modules/manager/bitbucket-pipelines/util.ts b/lib/modules/manager/bitbucket-pipelines/util.ts new file mode 100644 index 00000000000000..0d04cf9c2d04e4 --- /dev/null +++ b/lib/modules/manager/bitbucket-pipelines/util.ts @@ -0,0 +1,66 @@ +import { regEx } from '../../../util/regex'; +import { BitbucketTagsDatasource } from '../../datasource/bitbucket-tags'; +import { getDep } from '../dockerfile/extract'; +import type { PackageDependency } from '../types'; + +export const pipeRegex = regEx(`^\\s*-\\s?pipe:\\s*'?"?([^\\s'"]+)'?"?\\s*$`); +export const dockerImageRegex = regEx( + `^\\s*-?\\s?image:\\s*'?"?([^\\s'"]+)'?"?\\s*$` +); +export const dockerImageObjectRegex = regEx('^(?\\s*)image:\\s*$'); + +export function addDepAsBitbucketTag( + deps: PackageDependency[], + pipe: string +): void { + const [depName, currentValue] = pipe.split(':'); + const dep: PackageDependency = { + depName, + currentValue, + datasource: BitbucketTagsDatasource.id, + }; + dep.depType = 'bitbucket-tags'; + deps.push(dep); +} + +export function addDepAsDockerImage( + deps: PackageDependency[], + currentDockerImage: string +): void { + const dep = getDep(currentDockerImage); + dep.depType = 'docker'; + deps.push(dep); +} + +export function addDepFromObject( + deps: PackageDependency[], + lines: string[], + start: number, + len: number, + spaces: string +): number { + const nameRegex = regEx( + `^${spaces}\\s+name:\\s*['"]?(?[^\\s'"]+)['"]?\\s*$` + ); + const indentRegex = regEx(`^${spaces}\\s+`); + + for (let idx = start + 1; idx < len; idx++) { + const line = lines[idx]; + + if (!indentRegex.test(line)) { + // malformed + return idx; + } + + const groups = nameRegex.exec(line)?.groups; + if (groups) { + const dep = getDep(groups.image); + dep.depType = 'docker'; + deps.push(dep); + return idx; + } + } + + // malformed + return start; +} diff --git a/lib/types/base.ts b/lib/types/base.ts index daa6d1f03c6890..934faec60c626b 100644 --- a/lib/types/base.ts +++ b/lib/types/base.ts @@ -3,6 +3,9 @@ import type { PackageJson } from 'type-fest'; export interface ModuleApi { displayName?: string; url?: string; + + /** optional URLs to add to docs as references */ + urls?: string[]; } export type RenovatePackageJson = PackageJson & { diff --git a/tools/docs/manager.ts b/tools/docs/manager.ts index dddd4511f925ff..a3c0641526be6f 100644 --- a/tools/docs/manager.ts +++ b/tools/docs/manager.ts @@ -2,7 +2,12 @@ import type { RenovateConfig } from '../../lib/config/types'; import { getManagers } from '../../lib/modules/manager'; import { readFile, updateFile } from '../utils'; import { OpenItems, generateFeatureAndBugMarkdown } from './github-query-items'; -import { getDisplayName, getNameWithUrl, replaceContent } from './utils'; +import { + formatUrls, + getDisplayName, + getNameWithUrl, + replaceContent, +} from './utils'; function getTitle(manager: string, displayName: string): string { if (manager === 'regex') { @@ -26,7 +31,7 @@ export async function generateManagers( const language = definition.language ?? 'other'; allLanguages[language] = allLanguages[language] || []; allLanguages[language].push(manager); - const { defaultConfig, supportedDatasources } = definition; + const { defaultConfig, supportedDatasources, urls } = definition; const { fileMatch } = defaultConfig as RenovateConfig; const displayName = getDisplayName(manager, definition); let md = `--- @@ -70,6 +75,9 @@ sidebar_label: ${displayName} .join(', '); md += `This manager supports extracting the following datasources: ${escapedDatasources}.\n\n`; + md += '## References'; + md += formatUrls(urls).replace('**References**:', ''); + md += '## Default config\n\n'; md += '```json\n'; md += JSON.stringify(definition.defaultConfig, null, 2) + '\n';