feat(manager/bitbucket-pipelines): support docker image object (#21102)

lib/modules/datasource/types.ts

@@ -103,7 +103,4 @@ export interface DatasourceApi extends ModuleApi {
    * false: caching is not performed, or performed within the datasource implementation
    */
   caching?: boolean | undefined;
-
-  /** optional URLs to add to docs as references */
-  urls?: string[];
 }

lib/modules/manager/bitbucket-pipelines/__fixtures__/bitbucket-pipelines.yaml

@@ -1,11 +1,55 @@
 image: node:10.15.1
 
+definitions:
+  steps:
+    - step: &build-test
+        name: Build and test
+        image:
+          # comment
+          name: node:18.15.0
+        script:
+          - mvn package
+        artifacts:
+          - target/**
+
+    - step: &build-test1
+        image:
+          username: xxxx
+          name: node:18.15.1
+
+    - step: &build-test2
+        image:
+          username: xxx
+          password: xxx
+
+          name: node:18.15.2
+
+    - step:
+        image:
+        test:
+          name: malformed
+
+    - step:
+        image:
+          username: xxx
+        test:
+          name: malformed
+
+    - step:
+        image:
+          username: xxx
+          password: xxx
+        test:
+          name: malformed
+
+
 pipelines:
   default:
     - step:
         name: Build and Test
         image: node:10.15.2
         script:
+          - step: *build-test
           - pipe: docker://jfrogecosystem/jfrog-setup-cli:2.0.2
           - npm install
           - npm test

lib/modules/manager/bitbucket-pipelines/extract.spec.ts

@@ -4,49 +4,77 @@ import { extractPackageFile } from '.';
 describe('modules/manager/bitbucket-pipelines/extract', () => {
   describe('extractPackageFile()', () => {
     it('returns null for empty', () => {
-      expect(extractPackageFile('nothing here')).toBeNull();
+      expect(
+        extractPackageFile('nothing here', 'bitbucket-pipelines.yaml')
+      ).toBeNull();
+    });
+
+    it('returns null for malformed', () => {
+      expect(
+        extractPackageFile(
+          'image:\n  username: ccc',
+          'bitbucket-pipelines.yaml'
+        )
+      ).toBeNull();
     });
 
     it('extracts dependencies', () => {
-      const res = extractPackageFile(Fixtures.get('bitbucket-pipelines.yaml'));
-      expect(res?.deps).toMatchInlineSnapshot(`
-        [
+      const res = extractPackageFile(
+        Fixtures.get('bitbucket-pipelines.yaml'),
+        'bitbucket-pipelines.yaml'
+      );
+      expect(res).toMatchObject({
+        deps: [
+          {
+            currentDigest: undefined,
+            currentValue: '10.15.1',
+            datasource: 'docker',
+            depName: 'node',
+            depType: 'docker',
+          },
+          {
+            currentDigest: undefined,
+            currentValue: '18.15.0',
+            datasource: 'docker',
+            depName: 'node',
+            depType: 'docker',
+          },
+          {
+            currentDigest: undefined,
+            currentValue: '18.15.1',
+            datasource: 'docker',
+            depName: 'node',
+            depType: 'docker',
+          },
           {
-            "autoReplaceStringTemplate": "{{depName}}{{#if newValue}}:{{newValue}}{{/if}}{{#if newDigest}}@{{newDigest}}{{/if}}",
-            "currentDigest": undefined,
-            "currentValue": "10.15.1",
-            "datasource": "docker",
-            "depName": "node",
-            "depType": "docker",
-            "replaceString": "node:10.15.1",
+            currentDigest: undefined,
+            currentValue: '18.15.2',
+            datasource: 'docker',
+            depName: 'node',
+            depType: 'docker',
           },
           {
-            "autoReplaceStringTemplate": "{{depName}}{{#if newValue}}:{{newValue}}{{/if}}{{#if newDigest}}@{{newDigest}}{{/if}}",
-            "currentDigest": undefined,
-            "currentValue": "10.15.2",
-            "datasource": "docker",
-            "depName": "node",
-            "depType": "docker",
-            "replaceString": "node:10.15.2",
+            currentDigest: undefined,
+            currentValue: '10.15.2',
+            datasource: 'docker',
+            depName: 'node',
+            depType: 'docker',
           },
           {
-            "autoReplaceStringTemplate": "{{depName}}{{#if newValue}}:{{newValue}}{{/if}}{{#if newDigest}}@{{newDigest}}{{/if}}",
-            "currentDigest": undefined,
-            "currentValue": "2.0.2",
-            "datasource": "docker",
-            "depName": "jfrogecosystem/jfrog-setup-cli",
-            "depType": "docker",
-            "replaceString": "jfrogecosystem/jfrog-setup-cli:2.0.2",
+            currentDigest: undefined,
+            currentValue: '2.0.2',
+            datasource: 'docker',
+            depName: 'jfrogecosystem/jfrog-setup-cli',
+            depType: 'docker',
           },
           {
-            "currentValue": "0.2.1",
-            "datasource": "bitbucket-tags",
-            "depName": "atlassian/aws-s3-deploy",
-            "depType": "bitbucket-tags",
+            currentValue: '0.2.1',
+            datasource: 'bitbucket-tags',
+            depName: 'atlassian/aws-s3-deploy',
+            depType: 'bitbucket-tags',
           },
-        ]
-      `);
-      expect(res?.deps).toHaveLength(4);
+        ],
+      });
     });
   });
 });

lib/modules/manager/bitbucket-pipelines/extract.ts

@@ -1,18 +1,44 @@
 import { logger } from '../../../logger';
-import { newlineRegex, regEx } from '../../../util/regex';
-import { BitbucketTagsDatasource } from '../../datasource/bitbucket-tags';
-import { getDep } from '../dockerfile/extract';
+import { newlineRegex } from '../../../util/regex';
 import type { PackageDependency, PackageFileContent } from '../types';
+import {
+  addDepAsBitbucketTag,
+  addDepAsDockerImage,
+  addDepFromObject,
+  dockerImageObjectRegex,
+  dockerImageRegex,
+  pipeRegex,
+} from './util';
 
-const pipeRegex = regEx(`^\\s*-\\s?pipe:\\s*'?"?([^\\s'"]+)'?"?\\s*$`);
-const dockerImageRegex = regEx(`^\\s*-?\\s?image:\\s*'?"?([^\\s'"]+)'?"?\\s*$`);
-
-export function extractPackageFile(content: string): PackageFileContent | null {
+export function extractPackageFile(
+  content: string,
+  filename: string
+): PackageFileContent | null {
   const deps: PackageDependency[] = [];
 
   try {
-    const lines = content.split(newlineRegex);
-    for (const line of lines) {
+    const lines = content
+      .replaceAll(/^\s*\r?\n/gm, '') // replace empty lines
+      .replaceAll(/^\s*#.*\r?\n/gm, '') // replace comment lines
+      .split(newlineRegex);
+    const len = lines.length;
+    for (let lineIdx = 0; lineIdx < len; lineIdx++) {
+      const line = lines[lineIdx];
+
+      const dockerImageObjectGroups = dockerImageObjectRegex.exec(line)?.groups;
+      if (dockerImageObjectGroups) {
+        // image object
+        // https://support.atlassian.com/bitbucket-cloud/docs/docker-image-options/
+        lineIdx = addDepFromObject(
+          deps,
+          lines,
+          lineIdx,
+          len,
+          dockerImageObjectGroups.spaces
+        );
+        continue;
+      }
+
       const pipeMatch = pipeRegex.exec(line);
       if (pipeMatch) {
         const pipe = pipeMatch[1];
@@ -23,6 +49,7 @@ export function extractPackageFile(content: string): PackageFileContent | null {
         } else {
           addDepAsBitbucketTag(deps, pipe);
         }
+        continue;
       }
 
       const dockerImageMatch = dockerImageRegex.exec(line);
@@ -32,32 +59,13 @@ export function extractPackageFile(content: string): PackageFileContent | null {
       }
     }
   } catch (err) /* istanbul ignore next */ {
-    logger.warn({ err }, 'Error extracting Bitbucket Pipes dependencies');
+    logger.debug(
+      { err, filename },
+      'Error extracting Bitbucket Pipes dependencies'
+    );
   }
   if (!deps.length) {
     return null;
   }
   return { deps };
 }
-function addDepAsBitbucketTag(
-  deps: PackageDependency<Record<string, any>>[],
-  pipe: string
-): void {
-  const [depName, currentValue] = pipe.split(':');
-  const dep: PackageDependency = {
-    depName,
-    currentValue,
-    datasource: BitbucketTagsDatasource.id,
-  };
-  dep.depType = 'bitbucket-tags';
-  deps.push(dep);
-}
-
-function addDepAsDockerImage(
-  deps: PackageDependency<Record<string, any>>[],
-  currentDockerImage: string
-): void {
-  const dep = getDep(currentDockerImage);
-  dep.depType = 'docker';
-  deps.push(dep);
-}

lib/modules/manager/bitbucket-pipelines/index.ts

@@ -8,3 +8,7 @@ export const defaultConfig = {
 };
 
 export const supportedDatasources = [DockerDatasource.id];
+
+export const urls = [
+  'https://support.atlassian.com/bitbucket-cloud/docs/bitbucket-pipelines-configuration-reference/',
+];

lib/modules/manager/bitbucket-pipelines/util.ts

@@ -0,0 +1,66 @@
+import { regEx } from '../../../util/regex';
+import { BitbucketTagsDatasource } from '../../datasource/bitbucket-tags';
+import { getDep } from '../dockerfile/extract';
+import type { PackageDependency } from '../types';
+
+export const pipeRegex = regEx(`^\\s*-\\s?pipe:\\s*'?"?([^\\s'"]+)'?"?\\s*$`);
+export const dockerImageRegex = regEx(
+  `^\\s*-?\\s?image:\\s*'?"?([^\\s'"]+)'?"?\\s*$`
+);
+export const dockerImageObjectRegex = regEx('^(?<spaces>\\s*)image:\\s*$');
+
+export function addDepAsBitbucketTag(
+  deps: PackageDependency[],
+  pipe: string
+): void {
+  const [depName, currentValue] = pipe.split(':');
+  const dep: PackageDependency = {
+    depName,
+    currentValue,
+    datasource: BitbucketTagsDatasource.id,
+  };
+  dep.depType = 'bitbucket-tags';
+  deps.push(dep);
+}
+
+export function addDepAsDockerImage(
+  deps: PackageDependency[],
+  currentDockerImage: string
+): void {
+  const dep = getDep(currentDockerImage);
+  dep.depType = 'docker';
+  deps.push(dep);
+}
+
+export function addDepFromObject(
+  deps: PackageDependency[],
+  lines: string[],
+  start: number,
+  len: number,
+  spaces: string
+): number {
+  const nameRegex = regEx(
+    `^${spaces}\\s+name:\\s*['"]?(?<image>[^\\s'"]+)['"]?\\s*$`
+  );
+  const indentRegex = regEx(`^${spaces}\\s+`);
+
+  for (let idx = start + 1; idx < len; idx++) {
+    const line = lines[idx];
+
+    if (!indentRegex.test(line)) {
+      // malformed
+      return idx;
+    }
+
+    const groups = nameRegex.exec(line)?.groups;
+    if (groups) {
+      const dep = getDep(groups.image);
+      dep.depType = 'docker';
+      deps.push(dep);
+      return idx;
+    }
+  }
+
+  // malformed
+  return start;
+}

lib/types/base.ts

@@ -3,6 +3,9 @@ import type { PackageJson } from 'type-fest';
 export interface ModuleApi {
   displayName?: string;
   url?: string;
+
+  /** optional URLs to add to docs as references */
+  urls?: string[];
 }
 
 export type RenovatePackageJson = PackageJson & {

tools/docs/manager.ts

@@ -2,7 +2,12 @@ import type { RenovateConfig } from '../../lib/config/types';
 import { getManagers } from '../../lib/modules/manager';
 import { readFile, updateFile } from '../utils';
 import { OpenItems, generateFeatureAndBugMarkdown } from './github-query-items';
-import { getDisplayName, getNameWithUrl, replaceContent } from './utils';
+import {
+  formatUrls,
+  getDisplayName,
+  getNameWithUrl,
+  replaceContent,
+} from './utils';
 
 function getTitle(manager: string, displayName: string): string {
   if (manager === 'regex') {
@@ -26,7 +31,7 @@ export async function generateManagers(
     const language = definition.language ?? 'other';
     allLanguages[language] = allLanguages[language] || [];
     allLanguages[language].push(manager);
-    const { defaultConfig, supportedDatasources } = definition;
+    const { defaultConfig, supportedDatasources, urls } = definition;
     const { fileMatch } = defaultConfig as RenovateConfig;
     const displayName = getDisplayName(manager, definition);
     let md = `---
@@ -70,6 +75,9 @@ sidebar_label: ${displayName}
         .join(', ');
       md += `This manager supports extracting the following datasources: ${escapedDatasources}.\n\n`;
 
+      md += '## References';
+      md += formatUrls(urls).replace('**References**:', '');
+
       md += '## Default config\n\n';
       md += '```json\n';
       md += JSON.stringify(definition.defaultConfig, null, 2) + '\n';