diff --git a/eng/common/pipelines/templates/jobs/generate-job-matrix.yml b/eng/common/pipelines/templates/jobs/generate-job-matrix.yml new file mode 100644 index 0000000000000..67ee92a93839e --- /dev/null +++ b/eng/common/pipelines/templates/jobs/generate-job-matrix.yml @@ -0,0 +1,140 @@ +parameters: +- name: AdditionalParameters + type: object +- name: DependsOn + type: object + default: null +- name: CloudConfig + type: object + default: {} +- name: MatrixConfigs + type: object + default: [] +- name: MatrixFilters + type: object + default: [] +- name: MatrixReplace + type: object + default: {} +- name: JobTemplatePath + type: string +# Set this to false to do a full checkout for private repositories with the azure pipelines service connection +- name: SparseCheckout + type: boolean + default: true +- name: SparseCheckoutPaths + type: object + default: [] +- name: Pool + type: string + default: $(LINUXPOOL) +- name: OsVmImage + type: string + default: $(LINUXVMIMAGE) +- name: Os + type: string + default: linux +# This parameter is only necessary if there are multiple invocations of this template within the SAME STAGE. +# When that occurs, provide a name other than the default value. +- name: GenerateJobName + type: string + default: 'generate_job_matrix' +- name: PreGenerationSteps + type: stepList + default: [] +# Mappings to OS name required at template compile time by 1es pipeline templates +- name: Pools + type: object + default: + - name: Linux + filter: .*Linux.*Pool$ + os: linux + - name: Windows + filter: .*Windows.*Pool$ + os: windows + - name: Mac + filter: .*MacPool$ + os: macOS + +jobs: +- job: ${{ parameters.GenerateJobName }} + variables: + - template: /eng/pipelines/templates/variables/image.yml + - name: skipComponentGovernanceDetection + value: true + - name: displayNameFilter + value: $[ coalesce(variables.jobMatrixFilter, '.*') ] + pool: + name: ${{ parameters.Pool }} + vmImage: ${{ parameters.OsVmImage }} + os: ${{ parameters.Os }} + ${{ if parameters.DependsOn }}: + dependsOn: ${{ parameters.DependsOn }} + steps: + # Skip sparse checkout for the `azure-sdk-for--pr` private mirrored repositories + # as we require the github service connection to be loaded. + - ${{ if and(parameters.SparseCheckout, not(contains(variables['Build.DefinitionName'], '-pr - '))) }}: + - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml + parameters: + ${{ if ne(length(parameters.SparseCheckoutPaths), 0) }}: + Paths: ${{ parameters.SparseCheckoutPaths }} + ${{ if and(eq(length(parameters.SparseCheckoutPaths), 0), ne(parameters.AdditionalParameters.ServiceDirectory, '')) }}: + Paths: + - "sdk/${{ parameters.AdditionalParameters.ServiceDirectory }}" + + - ${{ parameters.PreGenerationSteps }} + + - ${{ each config in parameters.MatrixConfigs }}: + - ${{ each pool in parameters.Pools }}: + - ${{ if eq(config.GenerateVMJobs, 'true') }}: + - task: Powershell@2 + inputs: + pwsh: true + filePath: eng/common/scripts/job-matrix/Create-JobMatrix.ps1 + arguments: > + -ConfigPath ${{ config.Path }} + -Selection ${{ config.Selection }} + -DisplayNameFilter '$(displayNameFilter)' + -Filters '${{ join(''',''', parameters.MatrixFilters) }}', 'container=^$', 'SupportedClouds=^$|${{ parameters.CloudConfig.Cloud }}', 'Pool=${{ pool.filter }}' + -Replace '${{ join(''',''', parameters.MatrixReplace) }}' + -NonSparseParameters '${{ join(''',''', config.NonSparseParameters) }}' + displayName: Create ${{ pool.name }} Matrix ${{ config.Name }} + name: vm_job_matrix_${{ config.Name }}_${{ pool.name }} + + - ${{ if eq(config.GenerateContainerJobs, 'true') }}: + - task: Powershell@2 + inputs: + pwsh: true + filePath: eng/common/scripts/job-matrix/Create-JobMatrix.ps1 + arguments: > + -ConfigPath ${{ config.Path }} + -Selection ${{ config.Selection }} + -DisplayNameFilter '$(displayNameFilter)' + -Filters '${{ join(''',''', parameters.MatrixFilters) }}', 'container=^$', 'SupportedClouds=^$|${{ parameters.CloudConfig.Cloud }}', 'Pool=${{ pool.filter }}' + -NonSparseParameters '${{ join(''',''', config.NonSparseParameters) }}' + displayName: Create ${{ pool.name }} Container Matrix ${{ config.Name }} + name: container_job_matrix_${{ config.Name }}_${{ pool.name }} + +- ${{ each config in parameters.MatrixConfigs }}: + - ${{ each pool in parameters.Pools }}: + - ${{ if eq(config.GenerateVMJobs, 'true') }}: + - template: ${{ parameters.JobTemplatePath }} + parameters: + UsePlatformContainer: false + OSName: ${{ pool.os }} + Matrix: dependencies.${{ parameters.GenerateJobName }}.outputs['vm_job_matrix_${{ config.Name }}_${{ pool.name }}.matrix'] + DependsOn: ${{ parameters.GenerateJobName }} + CloudConfig: ${{ parameters.CloudConfig }} + ${{ each param in parameters.AdditionalParameters }}: + ${{ param.key }}: ${{ param.value }} + + - ${{ if eq(config.GenerateContainerJobs, 'true') }}: + - template: ${{ parameters.JobTemplatePath }} + parameters: + UsePlatformContainer: true + OSName: ${{ pool.os }} + Matrix: dependencies.${{ parameters.GenerateJobName }}.outputs['vm_job_matrix_${{ config.Name }}_${{ pool.name }}.matrix'] + DependsOn: ${{ parameters.GenerateJobName }} + CloudConfig: ${{ parameters.CloudConfig }} + ${{ each param in parameters.AdditionalParameters }}: + ${{ param.key }}: ${{ param.value }} diff --git a/eng/common/pipelines/templates/steps/publish-1es-artifact.yml b/eng/common/pipelines/templates/steps/publish-1es-artifact.yml new file mode 100644 index 0000000000000..b06b2d0e444cc --- /dev/null +++ b/eng/common/pipelines/templates/steps/publish-1es-artifact.yml @@ -0,0 +1,30 @@ +# This step is used to prevent duplication of artifact publishes when there is an issue that would prevent the overall success of the job. +# Ensuring that we only publish when successful (and two a differently named artifact otherwise) will allow easy retry on a build pipeline +# without running into the "cannot override artifact" failure when we finally do get a passing run. + +# ArtifactName - The name of the artifact in the "successful" case. +# ArtifactPath - The path we will be publishing. +# CustomCondition - Used if there is additional logic necessary to prevent attempt of publish. + +parameters: + ArtifactName: '' + ArtifactPath: '' + CustomCondition: true + +steps: + - pwsh: | + Write-Host "##vso[task.setvariable variable=PublishArtifactName;]${{ parameters.ArtifactName }}" + condition: and(succeeded(), ${{ parameters.CustomCondition }}) + displayName: Set Artifact Name + + - pwsh: | + Write-Host "##vso[task.setvariable variable=PublishArtifactName;]${{ parameters.ArtifactName }}-FailedAttempt$(System.JobAttempt)" + condition: and(failed(), ${{ parameters.CustomCondition }}) + displayName: Set Failed Artifact Name + + - task: 1ES.PublishPipelineArtifact@1 + condition: and(succeeded(), ${{ parameters.CustomCondition }}) + displayName: 'Publish ${{ parameters.ArtifactName }} Artifacts' + inputs: + artifact: '$(PublishArtifactName)' + path: '${{ parameters.ArtifactPath }}'