spark-python-pipelines.yml

variables:
- group: Databricks-dev-environment
- group: Databricks-stg-environment
- group: Databricks-prod-environment
- name: jobPath
  value: 'spark_python_jobs/main.py'
- name: jobTestPath
  value: 'spark_python_jobs/tests/integration/main_test.py'
- name: jobName
  value: 'spark_app_python_job'
- name: jobTestName
  value: 'spark_app_python_job_integration_test'


trigger:
  batch: true
  branches:
    include:
    - '*'

  tags:
    include:
      - v*.*
      - prod

  paths:
    include:
      - spark_python_jobs/**

stages:


- stage: test_at_stg
  displayName: 'Test at Staging'
  condition: |
    or(
      startsWith(variables['Build.SourceBranch'], 'refs/heads/staging'),
      startsWith(variables['Build.SourceBranch'], 'refs/heads/production')
    )
  jobs:
  - template: ./template/test-lib-job.yml
    parameters:
      environmentName: 'STG'  
  - template: ./template/test-spark-python-job.yml
    parameters:
      environmentName: 'STG'
      databricksDomain: '$(databricksDomain_stg)'
      databricksClusterId: '$(databricksClusterId_stg)'
      databricksToken: '$(databricksToken_stg)'
      databricksJobName: '$(jobTestName)'
      databricksArtifactsPath: '$(jobTestPath)'
    


- stage: deploy_to_prod
  displayName: 'Deploy to Prod'
  condition: |
    and(
      startsWith(variables['Build.SourceBranch'], 'refs/heads/production'),
      succeeded('test_at_stg')
    )
  jobs:
  - template: ./template/deploy-spark-python-job.yml
    parameters:
      environmentName: 'PROD'
      databricksDomain: '$(databricksDomain_prod)'
      databricksClusterId: '$(databricksClusterId_prod)'
      databricksToken: '$(databricksToken_prod)'
      databricksJobName: '$(jobName)'
      databricksArtifactsPath: '$(jobPath)'