generate-job-matrix.yml•8.28 kB
parameters:
- name: AdditionalParameters
  type: object
- name: DependsOn
  type: object
  default: null
- name: CloudConfig
  type: object
  default: {}
- name: MatrixConfigs
  type: object
  default: []
- name: MatrixFilters
  type: object
  default: []
- name: MatrixReplace
  type: object
  default: {}
- name: JobTemplatePath
  type: string
# Set this to false to do a full checkout for private repositories with the azure pipelines service connection
- name: SparseCheckout
  type: boolean
  default: true
- name: SparseCheckoutPaths
  type: object
  default: []
- name: Pool
  type: string
  default: $(LINUXPOOL)
- name: OsVmImage
  type: string
  default: $(LINUXVMIMAGE)
- name: Os
  type: string
  default: linux
# This parameter is only necessary if there are multiple invocations of this template within the SAME STAGE.
# When that occurs, provide a name other than the default value.
- name: GenerateJobName
  type: string
  default: 'generate_job_matrix'
- name: PreGenerationSteps
  type: stepList
  default: []
- name: EnablePRGeneration
  type: boolean
  default: false
# name of the variable that will be added when creating batches for the PR Job Matrix
- name: PRMatrixSetting
  type: string
  default: 'ArtifactPackageNames'
# name of the key in PackageInfo that will be used to get the identifier when generating matrix batches
- name: PRMatrixKey
  type: string
  default: 'ArtifactName'
- name: PRJobBatchSize
  type: number
  default: 10
- name: PRMatrixIndirectFilters
  type: object
  default: []
- name: PRMatrixSparseIndirect
  type: boolean
  default: true
# Mappings to OS name required at template compile time by 1es pipeline templates
- name: Pools
  type: object
  default:
    - name: Linux
      filter: .*Linux.*Pool$
      os: linux
    - name: Windows
      filter: .*Windows.*Pool$
      os: windows
    - name: Mac
      filter: .*MacPool$
      os: macOS
jobs:
- job: ${{ parameters.GenerateJobName }}
  variables:
    - template: /eng/pipelines/templates/variables/image.yml
    - name: skipComponentGovernanceDetection
      value: true
    - name: displayNameFilter
      value: $[ coalesce(variables.jobMatrixFilter, '.*') ]
  pool:
    name: ${{ parameters.Pool }}
    image: ${{ parameters.OsVmImage }}
    os: ${{ parameters.Os }}
  ${{ if parameters.DependsOn }}:
    dependsOn: ${{ parameters.DependsOn }}
  steps:
    # Skip sparse checkout for the `azure-sdk-for-<lang>-pr` private mirrored repositories
    # as we require the github service connection to be loaded.
    - ${{ if and(parameters.SparseCheckout, not(contains(variables['Build.DefinitionName'], '-pr - '))) }}:
      - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml
        parameters:
          ${{ if ne(length(parameters.SparseCheckoutPaths), 0) }}:
            Paths: ${{ parameters.SparseCheckoutPaths }}
          ${{ if and(eq(length(parameters.SparseCheckoutPaths), 0), ne(parameters.AdditionalParameters.ServiceDirectory, '')) }}:
            Paths:
              - "sdk/${{ parameters.AdditionalParameters.ServiceDirectory }}"
    - ${{ parameters.PreGenerationSteps }}
    - ${{ if eq(parameters.EnablePRGeneration, false) }}:
      - ${{ each config in parameters.MatrixConfigs }}:
        - ${{ each pool in parameters.Pools }}:
          - ${{ if eq(config.GenerateVMJobs, 'true') }}:
            - task: Powershell@2
              inputs:
                pwsh: true
                filePath: eng/common/scripts/job-matrix/Create-JobMatrix.ps1
                arguments: >
                  -ConfigPath ${{ config.Path }}
                  -Selection ${{ config.Selection }}
                  -DisplayNameFilter '$(displayNameFilter)'
                  -Filters '${{ join(''',''', parameters.MatrixFilters) }}', 'container=^$', 'SupportedClouds=^$|${{ parameters.CloudConfig.Cloud }}', 'Pool=${{ pool.filter }}'
                  -Replace '${{ join(''',''', parameters.MatrixReplace) }}'
                  -NonSparseParameters '${{ join(''',''', config.NonSparseParameters) }}'
              displayName: Create ${{ pool.name }} Matrix ${{ config.Name }}
              name: vm_job_matrix_${{ config.Name }}_${{ pool.name }}
          - ${{ if eq(config.GenerateContainerJobs, 'true') }}:
            - task: Powershell@2
              inputs:
                pwsh: true
                filePath: eng/common/scripts/job-matrix/Create-JobMatrix.ps1
                arguments: >
                  -ConfigPath ${{ config.Path }}
                  -Selection ${{ config.Selection }}
                  -DisplayNameFilter '$(displayNameFilter)'
                  -Filters '${{ join(''',''', parameters.MatrixFilters) }}', 'container=^$', 'SupportedClouds=^$|${{ parameters.CloudConfig.Cloud }}', 'Pool=${{ pool.filter }}'
                  -NonSparseParameters '${{ join(''',''', config.NonSparseParameters) }}'
              displayName: Create ${{ pool.name }} Container Matrix ${{ config.Name }}
              name: container_job_matrix_${{ config.Name }}_${{ pool.name }}
      # This else being set also currently assumes that the $(Build.ArtifactStagingDirectory)/PackageInfo folder is populated by PreGenerationSteps.
      # Not currently not hardcoded, so not doing the needful and populating this folder before we hit this step will result in generation errors.
    - ${{ else }}:
      - ${{ each pool in parameters.Pools }}:
        - pwsh: |
            '${{ convertToJson(parameters.MatrixConfigs) }}' | Set-Content matrix.json
            ./eng/common/scripts/job-matrix/Create-PrJobMatrix.ps1 `
              -PackagePropertiesFolder $(Build.ArtifactStagingDirectory)/PackageInfo `
              -PRMatrixFile matrix.json `
              -PRMatrixSetting ${{ parameters.PRMatrixSetting }} `
              -PRMatrixKey ${{ parameters.PRMatrixKey }} `
              -DisplayNameFilter '$(displayNameFilter)' `
              -Filters '${{ join(''',''', parameters.MatrixFilters) }}', 'container=^$', 'SupportedClouds=^$|${{ parameters.CloudConfig.Cloud }}', 'Pool=${{ pool.filter }}' `
              -IndirectFilters '${{ join(''',''', parameters.PRMatrixIndirectFilters) }}' `
              -Replace '${{ join(''',''', parameters.MatrixReplace) }}' `
              -PackagesPerPRJob ${{ parameters.PRJobBatchSize }} `
              -SparseIndirect $${{ parameters.PRMatrixSparseIndirect }}
          displayName: Create ${{ pool.name }} PR Matrix
          name: vm_job_matrix_pr_${{ pool.name }}
- ${{ if eq(parameters.EnablePRGeneration, false) }}:
  - ${{ each config in parameters.MatrixConfigs }}:
    - ${{ each pool in parameters.Pools }}:
      - ${{ if eq(config.GenerateVMJobs, 'true') }}:
        - template: ${{ parameters.JobTemplatePath }}
          parameters:
            UsePlatformContainer: false
            OSName: ${{ pool.os }}
            Matrix: dependencies.${{ parameters.GenerateJobName }}.outputs['vm_job_matrix_${{ config.Name }}_${{ pool.name }}.matrix']
            DependsOn: ${{ parameters.GenerateJobName }}
            CloudConfig: ${{ parameters.CloudConfig }}
            ${{ each param in parameters.AdditionalParameters }}:
              ${{ param.key }}: ${{ param.value }}
      - ${{ if eq(config.GenerateContainerJobs, 'true') }}:
        - template: ${{ parameters.JobTemplatePath }}
          parameters:
            UsePlatformContainer: true
            OSName: ${{ pool.os }}
            Matrix: dependencies.${{ parameters.GenerateJobName }}.outputs['vm_job_matrix_${{ config.Name }}_${{ pool.name }}.matrix']
            DependsOn: ${{ parameters.GenerateJobName }}
            CloudConfig: ${{ parameters.CloudConfig }}
            ${{ each param in parameters.AdditionalParameters }}:
              ${{ param.key }}: ${{ param.value }}
- ${{ else }}:
  - ${{ each pool in parameters.Pools }}:
    - template: ${{ parameters.JobTemplatePath }}
      parameters:
        UsePlatformContainer: false
        OSName: ${{ pool.os }}
        Matrix: dependencies.${{ parameters.GenerateJobName }}.outputs['vm_job_matrix_pr_${{ pool.name }}.matrix']
        DependsOn: ${{ parameters.GenerateJobName }}
        CloudConfig: ${{ parameters.CloudConfig }}
        ${{ each param in parameters.AdditionalParameters }}:
          ${{ param.key }}: ${{ param.value }}