unittest.yml

on:
  pull_request:
    branches:
      - main
      - preview
  # Trigger workflow on GitHub merge queue events
  # See https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#merge_group
  merge_group:
    types: [checks_requested]
name: unittest

permissions:
  contents: read

concurrency:
  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
  cancel-in-progress: true

env:
  PACKAGE_DIRS: packages preview-packages
  NOX_REUSE_EXISTING_VIRTUALENVS: "true"
  NOX_ENVDIR: "/tmp/shared_nox_envs"
  NOX_DEFAULT_VENV_BACKEND: "uv"
  UV_VENV_SEED: "1"
  # Note: PARALLEL_WORKERS must remain "1" for unit tests because running pytest concurrently
  # on the same runner VM can cause socket/IPC/process deadlocks across packages.
  PARALLEL_WORKERS: "1"

jobs:
  initialize:
    runs-on: ubuntu-latest
    outputs:
      matrix: ${{ steps.set-matrix.outputs.matrix }}
      packages: ${{ steps.set-matrix.outputs.packages }}
      is_full_run: ${{ steps.check-label.outputs.is_full_run }}
    env:
      MAX_SHARDS: 16
      # Define weights for long-running unit tests to balance shard execution time
      # Each weight is roughly 1 minute of expected execution time
      # Default for unset packages is 1
      PACKAGE_WEIGHTS: |
        django-google-spanner: 2
        gapic-generator: 3
        google-ai-generativelanguage: 3
        google-api-core: 2
        google-auth: 2
        google-cloud-bigquery: 4
        google-cloud-bigquery-storage: 2
        google-cloud-bigtable: 4
        google-cloud-compute: 12
        google-cloud-compute-v1beta: 12
        google-cloud-datastore: 3
        google-cloud-dialogflow: 4
        google-cloud-dialogflow-cx: 4
        google-cloud-discoveryengine: 4
        google-cloud-firestore: 3
        google-cloud-logging: 3
        google-cloud-monitoring: 3
        google-cloud-ndb: 2
        google-cloud-pubsub: 2
        google-cloud-retail: 3
        google-cloud-spanner: 4
        google-cloud-storage: 4
        google-shopping-merchant-accounts: 3
        pandas-gbq: 2
        proto-plus: 1
        sqlalchemy-bigquery: 2
        sqlalchemy-spanner: 2
    steps:
    - name: Checkout
      uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
      with:
        fetch-depth: 0
        filter: blob:none
        persist-credentials: false
    - name: Check for unit_test:all_packages label
      id: check-label
      run: |
        if [[ "${{ contains(github.event.pull_request.labels.*.name, 'unit_test:all_packages') }}" == "true" ]]; then
          echo "is_full_run=true" >> $GITHUB_OUTPUT
        else
          echo "is_full_run=false" >> $GITHUB_OUTPUT
        fi
    - name: Setup Python
      uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
      with:
        python-version: "3.10"
    - name: Get package shards
      id: set-matrix
      env:
        BUILD_TYPE: presubmit
        TARGET_BRANCH: ${{ github.base_ref || github.event.merge_group.base_ref }}
        TEST_ALL_PACKAGES: ${{ steps.check-label.outputs.is_full_run }}
        MAX_SHARDS: ${{ env.MAX_SHARDS }}
        PACKAGE_WEIGHTS: ${{ env.PACKAGE_WEIGHTS }}
      run: |
        if [ -n "$TARGET_BRANCH" ]; then
          git fetch --no-tags --quiet origin "$TARGET_BRANCH" || true
        fi
        python3 ci/get_package_shards.py

  unit:
    needs: initialize
    if: needs.initialize.outputs.matrix != '[]' && needs.initialize.outputs.matrix != ''
    runs-on: ubuntu-22.04
    strategy:
      fail-fast: true
      matrix:
        python: ["3.10", "3.11", "3.12", "3.13", "3.14", "3.15"]
        package_shard: ${{ fromJson(needs.initialize.outputs.matrix) }}
    name: ${{ matrix.package_shard.is_sharded && format('unit ({0}, {1})', matrix.python, matrix.package_shard.name) || format('unit ({0})', matrix.python) }}
    steps:
    - name: Checkout
      uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
      # Use a fetch-depth of 2 to avoid error `fatal: origin/main...HEAD: no merge base`
      # See https://github.com/googleapis/google-cloud-python/issues/12013
      # and https://github.com/actions/checkout#checkout-head.
      with:
        fetch-depth: 2
        persist-credentials: false
    - name: Setup Python
      uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6
      with:
        python-version: ${{ matrix.python }}
        cache: 'pip'
        allow-prereleases: true
    - name: Setup uv
      uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86 # v5
      with:
        enable-cache: true
        cache-dependency-glob: 'packages/**/testing/constraints*.txt'
    - name: Install nox and nox-uv
      run: |
        uv pip install --system nox nox-uv
    - name: Run unit tests for ${{ matrix.package_shard.description }}
      env:
        BUILD_TYPE: presubmit
        TARGET_BRANCH: ${{ github.base_ref || github.event.merge_group.base_ref }}
        TEST_TYPE: unit
        PY_VERSION: ${{ matrix.python }}
        PACKAGE_LIST: ${{ matrix.package_shard.packages }}
      run: |
        ci/run_conditional_tests.sh
    - name: Upload coverage results
      uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
      with:
        name: coverage-artifact-${{ matrix.python }}-${{ matrix.package_shard.index }}
        path: .coverage.${{ matrix.python }}.*
        include-hidden-files: true

  core-deps:
    needs: initialize
    if: needs.initialize.outputs.matrix != '[]' && needs.initialize.outputs.matrix != ''
    runs-on: ubuntu-22.04
    strategy:
      fail-fast: true
      matrix:
        python: ["3.14"]
        package_shard: ${{ fromJson(needs.initialize.outputs.matrix) }}
    name: ${{ matrix.package_shard.is_sharded && format('core-deps handwritten ({0}, {1})', matrix.python, matrix.package_shard.name) || format('core-deps handwritten ({0})', matrix.python) }}
    steps:
    - name: Checkout
      uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
      # Use a fetch-depth of 2 to avoid error `fatal: origin/main...HEAD: no merge base`
      # See https://github.com/googleapis/google-cloud-python/issues/12013
      # and https://github.com/actions/checkout#checkout-head.
      with:
        fetch-depth: 2
        persist-credentials: false
    - name: Setup Python
      uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6
      with:
        python-version: ${{ matrix.python }}
        cache: 'pip'
        allow-prereleases: true
    - name: Install nox
      run: |
        pip install nox
    - name: Run core_deps_from_source for ${{ matrix.package_shard.description }}
      env:
        BUILD_TYPE: presubmit
        TARGET_BRANCH: ${{ github.base_ref || github.event.merge_group.base_ref }}
        TEST_TYPE: core_deps_from_source
        PY_VERSION: ${{ matrix.python }}
        PACKAGE_LIST: ${{ matrix.package_shard.packages }}
        NOX_DEFAULT_VENV_BACKEND: "virtualenv"
        NOXFORCEPYTHON: ${{ matrix.python }}
      run: |
        ci/run_conditional_tests.sh

  all-tests:
    needs: [initialize, unit, core-deps]
    if: always()
    runs-on: ubuntu-latest
    steps:
      - name: Check test results
        run: |
          # 1. Check initialize job
          if [[ "${{ needs.initialize.result }}" != "success" ]]; then
            echo "Error: The initialize job status was: ${{ needs.initialize.result }}"
            exit 1
          fi
          # 2. Check unit test shards
          if [[ "${{ needs.unit.result }}" != "success" && "${{ needs.unit.result }}" != "skipped" ]]; then
            echo "Unit tests failed"
            exit 1
          fi
          # 3. Check core dependencies test shards
          if [[ "${{ needs.core-deps.result }}" != "success" && "${{ needs.core-deps.result }}" != "skipped" ]]; then
            echo "Core dependencies tests failed"
            exit 1
          fi
          echo "All unit and core dependencies tests passed or were skipped"

  cover:
    if: always() && !cancelled() && needs.all-tests.result == 'success'
    runs-on: ubuntu-latest
    needs:
        - all-tests
        - initialize
    steps:
    - name: Checkout
      uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
      # Use a fetch-depth of 2 to avoid error `fatal: origin/main...HEAD: no merge base`
      # See https://github.com/googleapis/google-cloud-python/issues/12013
      # and https://github.com/actions/checkout#checkout-head.
      with:
        fetch-depth: 2
        persist-credentials: false
    - name: Setup Python
      uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
      with:
        python-version: "3.10"
    - name: Install coverage
      if: ${{ needs.initialize.outputs.packages != '' }}
      run: |
        python -m pip install --upgrade setuptools pip wheel
        python -m pip install coverage
    - name: Download coverage results
      if: ${{ needs.initialize.outputs.packages != '' }}
      uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5
      with:
        path: /dev/shm/.coverage-results/
        merge-multiple: true
    - name: Report coverage results
      env:
        # TODO: default to 100% coverage after next gapic-generator release
        # https://github.com/googleapis/google-cloud-python/issues/17459
        DEFAULT_FAIL_UNDER: 99
        TEST_ALL_PACKAGES: ${{ needs.initialize.outputs.is_full_run }}
        PACKAGE_LIST: ${{ needs.initialize.outputs.packages }}
        TARGET_BRANCH: ${{ github.base_ref || github.event.merge_group.base_ref }}
        BUILD_TYPE: presubmit
      run: |
        ci/report_coverage.sh