python.yml

name: Python client CI

permissions:
    contents: read

on:
    push:
        branches:
            - main
            - release-*
            - v*
        paths:
            - python/**
            - glide-core/src/**
            - glide-core/redis-rs/redis/src/**/**
            - utils/cluster_manager.py
            - .github/workflows/python.yml
            - .github/workflows/build-python-wrapper/action.yml
            - .github/actions/install-shared-dependencies/action.yml
            - .github/workflows/test-benchmark/action.yml
            - .github/workflows/lint-rust/action.yml
            - .github/actions/install-engine/action.yml
            - .github/workflows/start-self-hosted-runner/action.yml
            - .github/workflows/create-test-matrices/action.yml
            - .github/json_matrices/**
            - "!**/*.md"
            - "!**/THIRD_PARTY_LICENSES*"

    pull_request:
        paths:
            - python/**
            - glide-core/src/**
            - glide-core/redis-rs/redis/src/**
            - utils/cluster_manager.py
            - .github/workflows/python.yml
            - .github/workflows/build-python-wrapper/action.yml
            - .github/actions/install-shared-dependencies/action.yml
            - .github/workflows/test-benchmark/action.yml
            - .github/workflows/lint-rust/action.yml
            - .github/actions/install-engine/action.yml
            - .github/workflows/start-self-hosted-runner/action.yml
            - .github/workflows/create-test-matrices/action.yml
            - .github/json_matrices/**
            - "!**/*.md"
            - "!**/THIRD_PARTY_LICENSES*"

    workflow_dispatch:
        inputs:
            full-matrix:
                description: "Run the full engine, host, and language version matrix"
                type: boolean
                default: false
            run-with-macos:
                description: Include macOS runs (avoid GitHub runners unless specifically required)
                type: choice
                options:
                    - false
                    - use-self-hosted
                    - use-github
                default: false
            use-self-hosted-linux:
                description: "Use self-hosted runners for Linux x64 (for release-day runs)"
                type: boolean
                default: false
            name:
                required: false
                type: string
                description: "(Optional) Test run name"
            run_async_tests:
                description: "Run only async tests"
                required: false
                type: boolean
                default: false
            run_sync_tests:
                description: "run only sync tests"
                required: false
                type: boolean
                default: false
            run-dynamic-pubsub-with-mock:
                description: "Build with mock pubsub and run dynamic pubsub tests"
                type: boolean
                default: false
            containers-only:
                description: "Run only container tests (skip regular test-python job)"
                type: boolean
                default: false

    workflow_call:
        inputs:
            full-matrix:
                description: "Run the full engine, host, and language version matrix"
                type: boolean
                default: false
            run-with-macos:
                description: "Run with macos included (only when necessary)"
                type: string
                default: "false"
            use-self-hosted-linux:
                description: "Use self-hosted runners for Linux x64"
                type: boolean
                default: false

concurrency:
    group: python-${{ github.head_ref || github.ref }}-${{ toJson(inputs) }}
    cancel-in-progress: true

run-name:
    # Set custom name if job is started manually and name is given
    ${{ github.event_name == 'workflow_dispatch' && (inputs.name == '' && format('{0} @ {1} {2}', github.ref_name, github.sha, toJson(inputs)) || inputs.name) || '' }}

env:
    # Run full test matrix if job started by cron or it was explictly specified by a person who triggered the workflow
    RUN_FULL_MATRIX: ${{ (github.event.inputs.full-matrix == 'true' || github.event_name == 'schedule' || inputs.full-matrix == true) }}
    RUN_WITH_MACOS: ${{ (github.event.inputs.run-with-macos) }}
    RUN_SYNC_ONLY: ${{ (github.event.inputs.run_sync_tests == 'true' && github.event.inputs.run_async_tests == 'false') }}
    RUN_ASYNC_ONLY: ${{ (github.event.inputs.run_async_tests == 'true' && github.event.inputs.run_sync_tests == 'false') }}
    BASE_PYTHON_VERSION: "3.13"

jobs:
    get-matrices:
        runs-on: ubuntu-24.04
        outputs:
            engine-matrix-output: ${{ steps.get-matrices.outputs.engine-matrix-output }}
            host-matrix-output: ${{ steps.get-matrices.outputs.host-matrix-output }}
            version-matrix-output: ${{ steps.get-matrices.outputs.version-matrix-output }}
        steps:
            - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
            - id: get-matrices
              uses: ./.github/workflows/create-test-matrices
              with:
                  language-name: python
                  run-full-matrix: ${{ env.RUN_FULL_MATRIX == 'true' }}
                  run-with-macos: ${{ env.RUN_WITH_MACOS }}
                  use-self-hosted-linux: ${{ inputs.use-self-hosted-linux }}

    approve-self-hosted:
        if: inputs.use-self-hosted-linux == true || contains(github.event.pull_request.labels.*.name, 'ci/use-self-hosted')
        runs-on: ubuntu-24.04
        environment: AWS_ACTIONS
        needs: get-matrices
        steps:
            - name: Approval gate
              run: echo "Self-hosted Linux runner usage approved"

    test-python:
        name: Python Tests - ${{ matrix.python }}, EngineVersion - ${{ matrix.engine.version }}, Target - ${{ matrix.host.TARGET }}
        runs-on: ${{ matrix.host.RUNNER }}
        needs: [get-matrices, approve-self-hosted]
        if: always() && needs.get-matrices.result == 'success' && (needs.approve-self-hosted.result == 'success' || needs.approve-self-hosted.result == 'skipped') && github.event.inputs.containers-only != 'true'
        timeout-minutes: 120
        strategy:
            fail-fast: false
            matrix:
                python: ${{ fromJson(needs.get-matrices.outputs.version-matrix-output) }}
                engine: ${{ fromJson(needs.get-matrices.outputs.engine-matrix-output) }}
                host: ${{ fromJson(needs.get-matrices.outputs.host-matrix-output) }}
        env:
            VALKEY_GLIDE_DNS_TESTS_ENABLED: "1"
            CARGO_TARGET_DIR: ${{ github.workspace }}/python/target
            CARGO_INCREMENTAL: "0"
        steps:
            - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

            - name: Output Matrix Parameters for this job
              run: |
                  echo "Job running with the following matrix configuration:"
                  echo "${{ toJson(matrix) }}"

            - name: Set up Python
              uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
              with:
                  python-version: ${{ matrix.python }}

            - name: Install dependencies
              working-directory: ./python
              run: |
                  python -m pip install --upgrade pip

            - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
              with:
                  path: |
                      python/target
                  key: ${{ matrix.host.TARGET }}-python-v2-${{ hashFiles('python/glide-async/Cargo.lock', 'python/glide-shared/Cargo.lock', 'ffi/Cargo.lock') }}
                  restore-keys: |
                      ${{ matrix.host.TARGET }}-python-v2-

            - name: Build Python wrapper
              uses: ./.github/workflows/build-python-wrapper
              with:
                  os: ${{ matrix.host.OS }}
                  target: ${{ matrix.host.TARGET }}
                  github-token: ${{ secrets.GITHUB_TOKEN }}
                  engine-version: ${{ matrix.engine.version }}

            - name: Configure DNS for tests
              uses: ./.github/actions/setup-dns-tests
              with:
                  os: ${{ matrix.host.OS }}

            - name: Test with pytest
              if: ${{ env.RUN_FULL_MATRIX != 'true' && env.RUN_SYNC_ONLY == 'false' && env.RUN_ASYNC_ONLY == 'false' }}
              working-directory: ./python
              env:
                  TEST_NAME_ARG: ${{ inputs.name && format('-k {0}', inputs.name) || '' }}
                  AWS_ACCESS_KEY_ID: test_access_key
                  AWS_SECRET_ACCESS_KEY: test_secret_key
                  AWS_SESSION_TOKEN: test_session_token
              run: |
                  ulimit -n 65536 2>/dev/null || true
                  python3 dev.py test --args -n 4 --dist loadgroup -m "not serial_only" --async-backend=asyncio --html=pytest_report.html --self-contained-html "$TEST_NAME_ARG"

            - name: Test with pytest (full matrix, all async backends)
              if: ${{ env.RUN_FULL_MATRIX == 'true' }}
              working-directory: ./python
              env:
                  AWS_ACCESS_KEY_ID: test_access_key
                  AWS_SECRET_ACCESS_KEY: test_secret_key
                  AWS_SESSION_TOKEN: test_session_token
              run: |
                  ulimit -n 65536 2>/dev/null || true
                  python3 dev.py test --args -n 4 --dist loadgroup -m "not serial_only" --async-backend=asyncio --async-backend=trio --html=pytest_report.html --self-contained-html

            - name: Test with pytest (async only)
              if: ${{ env.RUN_ASYNC_ONLY == 'true' && env.RUN_FULL_MATRIX != 'true'}}
              working-directory: ./python
              env:
                  AWS_ACCESS_KEY_ID: test_access_key
                  AWS_SECRET_ACCESS_KEY: test_secret_key
                  AWS_SESSION_TOKEN: test_session_token
              run: |
                  ulimit -n 65536 2>/dev/null || true
                  python3 dev.py test --args -n 4 --dist loadgroup -m "not serial_only" tests/async_tests --async-backend=asyncio --async-backend=trio --html=pytest_report.html --self-contained-html

            - name: Test with pytest (sync only)
              if: ${{ env.RUN_SYNC_ONLY == 'true' && env.RUN_FULL_MATRIX != 'true'}}
              working-directory: ./python
              env:
                  AWS_ACCESS_KEY_ID: test_access_key
                  AWS_SECRET_ACCESS_KEY: test_secret_key
                  AWS_SESSION_TOKEN: test_session_token
              run: |
                  ulimit -n 65536 2>/dev/null || true
                  python3 dev.py test --args -n 4 --dist loadgroup -m "not serial_only" tests/sync_tests --html=pytest_report.html --self-contained-html

            - name: Test with pytest (serial-only tests)
              working-directory: ./python
              env:
                  TEST_NAME_ARG: ${{ inputs.name && format('-k {0}', inputs.name) || '' }}
                  AWS_ACCESS_KEY_ID: test_access_key
                  AWS_SECRET_ACCESS_KEY: test_secret_key
                  AWS_SESSION_TOKEN: test_session_token
              run: |
                  # Same mode selection as the xdist steps above, so each mode runs
                  # its serial tests with its own backends and directory.
                  if [ "$RUN_FULL_MATRIX" = "true" ]; then
                      python3 dev.py test --args -m serial_only --async-backend=asyncio --async-backend=trio --html=pytest_report_serial.html --self-contained-html
                  elif [ "$RUN_ASYNC_ONLY" = "true" ]; then
                      python3 dev.py test --args -m serial_only tests/async_tests --async-backend=asyncio --async-backend=trio --html=pytest_report_serial.html --self-contained-html
                  elif [ "$RUN_SYNC_ONLY" = "true" ]; then
                      python3 dev.py test --args -m serial_only tests/sync_tests --html=pytest_report_serial.html --self-contained-html
                  else
                      python3 dev.py test --args -m serial_only --async-backend=asyncio --html=pytest_report_serial.html --self-contained-html "$TEST_NAME_ARG"
                  fi

            - uses: ./.github/workflows/test-benchmark
              with:
                  language-flag: -python
                  engine-run: ${{ matrix.engine.run }}
                  target: ${{ matrix.host.TARGET }}
                  language-version: ${{ matrix.python }}
                  base-language-version: ${{ env.BASE_PYTHON_VERSION }}

            - name: Upload test reports
              if: always()
              continue-on-error: true
              uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
              with:
                  name: test-report-python-${{ matrix.python }}-${{ matrix.engine.type }}-${{ matrix.engine.version }}-${{ matrix.host.OS }}-${{ matrix.host.ARCH }}
                  path: |
                      python/tests/pytest_report.html
                      python/tests/pytest_report_serial.html
                      utils/clusters/**
                      benchmarks/results/**

    # run pubsub tests in another job - they take too much time
    test-pubsub-python:
        name: Python PubSubTests - ${{ matrix.python }}, EngineVersion - ${{ matrix.engine.version }}, Target - ${{ matrix.host.TARGET }}
        runs-on: ${{ matrix.host.RUNNER }}
        needs: [get-matrices, approve-self-hosted]
        if: always() && needs.get-matrices.result == 'success' && (needs.approve-self-hosted.result == 'success' || needs.approve-self-hosted.result == 'skipped') && github.event.inputs.containers-only != 'true'
        timeout-minutes: 120
        strategy:
            fail-fast: false
            matrix:
                python: ${{ fromJson(needs.get-matrices.outputs.version-matrix-output) }}
                engine: ${{ fromJson(needs.get-matrices.outputs.engine-matrix-output) }}
                host: ${{ fromJson(needs.get-matrices.outputs.host-matrix-output) }}
        env:
            CARGO_TARGET_DIR: ${{ github.workspace }}/python/target
            CARGO_INCREMENTAL: "0"
        steps:
            - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

            - name: Output Matrix Parameters for this job
              run: |
                  echo "Job running with the following matrix configuration:"
                  echo "${{ toJson(matrix) }}"

            - name: Set up Python
              uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
              with:
                  python-version: ${{ matrix.python }}

            - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
              with:
                  path: |
                      python/target
                  key: ${{ matrix.host.TARGET }}-python-v2-${{ hashFiles('python/glide-async/Cargo.lock', 'python/glide-shared/Cargo.lock', 'ffi/Cargo.lock') }}
                  restore-keys: |
                      ${{ matrix.host.TARGET }}-python-v2-

            - name: Build Python wrapper
              uses: ./.github/workflows/build-python-wrapper
              with:
                  os: ${{ matrix.host.OS }}
                  target: ${{ matrix.host.TARGET }}
                  github-token: ${{ secrets.GITHUB_TOKEN }}
                  engine-version: ${{ matrix.engine.version }}

            - name: Test pubsub with pytest
              if: ${{ env.RUN_FULL_MATRIX != 'true' && env.RUN_SYNC_ONLY == 'false' && env.RUN_ASYNC_ONLY == 'false' }}
              working-directory: ./python
              run: |
                  ulimit -n 65536 2>/dev/null || true
                  python3 dev.py test --args -k 'pubsub' --html=pytest_report.html --self-contained-html

            - name: Test pubsub with pytest (full matrix)
              if: ${{ env.RUN_FULL_MATRIX == 'true' }}
              working-directory: ./python
              run: |
                  ulimit -n 65536 2>/dev/null || true
                  python3 dev.py test --args -k 'pubsub' --async-backend=asyncio --async-backend=trio --html=pytest_report.html --self-contained-html

            - name: Test pubsub with pytest (async only)
              if: ${{ env.RUN_ASYNC_ONLY == 'true' && env.RUN_FULL_MATRIX != 'true'}}
              working-directory: ./python
              run: |
                  ulimit -n 65536 2>/dev/null || true
                  python3 dev.py test --args -k 'pubsub' tests/async_tests --async-backend=asyncio --async-backend=trio --html=pytest_report.html --self-contained-html

            - name: Test pubsub with pytest (sync only)
              if: ${{ env.RUN_SYNC_ONLY == 'true' && env.RUN_FULL_MATRIX != 'true'}}
              working-directory: ./python
              run: |
                  ulimit -n 65536 2>/dev/null || true
                  python3 dev.py test --args -k 'pubsub' tests/sync_tests --html=pytest_report.html --self-contained-html

            - name: Detect mock pubsub changes
              id: pubsub_mock_changes
              uses: dorny/paths-filter@ceb8a2b8f2d89434be7ff52d3de7ec3738c5cc9d # v4.0.3
              with:
                  filters: |
                      mock_pubsub:
                          - glide-core/src/pubsub/mock.rs

            - name: Rebuild Python wrapper with pubsub mock
              if: ${{ env.RUN_FULL_MATRIX != 'true'
                  && (github.event.inputs.run-dynamic-pubsub-with-mock == 'true'
                  || steps.pubsub_mock_changes.outputs.mock_pubsub == 'true') }}
              uses: ./.github/workflows/build-python-wrapper
              with:
                  os: ${{ matrix.host.OS }}
                  target: ${{ matrix.host.TARGET }}
                  github-token: ${{ secrets.GITHUB_TOKEN }}
                  engine-version: ${{ matrix.engine.version }}
                  features: mock-pubsub

            - name: Test dynamic pubsub with mock
              if: ${{ env.RUN_FULL_MATRIX != 'true'
                  && (github.event.inputs.run-dynamic-pubsub-with-mock == 'true'
                  || steps.pubsub_mock_changes.outputs.mock_pubsub == 'true') }}
              working-directory: ./python
              run: |
                  python3 dev.py test --mock-pubsub --args -k 'pubsub' tests/async_tests --html=pytest_report.html --self-contained-html

            - name: Upload test reports
              if: always()
              continue-on-error: true
              uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
              with:
                  name: pubsub-test-report-python-${{ matrix.python }}-${{ matrix.engine.type }}-${{ matrix.engine.version }}-${{ matrix.host.OS }}-${{ matrix.host.ARCH }}
                  path: |
                      python/tests/pytest_report.html

    lint:
        runs-on: ubuntu-24.04
        if: ${{ github.event.inputs.containers-only != 'true' }}
        timeout-minutes: 15
        steps:
            - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

            - name: lint rust
              uses: ./.github/workflows/lint-rust
              with:
                  cargo-toml-folder: python/glide-async
                  github-token: ${{ secrets.GITHUB_TOKEN }}

            - name: Install dependencies
              if: always()
              uses: threeal/pipx-install-action@b0bf0add7d5aefda03a3d4e47d651df807889e10 # v1.0.0
              with:
                  packages: flake8 isort black mypy

            - name: Lint python
              if: always()
              working-directory: ./python
              run: |
                  python3 dev.py lint --check

    docs-test:
        runs-on: ubuntu-24.04
        if: ${{ github.event.inputs.containers-only != 'true' }}
        timeout-minutes: 15
        steps:
            - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

            - name: Install dependencies
              working-directory: ./python
              run: |
                  python -m pip install --upgrade pip
                  pip install sphinx sphinx-rtd-theme

            - name: Build docs
              working-directory: ./python/docs
              run: |
                  sphinx-apidoc -o glide_async ../glide-async/python
                  sphinx-apidoc -o glide_sync ../glide-sync/glide_sync
                  sphinx-apidoc -o glide_shared ../glide-shared/glide_shared
                  make clean
                  make html SPHINXOPTS="-W --keep-going"

    get-containers:
        runs-on: ubuntu-24.04
        if: ${{ github.event.inputs.full-matrix == 'true' || github.event.inputs.containers-only == 'true' || github.event_name == 'schedule' || inputs.full-matrix == true }}
        outputs:
            engine-matrix-output: ${{ steps.get-matrices.outputs.engine-matrix-output }}
            host-matrix-output: ${{ steps.get-matrices.outputs.host-matrix-output }}
            version-matrix-output: ${{ steps.get-matrices.outputs.version-matrix-output }}

        steps:
            - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
            - id: get-matrices
              uses: ./.github/workflows/create-test-matrices
              with:
                  language-name: python
                  run-full-matrix: true
                  containers: true
                  use-self-hosted-linux: ${{ inputs.use-self-hosted-linux }}

    test-python-container:
        runs-on: ${{ matrix.host.RUNNER }}
        needs: [get-containers, approve-self-hosted]
        if: always() && needs.get-containers.result == 'success' && (needs.approve-self-hosted.result == 'success' || needs.approve-self-hosted.result == 'skipped')
        timeout-minutes: 90
        strategy:
            fail-fast: false
            matrix:
                # Don't use generated matrix for python until compatibility issues resolved on amazon linux
                # python: ${{ fromJson(needs.get-containers.outputs.version-matrix-output) }}
                engine: ${{ fromJson(needs.get-containers.outputs.engine-matrix-output) }}
                host: ${{ fromJson(needs.get-containers.outputs.host-matrix-output) }}
        env:
            VALKEY_GLIDE_DNS_TESTS_ENABLED: "1"
            CARGO_INCREMENTAL: "0"
        container:
            image: ${{ matrix.host.IMAGE }}
            options: ${{ join(' -q ', matrix.host.CONTAINER_OPTIONS) }} # adding `-q` to bypass empty options
        steps:
            - name: Install git and python
              run: |
                  yum update
                  yum install -y git tar python3
                  python3 -m ensurepip --upgrade
                  python3 -m pip install --upgrade pip
                  python3 -m pip install mypy-protobuf virtualenv
                  echo IMAGE=amazonlinux:latest | sed -r 's/:/-/g' >> $GITHUB_ENV
            # Replace `:` in the variable otherwise it can't be used in `upload-artifact`
            - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

            # Inside a container, `${{ github.workspace }}` is the host path, not
            # the mounted one (actions/runner#2058); only the shell's
            # $GITHUB_WORKSPACE matches where actions/cache resolves python/target.
            - name: Point cargo at the cached target dir
              run: echo "CARGO_TARGET_DIR=$GITHUB_WORKSPACE/python/target" >> "$GITHUB_ENV"

            - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
              with:
                  path: |
                      python/target
                  key: ${{ matrix.host.IMAGE }}-python-v2-${{ hashFiles('python/glide-async/Cargo.lock', 'python/glide-shared/Cargo.lock', 'ffi/Cargo.lock') }}
                  restore-keys: |
                      ${{ matrix.host.IMAGE }}-python-v2-

            - name: Build Python wrapper
              uses: ./.github/workflows/build-python-wrapper
              with:
                  os: ${{ matrix.host.OS }}
                  target: ${{ matrix.host.TARGET }}
                  github-token: ${{ secrets.GITHUB_TOKEN }}
                  engine-version: ${{ matrix.engine.version }}

            - name: Configure DNS for tests
              uses: ./.github/actions/setup-dns-tests
              with:
                  os: ${{ matrix.host.OS }}

            - name: Test with pytest (asyncio only)
              if: ${{ env.RUN_FULL_MATRIX != 'true' }}
              working-directory: ./python
              env:
                  AWS_ACCESS_KEY_ID: test_access_key
                  AWS_SECRET_ACCESS_KEY: test_secret_key
                  AWS_SESSION_TOKEN: test_session_token
              run: |
                  python3 dev.py test --args --html=pytest_report.html --self-contained-html

            - name: Test with pytest (full matrix)
              if: ${{ env.RUN_FULL_MATRIX == 'true' }}
              working-directory: ./python
              env:
                  AWS_ACCESS_KEY_ID: test_access_key
                  AWS_SECRET_ACCESS_KEY: test_secret_key
                  AWS_SESSION_TOKEN: test_session_token
              run: |
                  python3 dev.py test --args --async-backend=asyncio --async-backend=trio --html=pytest_report.html --self-contained-html

            - name: Upload test reports
              if: always()
              continue-on-error: true
              uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
              with:
                  name: test-report-python-${{ matrix.python }}-${{ matrix.engine.type }}-${{ matrix.engine.version }}-${{ env.IMAGE }}-${{ matrix.host.ARCH }}
                  path: |
                      python/tests/pytest_report.html
                      utils/clusters/**
                      benchmarks/results/**