name: Tests

# gh-84728: "paths-ignore" is not used to skip documentation-only PRs, because
# it prevents to mark a job as mandatory. A PR cannot be merged if a job is
# mandatory but not scheduled because of "paths-ignore".
on:
  workflow_dispatch:
  push:
    branches:
    - 'main'
    - '3.12'
    - '3.11'
    - '3.10'
    - '3.9'
    - '3.8'
  pull_request:
    branches:
    - 'main'
    - '3.12'
    - '3.11'
    - '3.10'
    - '3.9'
    - '3.8'

permissions:
  contents: read

concurrency:
  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}-reusable
  cancel-in-progress: true

jobs:
  check_source:
    name: 'Check for source changes'
    runs-on: ubuntu-latest
    timeout-minutes: 10
    outputs:
      run-docs: ${{ steps.docs-changes.outputs.run-docs || false }}
      run_tests: ${{ steps.check.outputs.run_tests }}
      run_hypothesis: ${{ steps.check.outputs.run_hypothesis }}
      run_cifuzz: ${{ steps.check.outputs.run_cifuzz }}
      config_hash: ${{ steps.config_hash.outputs.hash }}
    steps:
      - uses: actions/checkout@v4
      - name: Check for source changes
        id: check
        run: |
          if [ -z "$GITHUB_BASE_REF" ]; then
            echo "run_tests=true" >> $GITHUB_OUTPUT
          else
            git fetch origin $GITHUB_BASE_REF --depth=1
            # git diff "origin/$GITHUB_BASE_REF..." (3 dots) may be more
            # reliable than git diff "origin/$GITHUB_BASE_REF.." (2 dots),
            # but it requires to download more commits (this job uses
            # "git fetch --depth=1").
            #
            # git diff "origin/$GITHUB_BASE_REF..." (3 dots) works with Git
            # 2.26, but Git 2.28 is stricter and fails with "no merge base".
            #
            # git diff "origin/$GITHUB_BASE_REF.." (2 dots) should be enough on
            # GitHub, since GitHub starts by merging origin/$GITHUB_BASE_REF
            # into the PR branch anyway.
            #
            # https://github.com/python/core-workflow/issues/373
            git diff --name-only origin/$GITHUB_BASE_REF.. | grep -qvE '(\.rst$|^Doc|^Misc|^\.pre-commit-config\.yaml$|\.ruff\.toml$)' && echo "run_tests=true" >> $GITHUB_OUTPUT || true
          fi

          # Check if we should run hypothesis tests
          GIT_BRANCH=${GITHUB_BASE_REF:-${GITHUB_REF#refs/heads/}}
          echo $GIT_BRANCH
          if $(echo "$GIT_BRANCH" | grep -q -w '3\.\(8\|9\|10\|11\)'); then
            echo "Branch too old for hypothesis tests"
            echo "run_hypothesis=false" >> $GITHUB_OUTPUT
          else
            echo "Run hypothesis tests"
            echo "run_hypothesis=true" >> $GITHUB_OUTPUT
          fi

          # oss-fuzz maintains a configuration for fuzzing the main branch of
          # CPython, so CIFuzz should be run only for code that is likely to be
          # merged into the main branch; compatibility with older branches may
          # be broken.
          FUZZ_RELEVANT_FILES='(\.c$|\.h$|\.cpp$|^configure$|^\.github/workflows/build\.yml$|^Modules/_xxtestfuzz)'
          if [ "$GITHUB_BASE_REF" = "main" ] && [ "$(git diff --name-only origin/$GITHUB_BASE_REF.. | grep -qE $FUZZ_RELEVANT_FILES; echo $?)" -eq 0 ]; then
            # The tests are pretty slow so they are executed only for PRs
            # changing relevant files.
            echo "Run CIFuzz tests"
            echo "run_cifuzz=true" >> $GITHUB_OUTPUT
          else
            echo "Branch too old for CIFuzz tests; or no C files were changed"
            echo "run_cifuzz=false" >> $GITHUB_OUTPUT
          fi
      - name: Compute hash for config cache key
        id: config_hash
        run: |
          echo "hash=${{ hashFiles('configure', 'configure.ac', '.github/workflows/build.yml') }}" >> $GITHUB_OUTPUT
      - name: Get a list of the changed documentation-related files
        if: github.event_name == 'pull_request'
        id: changed-docs-files
        uses: Ana06/get-changed-files@v2.2.0
        with:
          filter: |
            Doc/**
            Misc/**
            .github/workflows/reusable-docs.yml
          format: csv  # works for paths with spaces
      - name: Check for docs changes
        if: >-
          github.event_name == 'pull_request'
          && steps.changed-docs-files.outputs.added_modified_renamed != ''
        id: docs-changes
        run: |
          echo "run-docs=true" >> "${GITHUB_OUTPUT}"

  check-docs:
    name: Docs
    needs: check_source
    if: fromJSON(needs.check_source.outputs.run-docs)
    uses: ./.github/workflows/reusable-docs.yml

  check_generated_files:
    name: 'Check if generated files are up to date'
    # Don't use ubuntu-latest but a specific version to make the job
    # reproducible: to get the same tools versions (autoconf, aclocal, ...)
    runs-on: ubuntu-22.04
    timeout-minutes: 60
    needs: check_source
    if: needs.check_source.outputs.run_tests == 'true'
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: '3.x'
      - name: Restore config.cache
        uses: actions/cache@v4
        with:
          path: config.cache
          key: ${{ github.job }}-${{ runner.os }}-${{ needs.check_source.outputs.config_hash }}-${{ env.pythonLocation }}
      - name: Install Dependencies
        run: sudo ./.github/workflows/posix-deps-apt.sh
      - name: Add ccache to PATH
        run: echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV
      - name: Configure ccache action
        uses: hendrikmuhs/ccache-action@v1.2
        with:
          save: false
      - name: Check Autoconf and aclocal versions
        run: |
          grep "Generated by GNU Autoconf 2.71" configure
          grep "aclocal 1.16.5" aclocal.m4
          grep -q "runstatedir" configure
          grep -q "PKG_PROG_PKG_CONFIG" aclocal.m4
      - name: Configure CPython
        run: |
          # Build Python with the libpython dynamic library
          ./configure --config-cache --with-pydebug --enable-shared
      - name: Regenerate autoconf files
        # Same command used by Tools/build/regen-configure.sh ($AUTORECONF)
        run: autoreconf -ivf -Werror
      - name: Build CPython
        run: |
          make -j4 regen-all
          make regen-stdlib-module-names
      - name: Check for changes
        run: |
          git add -u
          changes=$(git status --porcelain)
          # Check for changes in regenerated files
          if test -n "$changes"; then
            echo "Generated files not up to date."
            echo "Perhaps you forgot to run make regen-all or build.bat --regen. ;)"
            echo "configure files must be regenerated with a specific version of autoconf."
            echo "$changes"
            echo ""
            git diff --staged || true
            exit 1
          fi
      - name: Check exported libpython symbols
        run: make smelly
      - name: Check limited ABI symbols
        run: make check-limited-abi
      - name: Check for unsupported C global variables
        if: github.event_name == 'pull_request'  # $GITHUB_EVENT_NAME
        run: make check-c-globals

  build_windows:
    name: 'Windows'
    needs: check_source
    if: needs.check_source.outputs.run_tests == 'true'
    uses: ./.github/workflows/reusable-windows.yml

  build_windows_free_threading:
    name: 'Windows (free-threading)'
    needs: check_source
    if: needs.check_source.outputs.run_tests == 'true'
    uses: ./.github/workflows/reusable-windows.yml
    with:
      free-threading: true

  build_macos:
    name: 'macOS'
    needs: check_source
    if: needs.check_source.outputs.run_tests == 'true'
    uses: ./.github/workflows/reusable-macos.yml
    with:
      config_hash: ${{ needs.check_source.outputs.config_hash }}

  build_macos_free_threading:
    name: 'macOS (free-threading)'
    needs: check_source
    if: needs.check_source.outputs.run_tests == 'true'
    uses: ./.github/workflows/reusable-macos.yml
    with:
      config_hash: ${{ needs.check_source.outputs.config_hash }}
      free-threading: true

  build_ubuntu:
    name: 'Ubuntu'
    needs: check_source
    if: needs.check_source.outputs.run_tests == 'true'
    uses: ./.github/workflows/reusable-ubuntu.yml
    with:
      config_hash: ${{ needs.check_source.outputs.config_hash }}
      options: |
        ../cpython-ro-srcdir/configure \
          --config-cache \
          --with-pydebug \
          --with-openssl=$OPENSSL_DIR

  build_ubuntu_free_threading:
    name: 'Ubuntu (free-threading)'
    needs: check_source
    if: needs.check_source.outputs.run_tests == 'true'
    uses: ./.github/workflows/reusable-ubuntu.yml
    with:
      config_hash: ${{ needs.check_source.outputs.config_hash }}
      options: |
        ../cpython-ro-srcdir/configure \
          --config-cache \
          --with-pydebug \
          --with-openssl=$OPENSSL_DIR \
          --disable-gil

  build_ubuntu_ssltests:
    name: 'Ubuntu SSL tests with OpenSSL'
    runs-on: ubuntu-20.04
    timeout-minutes: 60
    needs: check_source
    if: needs.check_source.outputs.run_tests == 'true'
    strategy:
      fail-fast: false
      matrix:
        openssl_ver: [1.1.1w, 3.0.11, 3.1.3]
    env:
      OPENSSL_VER: ${{ matrix.openssl_ver }}
      MULTISSL_DIR: ${{ github.workspace }}/multissl
      OPENSSL_DIR: ${{ github.workspace }}/multissl/openssl/${{ matrix.openssl_ver }}
      LD_LIBRARY_PATH: ${{ github.workspace }}/multissl/openssl/${{ matrix.openssl_ver }}/lib
    steps:
    - uses: actions/checkout@v4
    - name: Restore config.cache
      uses: actions/cache@v4
      with:
        path: config.cache
        key: ${{ github.job }}-${{ runner.os }}-${{ needs.check_source.outputs.config_hash }}
    - name: Register gcc problem matcher
      run: echo "::add-matcher::.github/problem-matchers/gcc.json"
    - name: Install Dependencies
      run: sudo ./.github/workflows/posix-deps-apt.sh
    - name: Configure OpenSSL env vars
      run: |
        echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> $GITHUB_ENV
        echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> $GITHUB_ENV
        echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> $GITHUB_ENV
    - name: 'Restore OpenSSL build'
      id: cache-openssl
      uses: actions/cache@v4
      with:
        path: ./multissl/openssl/${{ env.OPENSSL_VER }}
        key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }}
    - name: Install OpenSSL
      if: steps.cache-openssl.outputs.cache-hit != 'true'
      run: python3 Tools/ssl/multissltests.py --steps=library --base-directory $MULTISSL_DIR --openssl $OPENSSL_VER --system Linux
    - name: Add ccache to PATH
      run: |
        echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV
    - name: Configure ccache action
      uses: hendrikmuhs/ccache-action@v1.2
      with:
        save: false
    - name: Configure CPython
      run: ./configure --config-cache --with-pydebug --with-openssl=$OPENSSL_DIR
    - name: Build CPython
      run: make -j4
    - name: Display build info
      run: make pythoninfo
    - name: SSL tests
      run: ./python Lib/test/ssltests.py

  test_hypothesis:
    name: "Hypothesis tests on Ubuntu"
    runs-on: ubuntu-20.04
    timeout-minutes: 60
    needs: check_source
    if: needs.check_source.outputs.run_tests == 'true' && needs.check_source.outputs.run_hypothesis == 'true'
    env:
      OPENSSL_VER: 3.0.11
      PYTHONSTRICTEXTENSIONBUILD: 1
    steps:
    - uses: actions/checkout@v4
    - name: Register gcc problem matcher
      run: echo "::add-matcher::.github/problem-matchers/gcc.json"
    - name: Install Dependencies
      run: sudo ./.github/workflows/posix-deps-apt.sh
    - name: Configure OpenSSL env vars
      run: |
        echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> $GITHUB_ENV
        echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> $GITHUB_ENV
        echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> $GITHUB_ENV
    - name: 'Restore OpenSSL build'
      id: cache-openssl
      uses: actions/cache@v4
      with:
        path: ./multissl/openssl/${{ env.OPENSSL_VER }}
        key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }}
    - name: Install OpenSSL
      if: steps.cache-openssl.outputs.cache-hit != 'true'
      run: python3 Tools/ssl/multissltests.py --steps=library --base-directory $MULTISSL_DIR --openssl $OPENSSL_VER --system Linux
    - name: Add ccache to PATH
      run: |
        echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV
    - name: Configure ccache action
      uses: hendrikmuhs/ccache-action@v1.2
      with:
        save: false
    - name: Setup directory envs for out-of-tree builds
      run: |
        echo "CPYTHON_RO_SRCDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-ro-srcdir)" >> $GITHUB_ENV
        echo "CPYTHON_BUILDDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-builddir)" >> $GITHUB_ENV
    - name: Create directories for read-only out-of-tree builds
      run: mkdir -p $CPYTHON_RO_SRCDIR $CPYTHON_BUILDDIR
    - name: Bind mount sources read-only
      run: sudo mount --bind -o ro $GITHUB_WORKSPACE $CPYTHON_RO_SRCDIR
    - name: Restore config.cache
      uses: actions/cache@v4
      with:
        path: ${{ env.CPYTHON_BUILDDIR }}/config.cache
        key: ${{ github.job }}-${{ runner.os }}-${{ needs.check_source.outputs.config_hash }}
    - name: Configure CPython out-of-tree
      working-directory: ${{ env.CPYTHON_BUILDDIR }}
      run: |
        ../cpython-ro-srcdir/configure \
          --config-cache \
          --with-pydebug \
          --with-openssl=$OPENSSL_DIR
    - name: Build CPython out-of-tree
      working-directory: ${{ env.CPYTHON_BUILDDIR }}
      run: make -j4
    - name: Display build info
      working-directory: ${{ env.CPYTHON_BUILDDIR }}
      run: make pythoninfo
    - name: Remount sources writable for tests
      # some tests write to srcdir, lack of pyc files slows down testing
      run: sudo mount $CPYTHON_RO_SRCDIR -oremount,rw
    - name: Setup directory envs for out-of-tree builds
      run: |
        echo "CPYTHON_BUILDDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-builddir)" >> $GITHUB_ENV
    - name: "Create hypothesis venv"
      working-directory: ${{ env.CPYTHON_BUILDDIR }}
      run: |
        VENV_LOC=$(realpath -m .)/hypovenv
        VENV_PYTHON=$VENV_LOC/bin/python
        echo "HYPOVENV=${VENV_LOC}" >> $GITHUB_ENV
        echo "VENV_PYTHON=${VENV_PYTHON}" >> $GITHUB_ENV
        ./python -m venv $VENV_LOC && $VENV_PYTHON -m pip install -r ${GITHUB_WORKSPACE}/Tools/requirements-hypothesis.txt
    - name: 'Restore Hypothesis database'
      id: cache-hypothesis-database
      uses: actions/cache@v4
      with:
        path: ./hypothesis
        key: hypothesis-database-${{ github.head_ref || github.run_id }}
        restore-keys: |
          - hypothesis-database-
    - name: "Run tests"
      working-directory: ${{ env.CPYTHON_BUILDDIR }}
      run: |
        # Most of the excluded tests are slow test suites with no property tests
        #
        # (GH-104097) test_sysconfig is skipped because it has tests that are
        # failing when executed from inside a virtual environment.
        ${{ env.VENV_PYTHON }} -m test \
          -W \
          -o \
          -j4 \
          -x test_asyncio \
          -x test_multiprocessing_fork \
          -x test_multiprocessing_forkserver \
          -x test_multiprocessing_spawn \
          -x test_concurrent_futures \
          -x test_socket \
          -x test_subprocess \
          -x test_signal \
          -x test_sysconfig
    - uses: actions/upload-artifact@v4
      if: always()
      with:
        name: hypothesis-example-db
        path: .hypothesis/examples/


  build_asan:
    name: 'Address sanitizer'
    runs-on: ubuntu-20.04
    timeout-minutes: 60
    needs: check_source
    if: needs.check_source.outputs.run_tests == 'true'
    env:
      OPENSSL_VER: 3.0.11
      PYTHONSTRICTEXTENSIONBUILD: 1
      ASAN_OPTIONS: detect_leaks=0:allocator_may_return_null=1:handle_segv=0
    steps:
    - uses: actions/checkout@v4
    - name: Restore config.cache
      uses: actions/cache@v4
      with:
        path: config.cache
        key: ${{ github.job }}-${{ runner.os }}-${{ needs.check_source.outputs.config_hash }}
    - name: Register gcc problem matcher
      run: echo "::add-matcher::.github/problem-matchers/gcc.json"
    - name: Install Dependencies
      run: sudo ./.github/workflows/posix-deps-apt.sh
    - name: Set up GCC-10 for ASAN
      uses: egor-tensin/setup-gcc@v1
      with:
        version: 10
    - name: Configure OpenSSL env vars
      run: |
        echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> $GITHUB_ENV
        echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> $GITHUB_ENV
        echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> $GITHUB_ENV
    - name: 'Restore OpenSSL build'
      id: cache-openssl
      uses: actions/cache@v4
      with:
        path: ./multissl/openssl/${{ env.OPENSSL_VER }}
        key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }}
    - name: Install OpenSSL
      if: steps.cache-openssl.outputs.cache-hit != 'true'
      run: python3 Tools/ssl/multissltests.py --steps=library --base-directory $MULTISSL_DIR --openssl $OPENSSL_VER --system Linux
    - name: Add ccache to PATH
      run: |
        echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV
    - name: Configure ccache action
      uses: hendrikmuhs/ccache-action@v1.2
      with:
        save: ${{ github.event_name == 'push' }}
        max-size: "200M"
    - name: Configure CPython
      run: ./configure --config-cache --with-address-sanitizer --without-pymalloc
    - name: Build CPython
      run: make -j4
    - name: Display build info
      run: make pythoninfo
    - name: Tests
      run: xvfb-run make test

  # CIFuzz job based on https://google.github.io/oss-fuzz/getting-started/continuous-integration/
  cifuzz:
    name: CIFuzz
    runs-on: ubuntu-latest
    timeout-minutes: 60
    needs: check_source
    if: needs.check_source.outputs.run_cifuzz == 'true'
    permissions:
      security-events: write
    strategy:
      fail-fast: false
      matrix:
        sanitizer: [address, undefined, memory]
    steps:
      - name: Build fuzzers (${{ matrix.sanitizer }})
        id: build
        uses: google/oss-fuzz/infra/cifuzz/actions/build_fuzzers@master
        with:
          oss-fuzz-project-name: cpython3
          sanitizer: ${{ matrix.sanitizer }}
      - name: Run fuzzers (${{ matrix.sanitizer }})
        uses: google/oss-fuzz/infra/cifuzz/actions/run_fuzzers@master
        with:
          fuzz-seconds: 600
          oss-fuzz-project-name: cpython3
          output-sarif: true
          sanitizer: ${{ matrix.sanitizer }}
      - name: Upload crash
        uses: actions/upload-artifact@v4
        if: failure() && steps.build.outcome == 'success'
        with:
          name: ${{ matrix.sanitizer }}-artifacts
          path: ./out/artifacts
      - name: Upload SARIF
        if: always() && steps.build.outcome == 'success'
        uses: github/codeql-action/upload-sarif@v3
        with:
          sarif_file: cifuzz-sarif/results.sarif
          checkout_path: cifuzz-sarif

  all-required-green:  # This job does nothing and is only used for the branch protection
    name: All required checks pass
    if: always()

    needs:
    - check_source  # Transitive dependency, needed to access `run_tests` value
    - check-docs
    - check_generated_files
    - build_macos
    - build_macos_free_threading
    - build_ubuntu
    - build_ubuntu_free_threading
    - build_ubuntu_ssltests
    - build_windows
    - build_windows_free_threading
    - test_hypothesis
    - build_asan
    - cifuzz

    runs-on: ubuntu-latest

    steps:
    - name: Check whether the needed jobs succeeded or failed
      uses: re-actors/alls-green@05ac9388f0aebcb5727afa17fcccfecd6f8ec5fe
      with:
        allowed-failures: >-
          build_ubuntu_ssltests,
          cifuzz,
          test_hypothesis,
        allowed-skips: >-
          ${{
            !fromJSON(needs.check_source.outputs.run-docs)
            && '
            check-docs,
            '
            || ''
          }}
          ${{
            needs.check_source.outputs.run_tests != 'true'
            && '
            check_generated_files,
            build_macos,
            build_macos_free_threading,
            build_ubuntu,
            build_ubuntu_free_threading,
            build_ubuntu_ssltests,
            build_windows,
            build_windows_free_threading,
            build_asan,
            '
            || ''
          }}
          ${{
            !fromJSON(needs.check_source.outputs.run_cifuzz)
            && '
            cifuzz,
            '
            || ''
          }}
          ${{
            !fromJSON(needs.check_source.outputs.run_hypothesis)
            && '
            test_hypothesis,
            '
            || ''
          }}
        jobs: ${{ toJSON(needs) }}