dlt/.github/workflows/test_common.yml

name: common | common

on:
  workflow_call:


env:
  RUNTIME__LOG_LEVEL: ERROR

  # we need the secrets only for the rest_api_pipeline tests which are in tests/sources
  # so we inject them only at the end
  SOURCES__GITHUB__ACCESS_TOKEN: ${{ secrets.GITHUB_TOKEN }}
  # and also for the github_api_pipeline tests
  SOURCES__GITHUB_API_PIPELINE__ACCESS_TOKEN: ${{ secrets.GITHUB_TOKEN }}

jobs:
  run_common:
    name: test
    strategy:
      fail-fast: false
      matrix:
        include:

          # macos tests
          - os: macos-latest
            python-version: "3.11"
            shell: bash

          # linux tests
          - os: ubuntu-latest
            python-version: "3.9"
            shell: bash
          - os: ubuntu-latest
            python-version: "3.10"
            shell: bash
          - os: ubuntu-latest
            python-version: "3.11"
            shell: bash
          - os: ubuntu-latest
            python-version: "3.12"
            shell: bash
          - os: ubuntu-latest
            python-version: "3.13"
            shell: bash
          
          # windows tests
          - os: windows-latest
            python-version: "3.11"
            shell: cmd
            pytest_args: '-m "not forked"'
          - os: windows-latest
            python-version: "3.13"
            shell: cmd
            pytest_args: '-m "not forked"'

    defaults:
      run:
        shell: ${{ matrix.shell }}
    runs-on: ${{ matrix.os }}

    steps:
      - name: Check out
        uses: actions/checkout@master
      - name: Setup Python
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}

      - name: Install tzdata on windows
        run: |
          cd %USERPROFILE%
          curl https://data.iana.org/time-zones/releases/tzdata2021e.tar.gz --output tzdata.tar.gz
          mkdir tzdata
          tar --extract --file tzdata.tar.gz --directory tzdata
          mkdir %USERPROFILE%\Downloads\tzdata
          copy tzdata %USERPROFILE%\Downloads\tzdata
          curl https://raw.githubusercontent.com/unicode-org/cldr/master/common/supplemental/windowsZones.xml --output %USERPROFILE%\Downloads\tzdata\windowsZones.xml
        if: runner.os == 'Windows'
      - name: Install uv
        uses: astral-sh/setup-uv@v6
        with:
          python-version: ${{ matrix.python-version }}
          activate-environment: true

      - name: Install dependencies
        run: uv sync --group sentry-sdk
    
      - name: Run common tests with minimum dependencies
        run: |
          pytest tests/common tests/normalize tests/reflection tests/plugins tests/load/test_dummy_client.py tests/extract/test_extract.py tests/extract/test_sources.py tests/pipeline/test_pipeline_state.py ${{ matrix.pytest_args }}
  
      - name: Install duckdb dependencies
        run: uv sync --extra duckdb --group sentry-sdk

      - name: Run pipeline smoke tests with minimum deps
        run: |
          pytest tests/pipeline/test_pipeline.py tests/pipeline/test_import_export_schema.py ${{ matrix.pytest_args }}

      - name: Install pyarrow
        run: uv sync --extra duckdb --extra cli --extra parquet --group sentry-sdk

      - name: Run pipeline tests with pyarrow but no pandas installed
        run: |
          pytest tests/pipeline/test_pipeline_extra.py -k arrow ${{ matrix.pytest_args }}

      - name: Install pipeline and sources dependencies
        run: uv sync --extra duckdb --extra cli --extra parquet --extra deltalake --extra sql_database --group sentry-sdk --group pipeline --group sources

      - name: Run extract and pipeline tests
        run: |
          pytest tests/extract tests/pipeline tests/libs tests/cli/common tests/destinations tests/sources tests/transformations ${{ matrix.pytest_args }}

      # here we upgrade sql alchemy to 2 an run the sql_database tests again
      - name: Upgrade sql alchemy
        run: uv run pip install sqlalchemy==2.0.32

      - name: Run extract and pipeline tests 
        run: |
          pytest tests/sources/sql_database

      # test marimo app, does not work with python 3.13
      - name: Install dlt with duckdb and studio
        run: uv sync --extra duckdb --group sentry-sdk --group pipeline --group sources --group ibis --group marimo
        if: matrix.python-version != '3.13'

      - name: Install playwright
        run: playwright install
        if: matrix.python-version != '3.13'

      # Run marimo studio unit tests
      - name: Run marimo studio unit tests
        run: |
          pytest tests/helpers/studio
        if: matrix.python-version != '3.13'

      # Run marimo e2e tests (does not pass with python 3.9, does not pass on windows (playwright does not work somehow), does not pass on python 3.13 (ibis not available))
      - name: Run marimo e2e
        run: |
          marimo run --headless dlt/helpers/studio/app.py -- -- --pipelines_dir _storage/.dlt/pipelines/ --with_test_identifiers true & pytest --browser chromium tests/e2e
        if: matrix.python-version != '3.13' && matrix.python-version != '3.9' && matrix.os != 'windows-latest'



  matrix_job_required_check:
    name: common | common tests
    needs: run_common
    runs-on: ubuntu-latest
    if: always()
    steps:
      - name: Check matrix job results
        if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled')
        run: |
          echo "One or more matrix job tests failed or were cancelled. You may need to re-run them." && exit 1