mirror of
https://github.com/dlt-hub/dlt.git
synced 2025-12-17 19:31:30 +00:00
4a431d60ed
* renames pipeline to workspace dashboard * refreshes intro * review changes * sidebar, references, dataset.table( cleanup
131 lines
4.7 KiB
YAML
131 lines
4.7 KiB
YAML
name: common | common
|
|
|
|
on:
|
|
workflow_call:
|
|
|
|
|
|
env:
|
|
RUNTIME__LOG_LEVEL: ERROR
|
|
|
|
# we need the secrets only for the rest_api_pipeline tests which are in tests/sources
|
|
# so we inject them only at the end
|
|
SOURCES__GITHUB__ACCESS_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
# and also for the github_api_pipeline tests
|
|
SOURCES__GITHUB_API_PIPELINE__ACCESS_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
|
|
jobs:
|
|
run_common:
|
|
name: test
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
include:
|
|
|
|
# macos tests
|
|
- os: "macos-14"
|
|
python-version: "3.12"
|
|
shell: bash
|
|
|
|
# linux tests
|
|
- os: ubuntu-latest
|
|
python-version: "3.10"
|
|
shell: bash
|
|
- os: ubuntu-latest
|
|
python-version: "3.11"
|
|
shell: bash
|
|
- os: ubuntu-latest
|
|
python-version: "3.12"
|
|
shell: bash
|
|
- os: ubuntu-latest
|
|
python-version: "3.13"
|
|
shell: bash
|
|
|
|
# linux test with minimal dependencies
|
|
- os: ubuntu-latest
|
|
python-version: "3.11"
|
|
shell: bash
|
|
uv_sync_args: '--resolution lowest-direct' # could also be 'direct'
|
|
|
|
# mac test with newest available allowed packages (will update lockfile, should not be committed if run locally)
|
|
# TODO: revert to linux
|
|
- os: "macos-14"
|
|
python-version: "3.11"
|
|
shell: bash
|
|
uv_sync_args: '--upgrade' # could also be 'direct'
|
|
|
|
|
|
# windows tests
|
|
- os: windows-latest
|
|
python-version: "3.12"
|
|
shell: cmd
|
|
pytest_args: '-m "not forked"'
|
|
|
|
defaults:
|
|
run:
|
|
shell: ${{ matrix.shell }}
|
|
runs-on: ${{ matrix.os }}
|
|
|
|
steps:
|
|
- name: Check out
|
|
uses: actions/checkout@master
|
|
- name: Setup Python
|
|
uses: actions/setup-python@v5
|
|
with:
|
|
python-version: ${{ matrix.python-version }}
|
|
|
|
- name: Install tzdata on windows
|
|
run: |
|
|
cd %USERPROFILE%
|
|
curl https://data.iana.org/time-zones/releases/tzdata2021e.tar.gz --output tzdata.tar.gz
|
|
mkdir tzdata
|
|
tar --extract --file tzdata.tar.gz --directory tzdata
|
|
mkdir %USERPROFILE%\Downloads\tzdata
|
|
copy tzdata %USERPROFILE%\Downloads\tzdata
|
|
curl https://raw.githubusercontent.com/unicode-org/cldr/master/common/supplemental/windowsZones.xml --output %USERPROFILE%\Downloads\tzdata\windowsZones.xml
|
|
if: runner.os == 'Windows'
|
|
- name: Install uv
|
|
uses: astral-sh/setup-uv@v6
|
|
with:
|
|
python-version: ${{ matrix.python-version }}
|
|
activate-environment: true
|
|
|
|
# test workspace dashboard app, does not work with python 3.13
|
|
- name: Install dlt with duckdb and dashboard
|
|
# note: this also tests the workspace extra installation
|
|
run: uv sync ${{ matrix.uv_sync_args }} --extra duckdb --extra workspace --group sentry-sdk --group pipeline --group sources --group dashboard-tests
|
|
if: matrix.python-version != '3.14.0-beta.4'
|
|
|
|
- name: Install playwright & deps
|
|
run: playwright install && playwright install-deps
|
|
if: matrix.python-version != '3.14.0-beta.4'
|
|
|
|
# Run workspace dashboard unit tests
|
|
- name: Run workspace dashboard unit tests
|
|
run: |
|
|
pytest tests/workspace/helpers/dashboard
|
|
|
|
# Run workspace dashboard e2e tests (does not pass with python 3.9
|
|
- name: Run dashboard e2e
|
|
run: |
|
|
marimo run --headless dlt/_workspace/helpers/dashboard/dlt_dashboard.py -- -- --pipelines-dir _storage/.dlt/pipelines/ --with_test_identifiers true & pytest --browser chromium tests/e2e
|
|
if: matrix.python-version != '3.9' && matrix.python-version != '3.14.0-beta.4' && matrix.os != 'windows-latest'
|
|
|
|
# note that this test will pass only when running from cmd shell (_storage\.dlt\pipelines\ must stay)
|
|
- name: Run dashboard e2e windows
|
|
run: |
|
|
start marimo run --headless dlt/_workspace/helpers/dashboard/dlt_dashboard.py -- -- --pipelines-dir _storage\.dlt\pipelines\ --with_test_identifiers true
|
|
timeout /t 6 /nobreak
|
|
pytest --browser chromium tests/e2e
|
|
if: matrix.python-version != '3.9' && matrix.python-version != '3.14.0-beta.4' && matrix.os == 'windows-latest'
|
|
|
|
matrix_job_required_check:
|
|
name: common | common tests
|
|
needs: run_common
|
|
runs-on: ubuntu-latest
|
|
if: always()
|
|
steps:
|
|
- name: Check matrix job results
|
|
if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled')
|
|
run: |
|
|
echo "One or more matrix job tests failed or were cancelled. You may need to re-run them." && exit 1
|