mirror of
https://github.com/dlt-hub/dlt.git
synced 2025-12-17 19:31:30 +00:00
cbcff925ba
* add python 3.12 linting * update locked versions to make project installable on py 3.12 * update flake8 * downgrade poetry for all tests relying on python3.8 * drop python 3.8 * enable python3.13 * copy test updates from python3.13 branch * update locked sentry version * pin poetry to 1.8.5 * install ibis outside of poetry * rename to workflows for consistency * switch to published alpha version of dlt-pendulum for python 3.13 * fix images * add note to readme
98 lines
3.1 KiB
YAML
98 lines
3.1 KiB
YAML
# Tests destinations that can run without credentials.
|
|
# i.e. local postgres, duckdb, filesystem (with local fs/memory bucket)
|
|
|
|
name: dest | sqlalchemy mysql and sqlite
|
|
|
|
on:
|
|
pull_request:
|
|
branches:
|
|
- master
|
|
- devel
|
|
workflow_dispatch:
|
|
|
|
concurrency:
|
|
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
|
|
cancel-in-progress: true
|
|
|
|
env:
|
|
# NOTE: this workflow can't use github secrets!
|
|
# DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }}
|
|
|
|
# RUNTIME__SENTRY_DSN: https://6f6f7b6f8e0f458a89be4187603b55fe@o1061158.ingest.sentry.io/4504819859914752
|
|
RUNTIME__LOG_LEVEL: ERROR
|
|
RUNTIME__DLTHUB_TELEMETRY_ENDPOINT: ${{ secrets.RUNTIME__DLTHUB_TELEMETRY_ENDPOINT }}
|
|
ACTIVE_DESTINATIONS: "[\"sqlalchemy\"]"
|
|
ALL_FILESYSTEM_DRIVERS: "[\"memory\", \"file\"]"
|
|
|
|
jobs:
|
|
get_docs_changes:
|
|
name: docs changes
|
|
uses: ./.github/workflows/get_docs_changes.yml
|
|
|
|
run_loader:
|
|
name: dest | sqlalchemy mysql and sqlite
|
|
needs: get_docs_changes
|
|
if: needs.get_docs_changes.outputs.changes_outside_docs == 'true'
|
|
strategy:
|
|
fail-fast: false
|
|
# Run on sqlalchemy 1.4 and 2.0
|
|
matrix:
|
|
sqlalchemy: [1.4, 2]
|
|
defaults:
|
|
run:
|
|
shell: bash
|
|
runs-on: "ubuntu-latest"
|
|
|
|
# Service containers to run with `container-job`
|
|
services:
|
|
# Label used to access the service container
|
|
mysql:
|
|
image: mysql:8
|
|
env:
|
|
MYSQL_ROOT_PASSWORD: root
|
|
MYSQL_DATABASE: dlt_data
|
|
MYSQL_USER: loader
|
|
MYSQL_PASSWORD: loader
|
|
ports:
|
|
- 3306:3306
|
|
# Wait for the service to be ready before completing the job
|
|
options: >-
|
|
--health-cmd="mysqladmin ping -h localhost -u root -proot"
|
|
--health-interval=10s
|
|
--health-timeout=5s
|
|
--health-retries=5
|
|
|
|
steps:
|
|
- name: Check out
|
|
uses: actions/checkout@master
|
|
|
|
- name: Setup Python
|
|
uses: actions/setup-python@v4
|
|
with:
|
|
python-version: "3.10.x"
|
|
|
|
- name: Install Poetry
|
|
uses: snok/install-poetry@v1.3.2
|
|
with:
|
|
virtualenvs-create: true
|
|
virtualenvs-in-project: true
|
|
installer-parallel: true
|
|
version: 1.8.5
|
|
|
|
- name: Load cached venv
|
|
id: cached-poetry-dependencies
|
|
uses: actions/cache@v3
|
|
with:
|
|
path: .venv
|
|
key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}-local-destinations
|
|
|
|
- name: Install dependencies
|
|
run: poetry install --no-interaction -E parquet -E filesystem -E sqlalchemy -E cli --with sentry-sdk --with pipeline && poetry run pip install ibis-framework[duckdb,postgres,bigquery,snowflake,mssql,clickhouse] && poetry run pip install mysqlclient && poetry run pip install "sqlalchemy==${{ matrix.sqlalchemy }}"
|
|
|
|
- name: create secrets.toml
|
|
run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml
|
|
|
|
# always run full suite, also on branches
|
|
- run: poetry run pytest tests/load -x --ignore tests/load/sources
|
|
name: Run tests Linux
|