mirror of
https://github.com/dlt-hub/dlt.git
synced 2025-12-17 19:31:30 +00:00
cbcff925ba
* add python 3.12 linting * update locked versions to make project installable on py 3.12 * update flake8 * downgrade poetry for all tests relying on python3.8 * drop python 3.8 * enable python3.13 * copy test updates from python3.13 branch * update locked sentry version * pin poetry to 1.8.5 * install ibis outside of poetry * rename to workflows for consistency * switch to published alpha version of dlt-pendulum for python 3.13 * fix images * add note to readme
111 lines
3.3 KiB
YAML
111 lines
3.3 KiB
YAML
|
|
name: docs | snippets & examples
|
|
|
|
on:
|
|
pull_request:
|
|
branches:
|
|
- master
|
|
- devel
|
|
workflow_dispatch:
|
|
|
|
concurrency:
|
|
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
|
|
cancel-in-progress: true
|
|
|
|
env:
|
|
DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }}
|
|
|
|
# RUNTIME__SENTRY_DSN: https://6f6f7b6f8e0f458a89be4187603b55fe@o1061158.ingest.sentry.io/4504819859914752
|
|
RUNTIME__LOG_LEVEL: ERROR
|
|
RUNTIME__DLTHUB_TELEMETRY_ENDPOINT: ${{ secrets.RUNTIME__DLTHUB_TELEMETRY_ENDPOINT }}
|
|
|
|
# Slack hook for chess in production example
|
|
RUNTIME__SLACK_INCOMING_HOOK: ${{ secrets.RUNTIME__SLACK_INCOMING_HOOK }}
|
|
# Path to local qdrant database
|
|
DESTINATION__QDRANT__CREDENTIALS__PATH: zendesk.qdb
|
|
# detect if the workflow is executed in a repo fork
|
|
IS_FORK: ${{ github.event.pull_request.head.repo.fork }}
|
|
|
|
jobs:
|
|
|
|
run_lint:
|
|
name: docs | snippets & examples lint and test
|
|
runs-on: ubuntu-latest
|
|
# Do not run on forks, unless allowed, secrets are used here
|
|
if: ${{ !github.event.pull_request.head.repo.fork || contains(github.event.pull_request.labels.*.name, 'ci from fork')}}
|
|
|
|
# Service containers to run with `container-job`
|
|
services:
|
|
# Label used to access the service container
|
|
postgres:
|
|
# Docker Hub image
|
|
image: postgres
|
|
# Provide the password for postgres
|
|
env:
|
|
POSTGRES_DB: dlt_data
|
|
POSTGRES_USER: loader
|
|
POSTGRES_PASSWORD: loader
|
|
ports:
|
|
- 5432:5432
|
|
# Set health checks to wait until postgres has started
|
|
options: >-
|
|
--health-cmd pg_isready
|
|
--health-interval 10s
|
|
--health-timeout 5s
|
|
--health-retries 5
|
|
|
|
steps:
|
|
|
|
- name: Check out
|
|
uses: actions/checkout@master
|
|
|
|
- name: Start weaviate
|
|
run: docker compose -f "tests/load/weaviate/docker-compose.yml" up -d
|
|
|
|
- name: Setup Python
|
|
uses: actions/setup-python@v4
|
|
with:
|
|
python-version: "3.10.x"
|
|
|
|
- name: Setup node 20
|
|
uses: actions/setup-node@v4
|
|
with:
|
|
node-version: 20
|
|
|
|
- name: Install Poetry
|
|
uses: snok/install-poetry@v1.3.2
|
|
with:
|
|
virtualenvs-create: true
|
|
virtualenvs-in-project: true
|
|
installer-parallel: true
|
|
version: 1.8.5
|
|
|
|
- name: Load cached venv
|
|
id: cached-poetry-dependencies
|
|
uses: actions/cache@v3
|
|
with:
|
|
path: .venv
|
|
key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}
|
|
|
|
- name: run docs preprocessor
|
|
run: make preprocess-docs
|
|
|
|
- name: Install dependencies
|
|
# if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
|
|
run: poetry install --no-interaction -E duckdb -E weaviate -E parquet -E qdrant -E bigquery -E postgres -E lancedb --with docs,sentry-sdk --without airflow -E s3
|
|
|
|
- name: create secrets.toml for examples
|
|
run: pwd && echo "$DLT_SECRETS_TOML" > docs/examples/.dlt/secrets.toml
|
|
|
|
- name: create secrets.toml for snippets
|
|
run: pwd && echo "$DLT_SECRETS_TOML" > docs/website/docs/.dlt/secrets.toml
|
|
|
|
- name: Run linter and tests on examples
|
|
run: make lint-and-test-examples
|
|
|
|
- name: Run linter and tests on snippets
|
|
run: make lint-and-test-snippets
|
|
|
|
|
|
|