mirror of
https://github.com/dbt-labs/dbt-core
synced 2025-12-17 19:31:34 +00:00
296 lines
8.8 KiB
YAML
296 lines
8.8 KiB
YAML
# **what?**
|
|
# Runs code quality checks, unit tests, integration tests and
|
|
# verifies python build on all code commited to the repository. This workflow
|
|
# should not require any secrets since it runs for PRs from forked repos. By
|
|
# default, secrets are not passed to workflows running from a forked repos.
|
|
|
|
# **why?**
|
|
# Ensure code for dbt meets a certain quality standard.
|
|
|
|
# **when?**
|
|
# This will run for all PRs, when code is pushed to a release
|
|
# branch, and when manually triggered.
|
|
|
|
name: Tests and Code Checks
|
|
|
|
on:
|
|
push:
|
|
branches:
|
|
- "main"
|
|
- "*.latest"
|
|
- "releases/*"
|
|
pull_request:
|
|
workflow_dispatch:
|
|
|
|
permissions: read-all
|
|
|
|
# will cancel previous workflows triggered by the same event and for the same ref for PRs or same SHA otherwise
|
|
concurrency:
|
|
group: ${{ github.workflow }}-${{ github.event_name }}-${{ contains(github.event_name, 'pull_request') && github.event.pull_request.head.ref || github.sha }}
|
|
cancel-in-progress: true
|
|
|
|
defaults:
|
|
run:
|
|
shell: bash
|
|
|
|
# top-level adjustments can be made here
|
|
env:
|
|
# number of parallel processes to spawn for python integration testing
|
|
PYTHON_INTEGRATION_TEST_WORKERS: 5
|
|
|
|
jobs:
|
|
code-quality:
|
|
name: code-quality
|
|
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 10
|
|
|
|
steps:
|
|
- name: Check out the repository
|
|
uses: actions/checkout@v4
|
|
|
|
- name: Set up Python
|
|
uses: actions/setup-python@v5
|
|
with:
|
|
python-version: '3.8'
|
|
|
|
- name: Install python dependencies
|
|
run: |
|
|
python -m pip install --user --upgrade pip
|
|
python -m pip --version
|
|
make dev
|
|
mypy --version
|
|
dbt --version
|
|
|
|
- name: Run pre-commit hooks
|
|
run: pre-commit run --all-files --show-diff-on-failure
|
|
|
|
unit:
|
|
name: unit test / python ${{ matrix.python-version }}
|
|
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 10
|
|
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
python-version: [ "3.8", "3.9", "3.10", "3.11", "3.12" ]
|
|
|
|
env:
|
|
TOXENV: "unit"
|
|
|
|
steps:
|
|
- name: Check out the repository
|
|
uses: actions/checkout@v4
|
|
|
|
- name: Set up Python ${{ matrix.python-version }}
|
|
uses: actions/setup-python@v5
|
|
with:
|
|
python-version: ${{ matrix.python-version }}
|
|
|
|
- name: Install python dependencies
|
|
run: |
|
|
python -m pip install --user --upgrade pip
|
|
python -m pip --version
|
|
python -m pip install tox
|
|
tox --version
|
|
|
|
- name: Run tox
|
|
run: tox
|
|
|
|
- name: Get current date
|
|
if: always()
|
|
id: date
|
|
run: |
|
|
CURRENT_DATE=$(date +'%Y-%m-%dT%H_%M_%S') # no colons allowed for artifacts
|
|
echo "date=$CURRENT_DATE" >> $GITHUB_OUTPUT
|
|
|
|
- name: Upload Unit Test Coverage to Codecov
|
|
if: ${{ matrix.python-version == '3.11' }}
|
|
uses: codecov/codecov-action@v3
|
|
with:
|
|
token: ${{ secrets.CODECOV_TOKEN }}
|
|
flags: unit
|
|
|
|
integration-metadata:
|
|
name: integration test metadata generation
|
|
runs-on: ubuntu-latest
|
|
outputs:
|
|
split-groups: ${{ steps.generate-split-groups.outputs.split-groups }}
|
|
include: ${{ steps.generate-include.outputs.include }}
|
|
|
|
steps:
|
|
- name: generate split-groups
|
|
id: generate-split-groups
|
|
run: |
|
|
MATRIX_JSON="["
|
|
for B in $(seq 1 ${{ env.PYTHON_INTEGRATION_TEST_WORKERS }}); do
|
|
MATRIX_JSON+=$(sed 's/^/"/;s/$/"/' <<< "${B}")
|
|
done
|
|
MATRIX_JSON="${MATRIX_JSON//\"\"/\", \"}"
|
|
MATRIX_JSON+="]"
|
|
echo "split-groups=${MATRIX_JSON}"
|
|
echo "split-groups=${MATRIX_JSON}" >> $GITHUB_OUTPUT
|
|
|
|
- name: generate include
|
|
id: generate-include
|
|
run: |
|
|
INCLUDE=('"python-version":"3.8","os":"windows-latest"' '"python-version":"3.8","os":"macos-latest"' )
|
|
INCLUDE_GROUPS="["
|
|
for include in ${INCLUDE[@]}; do
|
|
for group in $(seq 1 ${{ env.PYTHON_INTEGRATION_TEST_WORKERS }}); do
|
|
INCLUDE_GROUPS+=$(sed 's/$/, /' <<< "{\"split-group\":\"${group}\",${include}}")
|
|
done
|
|
done
|
|
INCLUDE_GROUPS=$(echo $INCLUDE_GROUPS | sed 's/,*$//g')
|
|
INCLUDE_GROUPS+="]"
|
|
echo "include=${INCLUDE_GROUPS}"
|
|
echo "include=${INCLUDE_GROUPS}" >> $GITHUB_OUTPUT
|
|
|
|
integration:
|
|
name: (${{ matrix.split-group }}) integration test / python ${{ matrix.python-version }} / ${{ matrix.os }}
|
|
|
|
runs-on: ${{ matrix.os }}
|
|
timeout-minutes: 30
|
|
needs:
|
|
- integration-metadata
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
python-version: [ "3.8", "3.9", "3.10", "3.11", "3.12" ]
|
|
os: [ubuntu-20.04]
|
|
split-group: ${{ fromJson(needs.integration-metadata.outputs.split-groups) }}
|
|
include: ${{ fromJson(needs.integration-metadata.outputs.include) }}
|
|
env:
|
|
TOXENV: integration
|
|
DBT_INVOCATION_ENV: github-actions
|
|
DBT_TEST_USER_1: dbt_test_user_1
|
|
DBT_TEST_USER_2: dbt_test_user_2
|
|
DBT_TEST_USER_3: dbt_test_user_3
|
|
DD_CIVISIBILITY_AGENTLESS_ENABLED: true
|
|
DD_API_KEY: ${{ secrets.DATADOG_API_KEY }}
|
|
DD_SITE: datadoghq.com
|
|
DD_ENV: ci
|
|
DD_SERVICE: ${{ github.event.repository.name }}
|
|
|
|
steps:
|
|
- name: Check out the repository
|
|
uses: actions/checkout@v4
|
|
|
|
- name: Set up Python ${{ matrix.python-version }}
|
|
uses: actions/setup-python@v5
|
|
with:
|
|
python-version: ${{ matrix.python-version }}
|
|
|
|
- name: Set up postgres (linux)
|
|
if: runner.os == 'Linux'
|
|
uses: ./.github/actions/setup-postgres-linux
|
|
|
|
- name: Set up postgres (macos)
|
|
if: runner.os == 'macOS'
|
|
uses: ./.github/actions/setup-postgres-macos
|
|
|
|
- name: Set up postgres (windows)
|
|
if: runner.os == 'Windows'
|
|
uses: ./.github/actions/setup-postgres-windows
|
|
|
|
- name: Install python tools
|
|
run: |
|
|
python -m pip install --user --upgrade pip
|
|
python -m pip --version
|
|
python -m pip install tox
|
|
tox --version
|
|
|
|
- name: Run tests
|
|
run: tox -- --ddtrace
|
|
env:
|
|
PYTEST_ADDOPTS: ${{ format('--splits {0} --group {1}', env.PYTHON_INTEGRATION_TEST_WORKERS, matrix.split-group) }}
|
|
|
|
- name: Get current date
|
|
if: always()
|
|
id: date
|
|
run: |
|
|
CURRENT_DATE=$(date +'%Y-%m-%dT%H_%M_%S') # no colons allowed for artifacts
|
|
echo "date=$CURRENT_DATE" >> $GITHUB_OUTPUT
|
|
|
|
- uses: actions/upload-artifact@v3
|
|
if: always()
|
|
with:
|
|
name: logs_${{ matrix.python-version }}_${{ matrix.os }}_${{ steps.date.outputs.date }}
|
|
path: ./logs
|
|
|
|
- name: Upload Integration Test Coverage to Codecov
|
|
if: ${{ matrix.python-version == '3.11' }}
|
|
uses: codecov/codecov-action@v3
|
|
with:
|
|
token: ${{ secrets.CODECOV_TOKEN }}
|
|
flags: integration
|
|
|
|
integration-report:
|
|
if: ${{ always() }}
|
|
name: Integration Test Suite
|
|
runs-on: ubuntu-latest
|
|
needs: integration
|
|
steps:
|
|
- name: "Integration Tests Failed"
|
|
if: ${{ contains(needs.integration.result, 'failure') || contains(needs.integration.result, 'cancelled') }}
|
|
# when this is true the next step won't execute
|
|
run: |
|
|
echo "::notice title='Integration test suite failed'"
|
|
exit 1
|
|
|
|
- name: "Integration Tests Passed"
|
|
run: |
|
|
echo "::notice title='Integration test suite passed'"
|
|
|
|
build:
|
|
name: build packages
|
|
|
|
runs-on: ubuntu-latest
|
|
|
|
steps:
|
|
- name: Check out the repository
|
|
uses: actions/checkout@v4
|
|
|
|
- name: Set up Python
|
|
uses: actions/setup-python@v5
|
|
with:
|
|
python-version: '3.8'
|
|
|
|
- name: Install python dependencies
|
|
run: |
|
|
python -m pip install --user --upgrade pip
|
|
python -m pip install --upgrade setuptools wheel twine check-wheel-contents
|
|
python -m pip --version
|
|
|
|
- name: Build distributions
|
|
run: ./scripts/build-dist.sh
|
|
|
|
- name: Show distributions
|
|
run: ls -lh dist/
|
|
|
|
- name: Check distribution descriptions
|
|
run: |
|
|
twine check dist/*
|
|
|
|
- name: Check wheel contents
|
|
run: |
|
|
check-wheel-contents dist/*.whl --ignore W007,W008
|
|
|
|
- name: Install wheel distributions
|
|
run: |
|
|
find ./dist/*.whl -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/
|
|
|
|
- name: Check wheel distributions
|
|
run: |
|
|
dbt --version
|
|
|
|
- name: Install source distributions
|
|
# ignore dbt-1.0.0, which intentionally raises an error when installed from source
|
|
run: |
|
|
find ./dist/dbt-[a-z]*.gz -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/
|
|
|
|
- name: Check source distributions
|
|
run: |
|
|
dbt --version
|