mirror of
https://github.com/dbt-labs/dbt-core
synced 2025-12-17 19:31:34 +00:00
384 lines
13 KiB
YAML
384 lines
13 KiB
YAML
# **what?**
|
|
# Runs code quality checks, unit tests, integration tests and
|
|
# verifies python build on all code commited to the repository. This workflow
|
|
# should not require any secrets since it runs for PRs from forked repos. By
|
|
# default, secrets are not passed to workflows running from a forked repos.
|
|
|
|
# **why?**
|
|
# Ensure code for dbt meets a certain quality standard.
|
|
|
|
# **when?**
|
|
# This will run for all PRs, when code is pushed to a release
|
|
# branch, and when manually triggered.
|
|
|
|
name: Tests and Code Checks
|
|
|
|
on:
|
|
push:
|
|
branches:
|
|
- "main"
|
|
- "*.latest"
|
|
- "releases/*"
|
|
pull_request:
|
|
merge_group:
|
|
types: [checks_requested]
|
|
workflow_dispatch:
|
|
|
|
permissions: read-all
|
|
|
|
# will cancel previous workflows triggered by the same event and for the same ref for PRs or same SHA otherwise
|
|
concurrency:
|
|
group: ${{ github.workflow }}-${{ github.event_name }}-${{ contains(github.event_name, 'pull_request') && github.event.pull_request.head.ref || github.sha }}
|
|
cancel-in-progress: true
|
|
|
|
defaults:
|
|
run:
|
|
shell: bash
|
|
|
|
# top-level adjustments can be made here
|
|
env:
|
|
# number of parallel processes to spawn for python integration testing
|
|
PYTHON_INTEGRATION_TEST_WORKERS: 5
|
|
|
|
jobs:
|
|
code-quality:
|
|
name: code-quality
|
|
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 10
|
|
|
|
steps:
|
|
- name: Check out the repository
|
|
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # actions/checkout@v4
|
|
|
|
- name: Set up Python
|
|
uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # actions/setup-python@v6
|
|
with:
|
|
python-version: "3.10"
|
|
|
|
- name: Install python dependencies
|
|
run: |
|
|
python -m pip install --user --upgrade pip
|
|
python -m pip --version
|
|
python -m pip install hatch
|
|
cd core
|
|
hatch run setup
|
|
|
|
- name: Verify dbt installation
|
|
run: |
|
|
cd core
|
|
hatch run dbt --version
|
|
|
|
- name: Run pre-commit hooks
|
|
run: |
|
|
cd core
|
|
hatch run code-quality
|
|
|
|
unit:
|
|
name: "unit test / python ${{ matrix.python-version }}"
|
|
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 10
|
|
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
python-version: ["3.10", "3.11", "3.12", "3.13"]
|
|
|
|
steps:
|
|
- name: Check out the repository
|
|
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # actions/checkout@v4
|
|
|
|
- name: Set up Python ${{ matrix.python-version }}
|
|
uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # actions/setup-python@v6
|
|
with:
|
|
python-version: ${{ matrix.python-version }}
|
|
|
|
- name: Install python dependencies
|
|
run: |
|
|
python -m pip install --user --upgrade pip
|
|
python -m pip --version
|
|
python -m pip install hatch
|
|
hatch --version
|
|
|
|
- name: Run unit tests
|
|
uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # nick-fields/retry@v3
|
|
with:
|
|
timeout_minutes: 10
|
|
max_attempts: 3
|
|
command: cd core && hatch run ci:unit-tests
|
|
|
|
- name: Get current date
|
|
if: always()
|
|
id: date
|
|
run: |
|
|
CURRENT_DATE=$(date +'%Y-%m-%dT%H_%M_%S') # no colons allowed for artifacts
|
|
echo "date=$CURRENT_DATE" >> $GITHUB_OUTPUT
|
|
|
|
- name: Upload Unit Test Coverage to Codecov
|
|
if: ${{ matrix.python-version == '3.11' }}
|
|
uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # codecov/codecov-action@v5
|
|
with:
|
|
token: ${{ secrets.CODECOV_TOKEN }}
|
|
flags: unit
|
|
fail_ci_if_error: false
|
|
|
|
integration-metadata:
|
|
name: integration test metadata generation
|
|
runs-on: ubuntu-latest
|
|
outputs:
|
|
split-groups: ${{ steps.generate-split-groups.outputs.split-groups }}
|
|
include: ${{ steps.generate-include.outputs.include }}
|
|
|
|
steps:
|
|
- name: generate split-groups
|
|
id: generate-split-groups
|
|
run: |
|
|
MATRIX_JSON="["
|
|
for B in $(seq 1 ${{ env.PYTHON_INTEGRATION_TEST_WORKERS }}); do
|
|
MATRIX_JSON+=$(sed 's/^/"/;s/$/"/' <<< "${B}")
|
|
done
|
|
MATRIX_JSON="${MATRIX_JSON//\"\"/\", \"}"
|
|
MATRIX_JSON+="]"
|
|
echo "split-groups=${MATRIX_JSON}"
|
|
echo "split-groups=${MATRIX_JSON}" >> $GITHUB_OUTPUT
|
|
|
|
- name: generate include
|
|
id: generate-include
|
|
run: |
|
|
INCLUDE=('"python-version":"3.10","os":"windows-latest"' '"python-version":"3.10","os":"macos-14"' )
|
|
INCLUDE_GROUPS="["
|
|
for include in ${INCLUDE[@]}; do
|
|
for group in $(seq 1 ${{ env.PYTHON_INTEGRATION_TEST_WORKERS }}); do
|
|
INCLUDE_GROUPS+=$(sed 's/$/, /' <<< "{\"split-group\":\"${group}\",${include}}")
|
|
done
|
|
done
|
|
INCLUDE_GROUPS=$(echo $INCLUDE_GROUPS | sed 's/,*$//g')
|
|
INCLUDE_GROUPS+="]"
|
|
echo "include=${INCLUDE_GROUPS}"
|
|
echo "include=${INCLUDE_GROUPS}" >> $GITHUB_OUTPUT
|
|
|
|
integration-postgres:
|
|
name: "(${{ matrix.split-group }}) integration test / python ${{ matrix.python-version }} / ${{ matrix.os }}"
|
|
|
|
runs-on: ${{ matrix.os }}
|
|
timeout-minutes: 30
|
|
needs:
|
|
- integration-metadata
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
python-version: ["3.10", "3.11", "3.12", "3.13"]
|
|
os: ["ubuntu-latest"]
|
|
split-group: ${{ fromJson(needs.integration-metadata.outputs.split-groups) }}
|
|
env:
|
|
DBT_INVOCATION_ENV: github-actions
|
|
DBT_TEST_USER_1: dbt_test_user_1
|
|
DBT_TEST_USER_2: dbt_test_user_2
|
|
DBT_TEST_USER_3: dbt_test_user_3
|
|
DD_CIVISIBILITY_AGENTLESS_ENABLED: true
|
|
DD_API_KEY: ${{ secrets.DATADOG_API_KEY }}
|
|
DD_SITE: datadoghq.com
|
|
DD_ENV: ci
|
|
DD_SERVICE: ${{ github.event.repository.name }}
|
|
|
|
services:
|
|
# Label used to access the service container
|
|
postgres:
|
|
# Docker Hub image
|
|
image: postgres
|
|
# Provide the password for postgres
|
|
env:
|
|
POSTGRES_PASSWORD: password
|
|
POSTGRES_USER: postgres
|
|
# Set health checks to wait until postgres has started
|
|
options: >-
|
|
--health-cmd pg_isready
|
|
--health-interval 10s
|
|
--health-timeout 5s
|
|
--health-retries 5
|
|
ports:
|
|
- 5432:5432
|
|
|
|
steps:
|
|
- name: Check out the repository
|
|
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # actions/checkout@v4
|
|
|
|
- name: Set up Python ${{ matrix.python-version }}
|
|
uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # actions/setup-python@v6
|
|
with:
|
|
python-version: ${{ matrix.python-version }}
|
|
|
|
- name: Run postgres setup script
|
|
run: |
|
|
./scripts/setup_db.sh
|
|
env:
|
|
PGHOST: localhost
|
|
PGPORT: 5432
|
|
PGPASSWORD: password
|
|
|
|
- name: Install python tools
|
|
run: |
|
|
python -m pip install --user --upgrade pip
|
|
python -m pip --version
|
|
python -m pip install hatch
|
|
hatch --version
|
|
|
|
- name: Run integration tests
|
|
uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # nick-fields/retry@v3
|
|
with:
|
|
timeout_minutes: 30
|
|
max_attempts: 3
|
|
shell: bash
|
|
command: cd core && hatch run ci:integration-tests -- --ddtrace --splits ${{ env.PYTHON_INTEGRATION_TEST_WORKERS }} --group ${{ matrix.split-group }}
|
|
|
|
- name: Get current date
|
|
if: always()
|
|
id: date
|
|
run: |
|
|
CURRENT_DATE=$(date +'%Y-%m-%dT%H_%M_%S') # no colons allowed for artifacts
|
|
echo "date=$CURRENT_DATE" >> $GITHUB_OUTPUT
|
|
|
|
- uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # actions/upload-artifact@v4
|
|
if: always()
|
|
with:
|
|
name: logs_${{ matrix.python-version }}_${{ matrix.os }}_${{ matrix.split-group }}_${{ steps.date.outputs.date }}
|
|
path: ./logs
|
|
|
|
- name: Upload Integration Test Coverage to Codecov
|
|
if: ${{ matrix.python-version == '3.11' }}
|
|
uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # codecov/codecov-action@v5
|
|
with:
|
|
token: ${{ secrets.CODECOV_TOKEN }}
|
|
flags: integration
|
|
fail_ci_if_error: false
|
|
|
|
integration-mac-windows:
|
|
name: (${{ matrix.split-group }}) integration test / python ${{ matrix.python-version }} / ${{ matrix.os }}
|
|
|
|
runs-on: ${{ matrix.os }}
|
|
timeout-minutes: 30
|
|
needs:
|
|
- integration-metadata
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
# already includes split group and runs mac + windows
|
|
include: ${{ fromJson(needs.integration-metadata.outputs.include) }}
|
|
env:
|
|
DBT_INVOCATION_ENV: github-actions
|
|
DBT_TEST_USER_1: dbt_test_user_1
|
|
DBT_TEST_USER_2: dbt_test_user_2
|
|
DBT_TEST_USER_3: dbt_test_user_3
|
|
DD_CIVISIBILITY_AGENTLESS_ENABLED: true
|
|
DD_API_KEY: ${{ secrets.DATADOG_API_KEY }}
|
|
DD_SITE: datadoghq.com
|
|
DD_ENV: ci
|
|
DD_SERVICE: ${{ github.event.repository.name }}
|
|
|
|
steps:
|
|
- name: Check out the repository
|
|
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # actions/checkout@v4
|
|
|
|
- name: Set up Python ${{ matrix.python-version }}
|
|
uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # actions/setup-python@v6
|
|
with:
|
|
python-version: ${{ matrix.python-version }}
|
|
|
|
- name: Set up postgres (macos)
|
|
if: runner.os == 'macOS'
|
|
|
|
uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # nick-fields/retry@v3
|
|
with:
|
|
timeout_minutes: 10
|
|
max_attempts: 3
|
|
command: ./scripts/setup_db.sh
|
|
|
|
- name: Set up postgres (windows)
|
|
if: runner.os == 'Windows'
|
|
uses: ./.github/actions/setup-postgres-windows
|
|
|
|
- name: Install python tools
|
|
run: |
|
|
python -m pip install --user --upgrade pip
|
|
python -m pip --version
|
|
python -m pip install hatch
|
|
hatch --version
|
|
|
|
- name: Run integration tests
|
|
uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # nick-fields/retry@v3
|
|
with:
|
|
timeout_minutes: 30
|
|
max_attempts: 3
|
|
shell: bash
|
|
command: cd core && hatch run ci:integration-tests -- --ddtrace --splits ${{ env.PYTHON_INTEGRATION_TEST_WORKERS }} --group ${{ matrix.split-group }}
|
|
|
|
- name: Get current date
|
|
if: always()
|
|
id: date
|
|
run: |
|
|
CURRENT_DATE=$(date +'%Y-%m-%dT%H_%M_%S') # no colons allowed for artifacts
|
|
echo "date=$CURRENT_DATE" >> $GITHUB_OUTPUT
|
|
|
|
- uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # actions/upload-artifact@v4
|
|
if: always()
|
|
with:
|
|
name: logs_${{ matrix.python-version }}_${{ matrix.os }}_${{ matrix.split-group }}_${{ steps.date.outputs.date }}
|
|
path: ./logs
|
|
|
|
- name: Upload Integration Test Coverage to Codecov
|
|
if: ${{ matrix.python-version == '3.11' }}
|
|
uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # codecov/codecov-action@v5
|
|
with:
|
|
token: ${{ secrets.CODECOV_TOKEN }}
|
|
flags: integration
|
|
fail_ci_if_error: false
|
|
|
|
integration-report:
|
|
if: ${{ always() }}
|
|
name: Integration Test Suite
|
|
runs-on: ubuntu-latest
|
|
needs: [integration-mac-windows, integration-postgres]
|
|
steps:
|
|
- name: "Integration Tests Failed"
|
|
if: ${{ contains(needs.integration-mac-windows.result, 'failure') || contains(needs.integration-mac-windows.result, 'cancelled') || contains(needs.integration-postgres.result, 'failure') || contains(needs.integration-postgres.result, 'cancelled') }}
|
|
# when this is true the next step won't execute
|
|
run: |
|
|
echo "::notice title='Integration test suite failed'"
|
|
exit 1
|
|
|
|
- name: "Integration Tests Passed"
|
|
run: |
|
|
echo "::notice title='Integration test suite passed'"
|
|
|
|
build:
|
|
name: build packages
|
|
|
|
runs-on: ubuntu-latest
|
|
|
|
steps:
|
|
- name: Check out the repository
|
|
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # actions/checkout@v4
|
|
|
|
- name: Set up Python
|
|
uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # actions/setup-python@v6
|
|
with:
|
|
python-version: "3.10"
|
|
|
|
- name: Install python dependencies
|
|
run: |
|
|
python -m pip install --user --upgrade pip
|
|
python -m pip install --upgrade hatch twine check-wheel-contents
|
|
python -m pip --version
|
|
|
|
- name: Build distributions
|
|
run: ./scripts/build-dist.sh
|
|
|
|
- name: Show distributions
|
|
run: ls -lh dist/
|
|
|
|
- name: Check and verify distributions
|
|
run: |
|
|
cd core
|
|
hatch run build:check-all
|