mirror of
https://github.com/dbt-labs/dbt-snowflake
synced 2025-12-17 19:31:31 +00:00
ADAP-1115: Move to hatch and pyproject.toml (#1250)
* move config files into pyproject.toml and hatch.toml * update workflow files to use hatch.toml * move test matrix into the workflow * fix pytest config in pyproject.toml
This commit is contained in:
@@ -1,35 +0,0 @@
|
||||
[bumpversion]
|
||||
current_version = 1.10.0a1
|
||||
parse = (?P<major>[\d]+) # major version number
|
||||
\.(?P<minor>[\d]+) # minor version number
|
||||
\.(?P<patch>[\d]+) # patch version number
|
||||
(?P<prerelease> # optional pre-release - ex: a1, b2, rc25
|
||||
(?P<prekind>a|b|rc) # pre-release type
|
||||
(?P<num>[\d]+) # pre-release version number
|
||||
)?
|
||||
( # optional nightly release indicator
|
||||
\.(?P<nightly>dev[0-9]+) # ex: .dev02142023
|
||||
)? # expected matches: `1.15.0`, `1.5.0a11`, `1.5.0a1.dev123`, `1.5.0.dev123457`, expected failures: `1`, `1.5`, `1.5.2-a1`, `text1.5.0`
|
||||
serialize =
|
||||
{major}.{minor}.{patch}{prekind}{num}.{nightly}
|
||||
{major}.{minor}.{patch}.{nightly}
|
||||
{major}.{minor}.{patch}{prekind}{num}
|
||||
{major}.{minor}.{patch}
|
||||
commit = False
|
||||
tag = False
|
||||
|
||||
[bumpversion:part:prekind]
|
||||
first_value = a
|
||||
optional_value = final
|
||||
values =
|
||||
a
|
||||
b
|
||||
rc
|
||||
final
|
||||
|
||||
[bumpversion:part:num]
|
||||
first_value = 1
|
||||
|
||||
[bumpversion:part:nightly]
|
||||
|
||||
[bumpversion:file:dbt/adapters/snowflake/__version__.py]
|
||||
6
.changes/unreleased/Under the Hood-20241117-184430.yaml
Normal file
6
.changes/unreleased/Under the Hood-20241117-184430.yaml
Normal file
@@ -0,0 +1,6 @@
|
||||
kind: Under the Hood
|
||||
body: Move from setup.py to pyproject.toml and to hatch as a dev tool
|
||||
time: 2024-11-17T18:44:30.85288-05:00
|
||||
custom:
|
||||
Author: mikealfare
|
||||
Issue: "1250"
|
||||
95
.github/scripts/integration-test-matrix.js
vendored
95
.github/scripts/integration-test-matrix.js
vendored
@@ -1,95 +0,0 @@
|
||||
module.exports = ({ context }) => {
|
||||
const defaultPythonVersion = "3.9";
|
||||
const supportedPythonVersions = ["3.9", "3.10", "3.11", "3.12"];
|
||||
const supportedAdapters = ["snowflake"];
|
||||
|
||||
// if PR, generate matrix based on files changed and PR labels
|
||||
if (context.eventName.includes("pull_request")) {
|
||||
// `changes` is a list of adapter names that have related
|
||||
// file changes in the PR
|
||||
// ex: ['postgres', 'snowflake']
|
||||
const changes = JSON.parse(process.env.CHANGES);
|
||||
const labels = context.payload.pull_request.labels.map(({ name }) => name);
|
||||
console.log("labels", labels);
|
||||
console.log("changes", changes);
|
||||
const testAllLabel = labels.includes("test all");
|
||||
const include = [];
|
||||
|
||||
for (const adapter of supportedAdapters) {
|
||||
if (
|
||||
changes.includes(adapter) ||
|
||||
testAllLabel ||
|
||||
labels.includes(`test ${adapter}`)
|
||||
) {
|
||||
for (const pythonVersion of supportedPythonVersions) {
|
||||
if (
|
||||
pythonVersion === defaultPythonVersion ||
|
||||
labels.includes(`test python${pythonVersion}`) ||
|
||||
testAllLabel
|
||||
) {
|
||||
// always run tests on ubuntu by default
|
||||
include.push({
|
||||
os: "ubuntu-latest",
|
||||
adapter,
|
||||
"python-version": pythonVersion,
|
||||
});
|
||||
|
||||
if (labels.includes("test windows") || testAllLabel) {
|
||||
include.push({
|
||||
os: "windows-latest",
|
||||
adapter,
|
||||
"python-version": pythonVersion,
|
||||
});
|
||||
}
|
||||
|
||||
if (labels.includes("test macos") || testAllLabel) {
|
||||
include.push({
|
||||
os: "macos-14",
|
||||
adapter,
|
||||
"python-version": pythonVersion,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
console.log("matrix", { include });
|
||||
|
||||
return {
|
||||
include,
|
||||
};
|
||||
}
|
||||
// if not PR, generate matrix of python version, adapter, and operating
|
||||
// system to run integration tests on
|
||||
|
||||
const include = [];
|
||||
// run for all adapters and python versions on ubuntu
|
||||
for (const adapter of supportedAdapters) {
|
||||
for (const pythonVersion of supportedPythonVersions) {
|
||||
include.push({
|
||||
os: 'ubuntu-latest',
|
||||
adapter: adapter,
|
||||
"python-version": pythonVersion,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// additionally include runs for all adapters, on macos and windows,
|
||||
// but only for the default python version
|
||||
for (const adapter of supportedAdapters) {
|
||||
for (const operatingSystem of ["windows-latest", "macos-14"]) {
|
||||
include.push({
|
||||
os: operatingSystem,
|
||||
adapter: adapter,
|
||||
"python-version": defaultPythonVersion,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
console.log("matrix", { include });
|
||||
|
||||
return {
|
||||
include,
|
||||
};
|
||||
};
|
||||
15
.github/scripts/update_dependencies.sh
vendored
15
.github/scripts/update_dependencies.sh
vendored
@@ -1,15 +0,0 @@
|
||||
#!/bin/bash -e
|
||||
set -e
|
||||
|
||||
git_branch=$1
|
||||
target_req_file="dev-requirements.txt"
|
||||
core_req_sed_pattern="s|dbt-core.git.*#egg=dbt-core|dbt-core.git@${git_branch}#egg=dbt-core|g"
|
||||
tests_req_sed_pattern="s|dbt-core.git.*#egg=dbt-tests|dbt-core.git@${git_branch}#egg=dbt-tests|g"
|
||||
if [[ "$OSTYPE" == darwin* ]]; then
|
||||
# mac ships with a different version of sed that requires a delimiter arg
|
||||
sed -i "" "$core_req_sed_pattern" $target_req_file
|
||||
sed -i "" "$tests_req_sed_pattern" $target_req_file
|
||||
else
|
||||
sed -i "$core_req_sed_pattern" $target_req_file
|
||||
sed -i "$tests_req_sed_pattern" $target_req_file
|
||||
fi
|
||||
@@ -5,17 +5,20 @@ set -e
|
||||
dbt_adapters_branch=$1
|
||||
dbt_core_branch=$2
|
||||
dbt_common_branch=$3
|
||||
target_req_file="dev-requirements.txt"
|
||||
core_req_sed_pattern="s|dbt-core.git.*#egg=dbt-core|dbt-core.git@${dbt_core_branch}#egg=dbt-core|g"
|
||||
target_req_file="hatch.toml"
|
||||
core_req_sed_pattern="s|dbt-core.git.*#subdirectory=core|dbt-core.git@${dbt_core_branch}#subdirectory=core|g"
|
||||
adapters_req_sed_pattern="s|dbt-adapters.git|dbt-adapters.git@${dbt_adapters_branch}|g"
|
||||
tests_req_sed_pattern="s|dbt-adapters.git.*#subdirectory=dbt-tests-adapter|dbt-adapters.git@${dbt_adapters_branch}#subdirectory=dbt-tests-adapter|g"
|
||||
common_req_sed_pattern="s|dbt-common.git|dbt-common.git@${dbt_common_branch}|g"
|
||||
if [[ "$OSTYPE" == darwin* ]]; then
|
||||
# mac ships with a different version of sed that requires a delimiter arg
|
||||
sed -i "" "$adapters_req_sed_pattern" $target_req_file
|
||||
sed -i "" "$tests_req_sed_pattern" $target_req_file
|
||||
sed -i "" "$core_req_sed_pattern" $target_req_file
|
||||
sed -i "" "$common_req_sed_pattern" $target_req_file
|
||||
else
|
||||
sed -i "$adapters_req_sed_pattern" $target_req_file
|
||||
sed -i "$tests_req_sed_pattern" $target_req_file
|
||||
sed -i "$core_req_sed_pattern" $target_req_file
|
||||
sed -i "$common_req_sed_pattern" $target_req_file
|
||||
fi
|
||||
|
||||
105
.github/workflows/integration.yml
vendored
105
.github/workflows/integration.yml
vendored
@@ -76,85 +76,31 @@ defaults:
|
||||
shell: bash
|
||||
|
||||
jobs:
|
||||
# generate test metadata about what files changed and the testing matrix to use
|
||||
test-metadata:
|
||||
# run if not a PR from a forked repository or has a label to mark as safe to test
|
||||
if: >-
|
||||
github.event_name != 'pull_request_target' ||
|
||||
github.event.pull_request.head.repo.full_name == github.repository ||
|
||||
contains(github.event.pull_request.labels.*.name, 'ok to test')
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
matrix: ${{ steps.generate-matrix.outputs.result }}
|
||||
|
||||
steps:
|
||||
- name: Check out the repository (non-PR)
|
||||
if: github.event_name != 'pull_request_target'
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Check out the repository (PR)
|
||||
if: github.event_name == 'pull_request_target'
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
ref: ${{ github.event.pull_request.head.sha }}
|
||||
|
||||
- name: Check if relevant files changed
|
||||
if: github.event_name == 'pull_request_target'
|
||||
# https://github.com/marketplace/actions/paths-changes-filter
|
||||
# For each filter, it sets output variable named by the filter to the text:
|
||||
# 'true' - if any of changed files matches any of filter rules
|
||||
# 'false' - if none of changed files matches any of filter rules
|
||||
# also, returns:
|
||||
# `changes` - JSON array with names of all filters matching any of the changed files
|
||||
uses: dorny/paths-filter@v3
|
||||
id: get-changes
|
||||
with:
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
filters: |
|
||||
snowflake:
|
||||
- '.github/**/*.yml'
|
||||
- '.github/**/*.sh'
|
||||
- 'dbt/**'
|
||||
- 'tests/**'
|
||||
- 'dev-requirements.txt'
|
||||
- '*.py'
|
||||
- name: Generate integration test matrix
|
||||
id: generate-matrix
|
||||
uses: actions/github-script@v7
|
||||
env:
|
||||
CHANGES: ${{ steps.get-changes.outputs.changes }}
|
||||
with:
|
||||
script: |
|
||||
const script = require('./.github/scripts/integration-test-matrix.js')
|
||||
const matrix = script({ context })
|
||||
console.log(matrix)
|
||||
return matrix
|
||||
test:
|
||||
name: ${{ matrix.adapter }} / python ${{ matrix.python-version }} / ${{ matrix.os }}
|
||||
name: ${{ matrix.os }} / python ${{ matrix.python-version }}
|
||||
|
||||
# run if not a PR from a forked repository or has a label to mark as safe to test
|
||||
# also checks that the matrix generated is not empty
|
||||
if: >-
|
||||
needs.test-metadata.outputs.matrix &&
|
||||
fromJSON( needs.test-metadata.outputs.matrix ).include[0] &&
|
||||
(
|
||||
github.event_name != 'pull_request_target' ||
|
||||
github.event.pull_request.head.repo.full_name == github.repository ||
|
||||
contains(github.event.pull_request.labels.*.name, 'ok to test')
|
||||
)
|
||||
runs-on: ${{ matrix.os }}
|
||||
needs: test-metadata
|
||||
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix: ${{ fromJSON(needs.test-metadata.outputs.matrix) }}
|
||||
matrix:
|
||||
os: [ubuntu-22.04]
|
||||
python-version: ["3.9", "3.10", "3.11", "3.12"]
|
||||
include:
|
||||
- os: macos-14
|
||||
python-version: "3.9"
|
||||
- os: windows-2022
|
||||
python-version: "3.9"
|
||||
|
||||
env:
|
||||
TOXENV: integration-${{ matrix.adapter }}
|
||||
PYTEST_ADDOPTS: "-v --color=yes -n4 --csv integration_results.csv"
|
||||
DBT_INVOCATION_ENV: github-actions
|
||||
DD_CIVISIBILITY_AGENTLESS_ENABLED: true
|
||||
DD_API_KEY: ${{ secrets.DATADOG_API_KEY }}
|
||||
@@ -190,24 +136,19 @@ jobs:
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
|
||||
- name: Update Adapters and Core branches (update dev_requirements.txt)
|
||||
- name: Update Adapters and Core branches (update hatch.toml)
|
||||
if: ${{ github.event_name == 'workflow_dispatch' }}
|
||||
run: |
|
||||
./.github/scripts/update_dev_dependency_branches.sh \
|
||||
${{ inputs.dbt_adapters_branch }} \
|
||||
${{ inputs.dbt_core_branch }} \
|
||||
${{ inputs.dbt_common_branch }}
|
||||
cat dev-requirements.txt
|
||||
cat hatch.toml
|
||||
|
||||
- name: Install python dependencies
|
||||
run: |
|
||||
python -m pip install --user --upgrade pip
|
||||
python -m pip install tox
|
||||
python -m pip --version
|
||||
tox --version
|
||||
uses: pypa/hatch@install
|
||||
|
||||
- name: Run tox (snowflake)
|
||||
if: matrix.adapter == 'snowflake'
|
||||
- run: hatch run integration-tests tests/functional --ddtrace
|
||||
env:
|
||||
SNOWFLAKE_TEST_ACCOUNT: ${{ secrets.SNOWFLAKE_TEST_ACCOUNT }}
|
||||
SNOWFLAKE_TEST_PASSWORD: ${{ secrets.SNOWFLAKE_TEST_PASSWORD }}
|
||||
@@ -226,26 +167,6 @@ jobs:
|
||||
DBT_TEST_USER_1: dbt_test_role_1
|
||||
DBT_TEST_USER_2: dbt_test_role_2
|
||||
DBT_TEST_USER_3: dbt_test_role_3
|
||||
run: tox -- --ddtrace
|
||||
|
||||
- name: Get current date
|
||||
if: always()
|
||||
id: date
|
||||
run: echo "date=$(date +'%Y-%m-%dT%H_%M_%S')" >> $GITHUB_OUTPUT #no colons allowed for artifacts
|
||||
|
||||
- uses: actions/upload-artifact@v4
|
||||
if: always()
|
||||
with:
|
||||
name: logs_${{ matrix.python-version }}_${{ matrix.os }}_${{ matrix.adapter }}-${{ steps.date.outputs.date }}
|
||||
path: ./logs
|
||||
overwrite: true
|
||||
|
||||
- uses: actions/upload-artifact@v4
|
||||
if: always()
|
||||
with:
|
||||
name: integration_results_${{ matrix.python-version }}_${{ matrix.os }}_${{ matrix.adapter }}-${{ steps.date.outputs.date }}.csv
|
||||
path: integration_results.csv
|
||||
overwrite: true
|
||||
|
||||
require-label-comment:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
56
.github/workflows/main.yml
vendored
56
.github/workflows/main.yml
vendored
@@ -52,16 +52,8 @@ jobs:
|
||||
with:
|
||||
python-version: '3.9'
|
||||
|
||||
- name: Install python dependencies
|
||||
run: |
|
||||
python -m pip install --user --upgrade pip
|
||||
python -m pip install -r dev-requirements.txt
|
||||
python -m pip --version
|
||||
pre-commit --version
|
||||
dbt --version
|
||||
|
||||
- name: Run pre-commit hooks
|
||||
run: pre-commit run --all-files --show-diff-on-failure
|
||||
uses: pre-commit/action@v3.0.1
|
||||
|
||||
unit:
|
||||
name: unit test / python ${{ matrix.python-version }}
|
||||
@@ -73,10 +65,6 @@ jobs:
|
||||
matrix:
|
||||
python-version: ['3.9', '3.10', '3.11', '3.12']
|
||||
|
||||
env:
|
||||
TOXENV: "unit"
|
||||
PYTEST_ADDOPTS: "-v --color=yes --csv unit_results.csv"
|
||||
|
||||
steps:
|
||||
- name: Check out the repository
|
||||
uses: actions/checkout@v4
|
||||
@@ -89,26 +77,9 @@ jobs:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
|
||||
- name: Install python dependencies
|
||||
run: |
|
||||
python -m pip install --user --upgrade pip
|
||||
python -m pip install tox
|
||||
python -m pip --version
|
||||
tox --version
|
||||
uses: pypa/hatch@install
|
||||
|
||||
- name: Run tox
|
||||
run: tox
|
||||
|
||||
- name: Get current date
|
||||
if: always()
|
||||
id: date
|
||||
run: echo "date=$(date +'%Y-%m-%dT%H_%M_%S')" >> $GITHUB_OUTPUT #no colons allowed for artifacts
|
||||
|
||||
- uses: actions/upload-artifact@v4
|
||||
if: always()
|
||||
with:
|
||||
name: unit_results_${{ matrix.python-version }}-${{ steps.date.outputs.date }}.csv
|
||||
path: unit_results.csv
|
||||
overwrite: true
|
||||
- run: hatch run unit-tests
|
||||
|
||||
build:
|
||||
name: build packages
|
||||
@@ -129,25 +100,16 @@ jobs:
|
||||
with:
|
||||
python-version: '3.9'
|
||||
|
||||
- name: Install python dependencies
|
||||
run: |
|
||||
python -m pip install --user --upgrade pip
|
||||
python -m pip install --upgrade setuptools wheel twine check-wheel-contents
|
||||
python -m pip --version
|
||||
- uses: pypa/hatch@install
|
||||
|
||||
- name: Build distributions
|
||||
run: ./scripts/build-dist.sh
|
||||
run: hatch build
|
||||
|
||||
- name: Show distributions
|
||||
run: ls -lh dist/
|
||||
|
||||
- name: Check distribution descriptions
|
||||
run: |
|
||||
twine check dist/*
|
||||
|
||||
- name: Check wheel contents
|
||||
run: |
|
||||
check-wheel-contents dist/*.whl --ignore W007,W008
|
||||
run: hatch run build:check-all
|
||||
|
||||
- name: Check if this is an alpha version
|
||||
id: check-is-alpha
|
||||
@@ -184,12 +146,6 @@ jobs:
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
|
||||
- name: Install python dependencies
|
||||
run: |
|
||||
python -m pip install --user --upgrade pip
|
||||
python -m pip install --upgrade wheel
|
||||
python -m pip --version
|
||||
|
||||
- uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: dist
|
||||
|
||||
14
.github/workflows/nightly-release.yml
vendored
14
.github/workflows/nightly-release.yml
vendored
@@ -50,11 +50,15 @@ jobs:
|
||||
commit_sha=$(git rev-parse HEAD)
|
||||
echo "release_commit=$commit_sha" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: "Get Current Version Number"
|
||||
id: version-number-sources
|
||||
run: |
|
||||
current_version=`awk -F"current_version = " '{print $2}' .bumpversion.cfg | tr '\n' ' '`
|
||||
echo "current_version=$current_version" >> $GITHUB_OUTPUT
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.9'
|
||||
|
||||
- uses: pypa/hatch@install
|
||||
|
||||
- id: version-number-sources
|
||||
run: echo "current_version=$(hatch version)" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: "Audit Version And Parse Into Parts"
|
||||
id: semver
|
||||
|
||||
108
.github/workflows/release.yml
vendored
108
.github/workflows/release.yml
vendored
@@ -119,86 +119,70 @@ defaults:
|
||||
shell: bash
|
||||
|
||||
jobs:
|
||||
log-inputs:
|
||||
name: Log Inputs
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: "[DEBUG] Print Variables"
|
||||
run: |
|
||||
echo The last commit sha in the release: ${{ inputs.sha }}
|
||||
echo The branch to release from: ${{ inputs.target_branch }}
|
||||
echo The release version number: ${{ inputs.version_number }}
|
||||
echo Build script path: ${{ inputs.build_script_path }}
|
||||
echo Environment setup script path: ${{ inputs.env_setup_script_path }}
|
||||
echo AWS S3 bucket name: ${{ inputs.s3_bucket_name }}
|
||||
echo Package test command: ${{ inputs.package_test_command }}
|
||||
echo Test run: ${{ inputs.test_run }}
|
||||
echo Nightly release: ${{ inputs.nightly_release }}
|
||||
echo Only Docker: ${{ inputs.only_docker }}
|
||||
|
||||
bump-version-generate-changelog:
|
||||
name: Bump package version, Generate changelog
|
||||
uses: dbt-labs/dbt-release/.github/workflows/release-prep.yml@main
|
||||
release-prep:
|
||||
name: "Release prep: generate changelog, bump version"
|
||||
uses: ./.github/workflows/release_prep_hatch.yml
|
||||
with:
|
||||
sha: ${{ inputs.sha }}
|
||||
version_number: ${{ inputs.version_number }}
|
||||
target_branch: ${{ inputs.target_branch }}
|
||||
env_setup_script_path: ${{ inputs.env_setup_script_path }}
|
||||
test_run: ${{ inputs.test_run }}
|
||||
nightly_release: ${{ inputs.nightly_release }}
|
||||
branch: ${{ inputs.branch }}
|
||||
version: ${{ inputs.version }}
|
||||
deploy-to: ${{ inputs.deploy-to }}
|
||||
secrets: inherit
|
||||
|
||||
log-outputs-bump-version-generate-changelog:
|
||||
name: "[Log output] Bump package version, Generate changelog"
|
||||
if: ${{ !failure() && !cancelled() && !inputs.only_docker }}
|
||||
needs: [bump-version-generate-changelog]
|
||||
build-release:
|
||||
name: "Build release"
|
||||
needs: release-prep
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
archive-name: ${{ steps.archive.outputs.name }}
|
||||
steps:
|
||||
- name: Print variables
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ needs.release-prep.outputs.release-branch }}
|
||||
persist-credentials: false
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: ${{ inputs.python_version }}
|
||||
- uses: pypa/hatch@install
|
||||
- id: archive
|
||||
run: |
|
||||
echo Final SHA : ${{ needs.bump-version-generate-changelog.outputs.final_sha }}
|
||||
echo Changelog path: ${{ needs.bump-version-generate-changelog.outputs.changelog_path }}
|
||||
|
||||
build-test-package:
|
||||
name: Build, Test, Package
|
||||
if: ${{ !failure() && !cancelled() && !inputs.only_docker }}
|
||||
needs: [bump-version-generate-changelog]
|
||||
uses: dbt-labs/dbt-release/.github/workflows/build.yml@main
|
||||
with:
|
||||
sha: ${{ needs.bump-version-generate-changelog.outputs.final_sha }}
|
||||
version_number: ${{ inputs.version_number }}
|
||||
changelog_path: ${{ needs.bump-version-generate-changelog.outputs.changelog_path }}
|
||||
build_script_path: ${{ inputs.build_script_path }}
|
||||
s3_bucket_name: ${{ inputs.s3_bucket_name }}
|
||||
package_test_command: ${{ inputs.package_test_command }}
|
||||
test_run: ${{ inputs.test_run }}
|
||||
nightly_release: ${{ inputs.nightly_release }}
|
||||
secrets:
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
||||
archive_name=${{ github.event.repository.name }}-${{ inputs.version }}-${{ inputs.deploy-to }}
|
||||
echo "name=$archive_name" >> $GITHUB_OUTPUT
|
||||
- run: hatch build && hatch run build:check-all
|
||||
- uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: ${{ steps.archive.outputs.name }}
|
||||
path: dist/
|
||||
retention-days: 3
|
||||
|
||||
github-release:
|
||||
name: GitHub Release
|
||||
if: ${{ !failure() && !cancelled() && !inputs.only_docker }}
|
||||
needs: [bump-version-generate-changelog, build-test-package]
|
||||
uses: dbt-labs/dbt-release/.github/workflows/github-release.yml@main
|
||||
needs: [build-release, release-prep]
|
||||
uses: dbt-labs/dbt-adapters/.github/workflows/github-release.yml@main
|
||||
with:
|
||||
sha: ${{ needs.bump-version-generate-changelog.outputs.final_sha }}
|
||||
version_number: ${{ inputs.version_number }}
|
||||
changelog_path: ${{ needs.bump-version-generate-changelog.outputs.changelog_path }}
|
||||
test_run: ${{ inputs.test_run }}
|
||||
archive_name: ${{ needs.build-release.outputs.archive-name }}
|
||||
|
||||
pypi-release:
|
||||
name: PyPI Release
|
||||
if: ${{ !failure() && !cancelled() && !inputs.only_docker }}
|
||||
needs: [bump-version-generate-changelog, build-test-package]
|
||||
uses: dbt-labs/dbt-release/.github/workflows/pypi-release.yml@main
|
||||
with:
|
||||
version_number: ${{ inputs.version_number }}
|
||||
test_run: ${{ inputs.test_run }}
|
||||
secrets:
|
||||
PYPI_API_TOKEN: ${{ secrets.PYPI_API_TOKEN }}
|
||||
TEST_PYPI_API_TOKEN: ${{ secrets.TEST_PYPI_API_TOKEN }}
|
||||
needs: build-release
|
||||
runs-on: ubuntu-latest
|
||||
environment:
|
||||
name: ${{ inputs.deploy-to }}
|
||||
url: ${{ vars.PYPI_PROJECT_URL }}
|
||||
permissions:
|
||||
# this permission is required for trusted publishing
|
||||
# see https://github.com/marketplace/actions/pypi-publish
|
||||
id-token: write
|
||||
steps:
|
||||
- uses: dbt-labs/dbt-adapters/.github/actions/publish-pypi@main
|
||||
with:
|
||||
repository-url: ${{ vars.PYPI_REPOSITORY_URL }}
|
||||
archive-name: ${{ needs.build-release.outputs.archive-name }}
|
||||
|
||||
docker-release:
|
||||
name: "Docker Release"
|
||||
@@ -206,7 +190,7 @@ jobs:
|
||||
# what we need to release but draft releases don't actually tag the commit so it
|
||||
# finds nothing to release
|
||||
if: ${{ !failure() && !cancelled() && (!inputs.test_run || inputs.only_docker) }}
|
||||
needs: [bump-version-generate-changelog, build-test-package, github-release]
|
||||
needs: github-release
|
||||
permissions:
|
||||
packages: write
|
||||
uses: dbt-labs/dbt-release/.github/workflows/release-docker.yml@main
|
||||
|
||||
440
.github/workflows/release_prep_hatch.yml
vendored
Normal file
440
.github/workflows/release_prep_hatch.yml
vendored
Normal file
@@ -0,0 +1,440 @@
|
||||
# **what?**
|
||||
# Perform the version bump, generate the changelog and run tests.
|
||||
#
|
||||
# Inputs:
|
||||
# branch: The branch that we will release from
|
||||
# version: The release version number (i.e. 1.0.0b1, 1.2.3rc2, 1.0.0)
|
||||
# deploy-to: If we are deploying to prod or test, if test then release from branch
|
||||
# is-nightly-release: Identifier that this is nightly release
|
||||
#
|
||||
# Outputs:
|
||||
# release-sha: The sha that will actually be released. This can differ from the
|
||||
# input sha if adding a version bump and/or changelog
|
||||
# changelog-path: Path to the changelog file (ex .changes/1.2.3-rc1.md)
|
||||
#
|
||||
# Branching strategy:
|
||||
# - During execution workflow execution the temp branch will be generated.
|
||||
# - For normal runs the temp branch will be removed once changes were merged to target branch;
|
||||
# - For test runs we will keep temp branch and will use it for release;
|
||||
# Naming strategy:
|
||||
# - For normal runs: prep-release/${{ inputs.deploy-to}}/${{ inputs.version }}_$GITHUB_RUN_ID
|
||||
# - For nightly releases: prep-release/nightly-release/${{ inputs.version }}_$GITHUB_RUN_ID
|
||||
#
|
||||
# **why?**
|
||||
# Reusable and consistent GitHub release process.
|
||||
#
|
||||
# **when?**
|
||||
# Call when ready to kick off a build and release
|
||||
#
|
||||
# Validation Checks
|
||||
#
|
||||
# 1. Bump the version if it has not been bumped
|
||||
# 2. Generate the changelog (via changie) if there is no markdown file for this version
|
||||
name: "Release prep"
|
||||
run-name: "Release prep: Generate changelog and bump to ${{ inputs.version }} for release to ${{ inputs.deploy-to }}"
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
branch:
|
||||
description: "The branch to release from"
|
||||
type: string
|
||||
default: "main"
|
||||
version:
|
||||
description: "The version to release"
|
||||
required: true
|
||||
type: string
|
||||
deploy-to:
|
||||
description: "Deploy to test or prod"
|
||||
type: string
|
||||
default: "prod"
|
||||
is-nightly-release:
|
||||
description: "Identify if this is a nightly release"
|
||||
type: boolean
|
||||
default: false
|
||||
outputs:
|
||||
release-branch:
|
||||
description: "The branch to be released from"
|
||||
value: ${{ jobs.release.outputs.branch }}
|
||||
release-sha:
|
||||
description: "The SHA to be released"
|
||||
value: ${{ jobs.release.outputs.sha }}
|
||||
changelog-path:
|
||||
description: "The path to the changelog from the repo root for this version, e.g. .changes/1.8.0-b1.md"
|
||||
value: ${{ jobs.release-inputs.outputs.changelog-path }}
|
||||
secrets:
|
||||
FISHTOWN_BOT_PAT:
|
||||
description: "Token to commit/merge changes into branches"
|
||||
required: true
|
||||
IT_TEAM_MEMBERSHIP:
|
||||
description: "Token that can view org level teams"
|
||||
required: true
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
|
||||
env:
|
||||
PYTHON_TARGET_VERSION: 3.12
|
||||
NOTIFICATION_PREFIX: "[Release Prep]"
|
||||
|
||||
jobs:
|
||||
release-inputs:
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
changelog-path: ${{ steps.changelog.outputs.path }}
|
||||
changelog-exists: ${{ steps.changelog.outputs.exists }}
|
||||
base-version: ${{ steps.semver.outputs.base-version }}
|
||||
pre-release: ${{ steps.semver.outputs.pre-release }}
|
||||
is-pre-release: ${{ steps.semver.outputs.is-pre-release }}
|
||||
version-is-current: ${{ steps.version.outputs.is-current }}
|
||||
|
||||
steps:
|
||||
- name: "[DEBUG] Log inputs"
|
||||
run: |
|
||||
# WORKFLOW INPUTS
|
||||
echo Branch: ${{ inputs.branch }}
|
||||
echo Release version: ${{ inputs.version }}
|
||||
echo Deploy to: ${{ inputs.deploy-to }}
|
||||
echo Nightly release: ${{ inputs.is-nightly-release }}
|
||||
# ENVIRONMENT VARIABLES
|
||||
echo Python version: ${{ env.PYTHON_TARGET_VERSION }}
|
||||
echo Notification prefix: ${{ env.NOTIFICATION_PREFIX }}
|
||||
|
||||
- name: "Checkout ${{ github.event.repository.name }}@${{ inputs.branch }}"
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ inputs.branch }}
|
||||
|
||||
- name: "Setup `hatch`"
|
||||
uses: dbt-labs/dbt-adapters/.github/actions/setup-hatch@main
|
||||
|
||||
- name: "Parse input version"
|
||||
id: semver
|
||||
uses: dbt-labs/actions/parse-semver@v1.1.1
|
||||
with:
|
||||
version: ${{ inputs.version }}
|
||||
|
||||
- name: "Audit version"
|
||||
id: version
|
||||
run: |
|
||||
is_current=false
|
||||
current_version=$(hatch version)
|
||||
if test "$current_version" = "${{ inputs.version }}"
|
||||
then
|
||||
is_current=true
|
||||
fi
|
||||
echo "is-current=$is_current" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: "[INFO] Skip version bump"
|
||||
if: steps.version.outputs.is-current == 'true'
|
||||
run: |
|
||||
title="Skip version bump"
|
||||
message="The version matches the input version ${{ inputs.version }}, skipping version bump"
|
||||
echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message"
|
||||
|
||||
- name: "Audit changelog"
|
||||
id: changelog
|
||||
run: |
|
||||
path=".changes/"
|
||||
if [[ ${{ steps.semver.outputs.is-pre-release }} -eq 1 ]]
|
||||
then
|
||||
path+="${{ steps.semver.outputs.base-version }}-${{ steps.semver.outputs.pre-release }}.md"
|
||||
else
|
||||
path+="${{ steps.semver.outputs.base-version }}.md"
|
||||
fi
|
||||
echo "path=$path" >> $GITHUB_OUTPUT
|
||||
|
||||
does_exist=false
|
||||
if test -f $path
|
||||
then
|
||||
does_exist=true
|
||||
fi
|
||||
echo "exists=$does_exist">> $GITHUB_OUTPUT
|
||||
|
||||
- name: "[INFO] Skip changelog generation"
|
||||
if: steps.changelog.outputs.exists == 'true'
|
||||
run: |
|
||||
title="Skip changelog generation"
|
||||
message="A changelog already exists at ${{ steps.changelog.outputs.path }}, skipping generating changelog"
|
||||
echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message"
|
||||
|
||||
release-branch:
|
||||
runs-on: ubuntu-latest
|
||||
needs: release-inputs
|
||||
if: |
|
||||
needs.release-inputs.outputs.changelog-exists == 'false' ||
|
||||
needs.release-inputs.outputs.version-is-current == 'false'
|
||||
outputs:
|
||||
name: ${{ steps.release-branch.outputs.name }}
|
||||
|
||||
steps:
|
||||
- name: "Checkout ${{ github.event.repository.name }}@${{ inputs.branch }}"
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ inputs.branch }}
|
||||
|
||||
- name: "Set release branch"
|
||||
id: release-branch
|
||||
run: |
|
||||
name="prep-release/"
|
||||
if [[ ${{ inputs.is-nightly-release }} == true ]]
|
||||
then
|
||||
name+="nightly-release/"
|
||||
else
|
||||
name+="${{ inputs.deploy-to }}/"
|
||||
fi
|
||||
name+="${{ inputs.version }}_$GITHUB_RUN_ID"
|
||||
echo "name=$name" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: "Create release branch ${{ steps.release-branch.outputs.name }}"
|
||||
run: |
|
||||
git checkout -b ${{ steps.release-branch.outputs.name }}
|
||||
git push -u origin ${{ steps.release-branch.outputs.name }}
|
||||
|
||||
- name: "[INFO] Create release branch"
|
||||
run: |
|
||||
title="Create release branch"
|
||||
message="Create release branch: ${{ steps.release-branch.outputs.name }}"
|
||||
echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message"
|
||||
|
||||
core-team:
|
||||
if: needs.release-inputs.outputs.changelog-exists == 'false'
|
||||
needs: release-inputs
|
||||
uses: dbt-labs/actions/.github/workflows/determine-team-membership.yml@main
|
||||
with:
|
||||
github_team: "core-group"
|
||||
secrets: inherit
|
||||
|
||||
generate-changelog:
|
||||
runs-on: ubuntu-latest
|
||||
if: needs.release-inputs.outputs.changelog-exists == 'false'
|
||||
# only runs if we need to make changes, determined by not skipping release-branch
|
||||
needs:
|
||||
- release-inputs
|
||||
- release-branch
|
||||
- core-team
|
||||
|
||||
steps:
|
||||
- name: "Checkout ${{ github.event.repository.name }}@${{ needs.release-branch.outputs.name }}"
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ needs.release-branch.outputs.name }}
|
||||
|
||||
- name: "Setup `hatch`"
|
||||
uses: dbt-labs/dbt-adapters/.github/actions/setup-hatch@main
|
||||
|
||||
- name: "Install `changie`"
|
||||
run: |
|
||||
brew tap miniscruff/changie https://github.com/miniscruff/changie
|
||||
brew install changie
|
||||
|
||||
- name: "Generate changelog at ${{ needs.release-inputs.outputs.changelog-path }}"
|
||||
run: |
|
||||
if [[ ${{ needs.release-inputs.outputs.is-pre-release }} -eq 1 ]]
|
||||
then
|
||||
changie batch ${{ needs.release-inputs.outputs.base-version }} \
|
||||
--move-dir '${{ needs.release-inputs.outputs.base-version }}' \
|
||||
--prerelease ${{ needs.release-inputs.outputs.pre-release }}
|
||||
elif [[ -d ".changes/${{ needs.release-inputs.outputs.base-version }}" ]]
|
||||
then
|
||||
changie batch ${{ needs.release-inputs.outputs.base-version }} \
|
||||
--include '${{ needs.release-inputs.outputs.base-version }}' \
|
||||
--remove-prereleases
|
||||
else # releasing a final patch with no pre-releases
|
||||
changie batch ${{ needs.release-inputs.outputs.base-version }}
|
||||
fi
|
||||
changie merge
|
||||
env:
|
||||
CHANGIE_CORE_TEAM: ${{ needs.core-team.outputs.team_membership }}
|
||||
|
||||
- name: "Remove trailing whitespace and missing new lines"
|
||||
# this step will fail on whitespace errors but also correct them
|
||||
continue-on-error: true
|
||||
run: hatch run code-quality
|
||||
|
||||
- name: "Commit & push changes"
|
||||
run: |
|
||||
git config user.name "$USER"
|
||||
git config user.email "$EMAIL"
|
||||
git pull
|
||||
git add .
|
||||
git commit -m "$COMMIT_MESSAGE"
|
||||
git push
|
||||
env:
|
||||
USER: "GitHub Build Bot"
|
||||
EMAIL: "buildbot@fishtownanalytics.com"
|
||||
COMMIT_MESSAGE: "Generate changelog at ${{ needs.release-inputs.outputs.changelog-path }}"
|
||||
|
||||
- name: "[INFO] Generated changelog at ${{ needs.release-inputs.outputs.changelog-path }}"
|
||||
run: |
|
||||
title="Changelog generation"
|
||||
if [[ -f ${{ needs.release-inputs.outputs.changelog-path }} ]]
|
||||
then
|
||||
message="Generated changelog file successfully"
|
||||
echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message"
|
||||
else
|
||||
message="Failed to generate changelog file"
|
||||
echo "::error title=${{ env.NOTIFICATION_PREFIX }}: $title::$message"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
bump-version:
|
||||
runs-on: ubuntu-latest
|
||||
if: needs.release-inputs.outputs.version-is-current == 'false'
|
||||
# only runs if we need to make changes, determined by not skipping release-branch
|
||||
needs:
|
||||
- release-inputs
|
||||
- release-branch
|
||||
- generate-changelog
|
||||
|
||||
steps:
|
||||
- name: "Checkout ${{ github.event.repository.name }}@${{ needs.release-branch.outputs.name }}"
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ needs.release-branch.outputs.name }}
|
||||
|
||||
- name: "Setup `hatch`"
|
||||
uses: dbt-labs/dbt-adapters/.github/actions/setup-hatch@main
|
||||
|
||||
- name: "Bump version to ${{ inputs.version }}"
|
||||
run: hatch version ${{ inputs.version }}
|
||||
|
||||
- name: "Commit & push changes"
|
||||
run: |
|
||||
git config user.name "$USER"
|
||||
git config user.email "$EMAIL"
|
||||
git pull
|
||||
git add .
|
||||
git commit -m "$COMMIT_MESSAGE"
|
||||
git push
|
||||
env:
|
||||
USER: "GitHub Build Bot"
|
||||
EMAIL: "buildbot@fishtownanalytics.com"
|
||||
COMMIT_MESSAGE: "Bump version to ${{ inputs.version }}"
|
||||
|
||||
- name: "[INFO] Bumped version to ${{ inputs.version }}"
|
||||
run: |
|
||||
title="Version bump"
|
||||
message="Bumped version to ${{ inputs.version }}"
|
||||
echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message"
|
||||
|
||||
unit-tests:
|
||||
runs-on: ubuntu-latest
|
||||
# only run unit tests if we created a release branch and already bumped the version and generated the changelog
|
||||
if: |
|
||||
!failure() && !cancelled() &&
|
||||
needs.release-branch.outputs.name != ''
|
||||
needs:
|
||||
- release-branch
|
||||
- generate-changelog
|
||||
- bump-version
|
||||
|
||||
steps:
|
||||
- name: "Checkout ${{ github.event.repository.name }}@${{ needs.release-branch.outputs.name }}"
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ needs.release-branch.outputs.name }}
|
||||
|
||||
- name: "Setup `hatch`"
|
||||
uses: dbt-labs/dbt-adapters/.github/actions/setup-hatch@main
|
||||
|
||||
- name: "Run unit tests"
|
||||
run: hatch run unit-tests
|
||||
|
||||
integration-tests:
|
||||
runs-on: ubuntu-latest
|
||||
# only run integration tests if we created a release branch and already bumped the version and generated the changelog
|
||||
if: |
|
||||
!failure() && !cancelled() &&
|
||||
needs.release-branch.outputs.name != ''
|
||||
needs:
|
||||
- release-branch
|
||||
- generate-changelog
|
||||
- bump-version
|
||||
|
||||
steps:
|
||||
- name: "Checkout ${{ github.event.repository.name }}@${{ needs.release-branch.outputs.name }}"
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ needs.release-branch.outputs.name }}\
|
||||
|
||||
- name: "Set up `hatch`"
|
||||
uses: dbt-labs/dbt-adapters/.github/actions/setup-hatch@main
|
||||
|
||||
- name: "Run integration tests"
|
||||
run: hatch run integration-tests
|
||||
|
||||
merge-release-branch:
|
||||
runs-on: ubuntu-latest
|
||||
needs:
|
||||
- unit-tests
|
||||
- integration-tests
|
||||
- release-branch
|
||||
- release-inputs
|
||||
if: |
|
||||
!failure() && !cancelled() &&
|
||||
needs.release-branch.result == 'success' &&
|
||||
inputs.deploy-to == 'prod'
|
||||
|
||||
steps:
|
||||
- name: "Checkout ${{ github.event.repository.name }}"
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: "Merge changes into ${{ inputs.branch }}"
|
||||
uses: everlytic/branch-merge@1.1.5
|
||||
with:
|
||||
source_ref: ${{ needs.release-branch.outputs.name }}
|
||||
target_branch: ${{ inputs.branch }}
|
||||
github_token: ${{ secrets.FISHTOWN_BOT_PAT }}
|
||||
commit_message_template: "[Automated] Merged {source_ref} into target {target_branch} during release process"
|
||||
|
||||
- name: "[INFO] Merge changes into ${{ inputs.branch }}"
|
||||
run: |
|
||||
title="Merge changes"
|
||||
message="Merge ${{ needs.release-branch.outputs.name }} into ${{ inputs.branch }}"
|
||||
echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message"
|
||||
|
||||
release:
|
||||
runs-on: ubuntu-latest
|
||||
needs:
|
||||
- release-branch
|
||||
- merge-release-branch
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
|
||||
# Get the SHA that will be released.
|
||||
# If the changelog already exists and the version was already current on the input branch, then release from there.
|
||||
# Otherwise, we generated a changelog and/or did the version bump in this workflow and there is a
|
||||
# new sha to use from the merge we just did. Grab that here instead.
|
||||
outputs:
|
||||
branch: ${{ steps.branch.outputs.name }}
|
||||
sha: ${{ steps.sha.outputs.sha }}
|
||||
|
||||
steps:
|
||||
- name: "Set release branch"
|
||||
id: branch
|
||||
# If a release branch was created and not merged, use the release branch
|
||||
# Otherwise, use the input branch because either nothing was done, or the changes were merged back in
|
||||
run: |
|
||||
if [[ ${{ needs.release-branch.result == 'success' }} && ${{ needs.merge-release-branch.result == 'skipped' }} ]]; then
|
||||
branch="${{ needs.release-branch.outputs.name }}"
|
||||
else
|
||||
branch="${{ inputs.branch }}"
|
||||
fi
|
||||
echo "name=$branch" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: "Checkout ${{ github.event.repository.name }}@${{ steps.branch.outputs.name }}"
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ steps.branch.outputs.name }}
|
||||
|
||||
- name: "Set release SHA"
|
||||
id: sha
|
||||
run: echo "sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT
|
||||
|
||||
# if this is a real release and a release branch was created, delete it
|
||||
- name: "Delete release branch: ${{ needs.release-branch.outputs.name }}"
|
||||
if: ${{ inputs.deploy-to == 'prod' && inputs.is-nightly-release == 'false' && needs.release-branch.outputs.name != '' }}
|
||||
run: git push origin -d ${{ needs.release-branch.outputs.name }}
|
||||
@@ -1 +0,0 @@
|
||||
recursive-include dbt/include *.sql *.yml *.md
|
||||
48
Makefile
48
Makefile
@@ -1,48 +0,0 @@
|
||||
.DEFAULT_GOAL:=help
|
||||
|
||||
.PHONY: dev
|
||||
dev: ## Installs adapter in develop mode along with development dependencies
|
||||
@\
|
||||
pip install -e . -r dev-requirements.txt && pre-commit install
|
||||
|
||||
.PHONY: dev-uninstall
|
||||
dev-uninstall: ## Uninstalls all packages while maintaining the virtual environment
|
||||
## Useful when updating versions, or if you accidentally installed into the system interpreter
|
||||
pip freeze | grep -v "^-e" | cut -d "@" -f1 | xargs pip uninstall -y
|
||||
pip uninstall -y dbt-snowflake
|
||||
|
||||
.PHONY: unit
|
||||
unit: ## Runs unit tests with py39.
|
||||
@\
|
||||
tox -e py39
|
||||
|
||||
.PHONY: test
|
||||
test: ## Runs unit tests with py39 and code checks against staged changes.
|
||||
@\
|
||||
tox -p -e py39; \
|
||||
pre-commit run --all-files
|
||||
|
||||
.PHONY: integration
|
||||
integration: ## Runs snowflake integration tests with py38.
|
||||
@\
|
||||
tox -e py39-snowflake --
|
||||
|
||||
.PHONY: clean
|
||||
@echo "cleaning repo"
|
||||
@git clean -f -X
|
||||
|
||||
.PHONY: help
|
||||
help: ## Show this help message.
|
||||
@echo 'usage: make [target]'
|
||||
@echo
|
||||
@echo 'targets:'
|
||||
@grep -E '^[7+a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
|
||||
|
||||
.PHONY: docker-dev
|
||||
docker-dev:
|
||||
docker build -f docker/dev.Dockerfile -t dbt-snowflake-dev .
|
||||
docker run --rm -it --name dbt-snowflake-dev -v $(shell pwd):/opt/code dbt-snowflake-dev
|
||||
|
||||
.PHONY: docker-prod
|
||||
docker-prod:
|
||||
docker build -f docker/Dockerfile -t dbt-snowflake .
|
||||
@@ -1,23 +0,0 @@
|
||||
# install latest changes in dbt-core
|
||||
git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-core&subdirectory=core
|
||||
git+https://github.com/dbt-labs/dbt-adapters.git
|
||||
git+https://github.com/dbt-labs/dbt-adapters.git#subdirectory=dbt-tests-adapter
|
||||
git+https://github.com/dbt-labs/dbt-common.git
|
||||
|
||||
# dev
|
||||
ipdb~=0.13.13
|
||||
pre-commit~=3.7.0
|
||||
|
||||
# test
|
||||
ddtrace==2.3.0
|
||||
pytest~=7.4
|
||||
pytest-csv~=3.0
|
||||
pytest-dotenv~=0.5.2
|
||||
pytest-logbook~=1.2
|
||||
pytest-xdist~=3.6
|
||||
tox~=4.16
|
||||
|
||||
# build
|
||||
bumpversion~=0.6.0
|
||||
twine~=5.1
|
||||
wheel~=0.43
|
||||
62
hatch.toml
Normal file
62
hatch.toml
Normal file
@@ -0,0 +1,62 @@
|
||||
[version]
|
||||
path = "dbt/adapters/snowflake/__version__.py"
|
||||
|
||||
[build.targets.sdist]
|
||||
packages = ["src/dbt"]
|
||||
sources = ["src"]
|
||||
|
||||
[build.targets.wheel]
|
||||
packages = ["src/dbt"]
|
||||
sources = ["src"]
|
||||
|
||||
[envs.default]
|
||||
dependencies = [
|
||||
"dbt-adapters @ git+https://github.com/dbt-labs/dbt-adapters.git",
|
||||
"dbt-common @ git+https://github.com/dbt-labs/dbt-common.git",
|
||||
"dbt-tests-adapter @ git+https://github.com/dbt-labs/dbt-adapters.git#subdirectory=dbt-tests-adapter",
|
||||
"dbt-core @ git+https://github.com/dbt-labs/dbt-core.git#subdirectory=core",
|
||||
"ddtrace==2.3.0",
|
||||
"ipdb~=0.13.13",
|
||||
"pre-commit~=3.7.0",
|
||||
"pytest~=7.4",
|
||||
"pytest-csv~=3.0",
|
||||
"pytest-dotenv~=0.5.2",
|
||||
"pytest-logbook~=1.2",
|
||||
"pytest-xdist~=3.6",
|
||||
"tox~=4.16", # does this pin deps transitively?
|
||||
]
|
||||
|
||||
[envs.default.scripts]
|
||||
setup = "pre-commit install"
|
||||
code-quality = "pre-commit run --all-files"
|
||||
unit-tests = "python -m pytest {args:tests/unit}"
|
||||
integration-tests = "- python -m pytest {args:tests/functional}"
|
||||
docker-dev = [
|
||||
"docker build -f docker/dev.Dockerfile -t dbt-snowflake-dev .",
|
||||
"docker run --rm -it --name dbt-snowflake-dev -v $(pwd):/opt/code dbt-snowflake-dev",
|
||||
]
|
||||
|
||||
[envs.build]
|
||||
detached = true
|
||||
dependencies = [
|
||||
"wheel",
|
||||
"twine",
|
||||
"check-wheel-contents",
|
||||
]
|
||||
|
||||
[envs.build.scripts]
|
||||
check-all = [
|
||||
"- check-wheel",
|
||||
"- check-sdist",
|
||||
]
|
||||
check-wheel = [
|
||||
"twine check dist/*",
|
||||
"find ./dist/dbt_snowflake-*.whl -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/",
|
||||
"pip freeze | grep dbt-snowflake",
|
||||
]
|
||||
check-sdist = [
|
||||
"check-wheel-contents dist/*.whl --ignore W007,W008",
|
||||
"find ./dist/dbt_snowflake-*.gz -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/",
|
||||
"pip freeze | grep dbt-snowflake",
|
||||
]
|
||||
docker-prod = "docker build -f docker/Dockerfile -t dbt-snowflake ."
|
||||
48
pyproject.toml
Normal file
48
pyproject.toml
Normal file
@@ -0,0 +1,48 @@
|
||||
[build-system]
|
||||
requires = ["hatchling"]
|
||||
build-backend = "hatchling.build"
|
||||
|
||||
[project]
|
||||
dynamic = ["version"]
|
||||
name = "dbt-snowflake"
|
||||
description = "The Snowflake adapter plugin for dbt"
|
||||
readme = "README.md"
|
||||
keywords = ["dbt", "adapter", "adapters", "database", "elt", "dbt-core", "dbt Core", "dbt Cloud", "dbt Labs", "snowflake"]
|
||||
requires-python = ">=3.9.0"
|
||||
authors = [{ name = "dbt Labs", email = "info@dbtlabs.com" }]
|
||||
maintainers = [{ name = "dbt Labs", email = "info@dbtlabs.com" }]
|
||||
classifiers = [
|
||||
"Development Status :: 5 - Production/Stable",
|
||||
"License :: OSI Approved :: Apache Software License",
|
||||
"Operating System :: MacOS :: MacOS X",
|
||||
"Operating System :: Microsoft :: Windows",
|
||||
"Operating System :: POSIX :: Linux",
|
||||
"Programming Language :: Python :: 3.9",
|
||||
"Programming Language :: Python :: 3.10",
|
||||
"Programming Language :: Python :: 3.11",
|
||||
"Programming Language :: Python :: 3.12",
|
||||
]
|
||||
dependencies = [
|
||||
"dbt-common>=1.10,<2.0",
|
||||
"dbt-adapters>=1.10.4,<2.0",
|
||||
"snowflake-connector-python[secure-local-storage]>=3.0.0,<3.12.4",
|
||||
# add dbt-core to ensure backwards compatibility of installation, this is not a functional dependency
|
||||
"dbt-core>=1.8.0",
|
||||
# installed via dbt-core but referenced directly; don't pin to avoid version conflicts with dbt-core
|
||||
"agate",
|
||||
]
|
||||
|
||||
[project.urls]
|
||||
Homepage = "https://github.com/dbt-labs/dbt-snowflake"
|
||||
Documentation = "https://docs.getdbt.com"
|
||||
Repository = "https://github.com/dbt-labs/dbt-snowflake.git"
|
||||
Issues = "https://github.com/dbt-labs/dbt-snowflake/issues"
|
||||
Changelog = "https://github.com/dbt-labs/dbt-snowflake/blob/main/CHANGELOG.md"
|
||||
|
||||
[tool.pytest.ini_options]
|
||||
testpaths = ["tests/functional", "tests/unit"]
|
||||
env_files = ["test.env"]
|
||||
addopts = "-v --color=yes -n auto"
|
||||
filterwarnings = [
|
||||
"ignore:datetime.datetime.utcnow:DeprecationWarning",
|
||||
]
|
||||
10
pytest.ini
10
pytest.ini
@@ -1,10 +0,0 @@
|
||||
[pytest]
|
||||
filterwarnings =
|
||||
ignore:.*'soft_unicode' has been renamed to 'soft_str'*:DeprecationWarning
|
||||
ignore:unclosed file .*:ResourceWarning
|
||||
env_files =
|
||||
test.env
|
||||
testpaths =
|
||||
tests/unit
|
||||
tests/integration
|
||||
tests/functional
|
||||
81
setup.py
81
setup.py
@@ -1,81 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import sys
|
||||
|
||||
# require python 3.8 or newer
|
||||
if sys.version_info < (3, 9):
|
||||
print("Error: dbt does not support this version of Python.")
|
||||
print("Please upgrade to Python 3.9 or higher.")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
# require version of setuptools that supports find_namespace_packages
|
||||
from setuptools import setup
|
||||
|
||||
try:
|
||||
from setuptools import find_namespace_packages
|
||||
except ImportError:
|
||||
# the user has a downlevel version of setuptools.
|
||||
print("Error: dbt requires setuptools v40.1.0 or higher.")
|
||||
print('Please upgrade setuptools with "pip install --upgrade setuptools" ' "and try again")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
# pull long description from README
|
||||
this_directory = os.path.abspath(os.path.dirname(__file__))
|
||||
with open(os.path.join(this_directory, "README.md")) as f:
|
||||
long_description = f.read()
|
||||
|
||||
|
||||
# used for this adapter's version
|
||||
VERSION = Path(__file__).parent / "dbt/adapters/snowflake/__version__.py"
|
||||
|
||||
|
||||
def _plugin_version() -> str:
|
||||
"""
|
||||
Pull the package version from the main package version file
|
||||
"""
|
||||
attributes = {}
|
||||
exec(VERSION.read_text(), attributes)
|
||||
return attributes["version"]
|
||||
|
||||
|
||||
package_name = "dbt-snowflake"
|
||||
description = """The Snowflake adapter plugin for dbt"""
|
||||
|
||||
setup(
|
||||
name=package_name,
|
||||
version=_plugin_version(),
|
||||
description=description,
|
||||
long_description=long_description,
|
||||
long_description_content_type="text/markdown",
|
||||
author="dbt Labs",
|
||||
author_email="info@dbtlabs.com",
|
||||
url="https://github.com/dbt-labs/dbt-snowflake",
|
||||
packages=find_namespace_packages(include=["dbt", "dbt.*"]),
|
||||
include_package_data=True,
|
||||
install_requires=[
|
||||
"dbt-common>=1.10,<2.0",
|
||||
"dbt-adapters>=1.10.4,<2.0",
|
||||
"snowflake-connector-python[secure-local-storage]~=3.0",
|
||||
# add dbt-core to ensure backwards compatibility of installation, this is not a functional dependency
|
||||
"dbt-core>=1.8.0",
|
||||
# installed via dbt-core but referenced directly; don't pin to avoid version conflicts with dbt-core
|
||||
"agate",
|
||||
],
|
||||
zip_safe=False,
|
||||
classifiers=[
|
||||
"Development Status :: 5 - Production/Stable",
|
||||
"License :: OSI Approved :: Apache Software License",
|
||||
"Operating System :: Microsoft :: Windows",
|
||||
"Operating System :: MacOS :: MacOS X",
|
||||
"Operating System :: POSIX :: Linux",
|
||||
"Programming Language :: Python :: 3.9",
|
||||
"Programming Language :: Python :: 3.10",
|
||||
"Programming Language :: Python :: 3.11",
|
||||
"Programming Language :: Python :: 3.12",
|
||||
],
|
||||
python_requires=">=3.9",
|
||||
)
|
||||
32
tox.ini
32
tox.ini
@@ -1,32 +0,0 @@
|
||||
[tox]
|
||||
skipsdist = True
|
||||
envlist = py39,py310,py311,py312
|
||||
|
||||
[testenv:{unit,py39,py310,py311,py312,py}]
|
||||
description = unit testing
|
||||
skip_install = true
|
||||
passenv =
|
||||
DBT_*
|
||||
PYTEST_ADDOPTS
|
||||
commands = {envpython} -m pytest {posargs} tests/unit
|
||||
deps =
|
||||
-rdev-requirements.txt
|
||||
-e.
|
||||
|
||||
[testenv:{integration,py39,py310,py311,py312,py}-{snowflake}]
|
||||
description = adapter plugin integration testing
|
||||
skip_install = true
|
||||
passenv =
|
||||
DBT_*
|
||||
SNOWFLAKE_TEST_*
|
||||
PYTEST_ADDOPTS
|
||||
DD_CIVISIBILITY_AGENTLESS_ENABLED
|
||||
DD_API_KEY
|
||||
DD_SITE
|
||||
DD_ENV
|
||||
DD_SERVICE
|
||||
commands =
|
||||
snowflake: {envpython} -m pytest {posargs} tests/functional
|
||||
deps =
|
||||
-rdev-requirements.txt
|
||||
-e.
|
||||
Reference in New Issue
Block a user