ADAP-1115: Move to hatch and pyproject.toml (#1250)

* move config files into pyproject.toml and hatch.toml
* update workflow files to use hatch.toml
* move test matrix into the workflow
* fix pytest config in pyproject.toml
This commit is contained in:
Mike Alfare
2024-12-20 11:02:31 -05:00
committed by GitHub
parent b8607e9d18
commit b687ac477a
18 changed files with 635 additions and 551 deletions

View File

@@ -1,35 +0,0 @@
[bumpversion]
current_version = 1.10.0a1
parse = (?P<major>[\d]+) # major version number
\.(?P<minor>[\d]+) # minor version number
\.(?P<patch>[\d]+) # patch version number
(?P<prerelease> # optional pre-release - ex: a1, b2, rc25
(?P<prekind>a|b|rc) # pre-release type
(?P<num>[\d]+) # pre-release version number
)?
( # optional nightly release indicator
\.(?P<nightly>dev[0-9]+) # ex: .dev02142023
)? # expected matches: `1.15.0`, `1.5.0a11`, `1.5.0a1.dev123`, `1.5.0.dev123457`, expected failures: `1`, `1.5`, `1.5.2-a1`, `text1.5.0`
serialize =
{major}.{minor}.{patch}{prekind}{num}.{nightly}
{major}.{minor}.{patch}.{nightly}
{major}.{minor}.{patch}{prekind}{num}
{major}.{minor}.{patch}
commit = False
tag = False
[bumpversion:part:prekind]
first_value = a
optional_value = final
values =
a
b
rc
final
[bumpversion:part:num]
first_value = 1
[bumpversion:part:nightly]
[bumpversion:file:dbt/adapters/snowflake/__version__.py]

View File

@@ -0,0 +1,6 @@
kind: Under the Hood
body: Move from setup.py to pyproject.toml and to hatch as a dev tool
time: 2024-11-17T18:44:30.85288-05:00
custom:
Author: mikealfare
Issue: "1250"

View File

@@ -1,95 +0,0 @@
module.exports = ({ context }) => {
const defaultPythonVersion = "3.9";
const supportedPythonVersions = ["3.9", "3.10", "3.11", "3.12"];
const supportedAdapters = ["snowflake"];
// if PR, generate matrix based on files changed and PR labels
if (context.eventName.includes("pull_request")) {
// `changes` is a list of adapter names that have related
// file changes in the PR
// ex: ['postgres', 'snowflake']
const changes = JSON.parse(process.env.CHANGES);
const labels = context.payload.pull_request.labels.map(({ name }) => name);
console.log("labels", labels);
console.log("changes", changes);
const testAllLabel = labels.includes("test all");
const include = [];
for (const adapter of supportedAdapters) {
if (
changes.includes(adapter) ||
testAllLabel ||
labels.includes(`test ${adapter}`)
) {
for (const pythonVersion of supportedPythonVersions) {
if (
pythonVersion === defaultPythonVersion ||
labels.includes(`test python${pythonVersion}`) ||
testAllLabel
) {
// always run tests on ubuntu by default
include.push({
os: "ubuntu-latest",
adapter,
"python-version": pythonVersion,
});
if (labels.includes("test windows") || testAllLabel) {
include.push({
os: "windows-latest",
adapter,
"python-version": pythonVersion,
});
}
if (labels.includes("test macos") || testAllLabel) {
include.push({
os: "macos-14",
adapter,
"python-version": pythonVersion,
});
}
}
}
}
}
console.log("matrix", { include });
return {
include,
};
}
// if not PR, generate matrix of python version, adapter, and operating
// system to run integration tests on
const include = [];
// run for all adapters and python versions on ubuntu
for (const adapter of supportedAdapters) {
for (const pythonVersion of supportedPythonVersions) {
include.push({
os: 'ubuntu-latest',
adapter: adapter,
"python-version": pythonVersion,
});
}
}
// additionally include runs for all adapters, on macos and windows,
// but only for the default python version
for (const adapter of supportedAdapters) {
for (const operatingSystem of ["windows-latest", "macos-14"]) {
include.push({
os: operatingSystem,
adapter: adapter,
"python-version": defaultPythonVersion,
});
}
}
console.log("matrix", { include });
return {
include,
};
};

View File

@@ -1,15 +0,0 @@
#!/bin/bash -e
set -e
git_branch=$1
target_req_file="dev-requirements.txt"
core_req_sed_pattern="s|dbt-core.git.*#egg=dbt-core|dbt-core.git@${git_branch}#egg=dbt-core|g"
tests_req_sed_pattern="s|dbt-core.git.*#egg=dbt-tests|dbt-core.git@${git_branch}#egg=dbt-tests|g"
if [[ "$OSTYPE" == darwin* ]]; then
# mac ships with a different version of sed that requires a delimiter arg
sed -i "" "$core_req_sed_pattern" $target_req_file
sed -i "" "$tests_req_sed_pattern" $target_req_file
else
sed -i "$core_req_sed_pattern" $target_req_file
sed -i "$tests_req_sed_pattern" $target_req_file
fi

View File

@@ -5,17 +5,20 @@ set -e
dbt_adapters_branch=$1
dbt_core_branch=$2
dbt_common_branch=$3
target_req_file="dev-requirements.txt"
core_req_sed_pattern="s|dbt-core.git.*#egg=dbt-core|dbt-core.git@${dbt_core_branch}#egg=dbt-core|g"
target_req_file="hatch.toml"
core_req_sed_pattern="s|dbt-core.git.*#subdirectory=core|dbt-core.git@${dbt_core_branch}#subdirectory=core|g"
adapters_req_sed_pattern="s|dbt-adapters.git|dbt-adapters.git@${dbt_adapters_branch}|g"
tests_req_sed_pattern="s|dbt-adapters.git.*#subdirectory=dbt-tests-adapter|dbt-adapters.git@${dbt_adapters_branch}#subdirectory=dbt-tests-adapter|g"
common_req_sed_pattern="s|dbt-common.git|dbt-common.git@${dbt_common_branch}|g"
if [[ "$OSTYPE" == darwin* ]]; then
# mac ships with a different version of sed that requires a delimiter arg
sed -i "" "$adapters_req_sed_pattern" $target_req_file
sed -i "" "$tests_req_sed_pattern" $target_req_file
sed -i "" "$core_req_sed_pattern" $target_req_file
sed -i "" "$common_req_sed_pattern" $target_req_file
else
sed -i "$adapters_req_sed_pattern" $target_req_file
sed -i "$tests_req_sed_pattern" $target_req_file
sed -i "$core_req_sed_pattern" $target_req_file
sed -i "$common_req_sed_pattern" $target_req_file
fi

View File

@@ -76,85 +76,31 @@ defaults:
shell: bash
jobs:
# generate test metadata about what files changed and the testing matrix to use
test-metadata:
# run if not a PR from a forked repository or has a label to mark as safe to test
if: >-
github.event_name != 'pull_request_target' ||
github.event.pull_request.head.repo.full_name == github.repository ||
contains(github.event.pull_request.labels.*.name, 'ok to test')
runs-on: ubuntu-latest
outputs:
matrix: ${{ steps.generate-matrix.outputs.result }}
steps:
- name: Check out the repository (non-PR)
if: github.event_name != 'pull_request_target'
uses: actions/checkout@v4
with:
persist-credentials: false
- name: Check out the repository (PR)
if: github.event_name == 'pull_request_target'
uses: actions/checkout@v4
with:
persist-credentials: false
ref: ${{ github.event.pull_request.head.sha }}
- name: Check if relevant files changed
if: github.event_name == 'pull_request_target'
# https://github.com/marketplace/actions/paths-changes-filter
# For each filter, it sets output variable named by the filter to the text:
# 'true' - if any of changed files matches any of filter rules
# 'false' - if none of changed files matches any of filter rules
# also, returns:
# `changes` - JSON array with names of all filters matching any of the changed files
uses: dorny/paths-filter@v3
id: get-changes
with:
token: ${{ secrets.GITHUB_TOKEN }}
filters: |
snowflake:
- '.github/**/*.yml'
- '.github/**/*.sh'
- 'dbt/**'
- 'tests/**'
- 'dev-requirements.txt'
- '*.py'
- name: Generate integration test matrix
id: generate-matrix
uses: actions/github-script@v7
env:
CHANGES: ${{ steps.get-changes.outputs.changes }}
with:
script: |
const script = require('./.github/scripts/integration-test-matrix.js')
const matrix = script({ context })
console.log(matrix)
return matrix
test:
name: ${{ matrix.adapter }} / python ${{ matrix.python-version }} / ${{ matrix.os }}
name: ${{ matrix.os }} / python ${{ matrix.python-version }}
# run if not a PR from a forked repository or has a label to mark as safe to test
# also checks that the matrix generated is not empty
if: >-
needs.test-metadata.outputs.matrix &&
fromJSON( needs.test-metadata.outputs.matrix ).include[0] &&
(
github.event_name != 'pull_request_target' ||
github.event.pull_request.head.repo.full_name == github.repository ||
contains(github.event.pull_request.labels.*.name, 'ok to test')
)
runs-on: ${{ matrix.os }}
needs: test-metadata
strategy:
fail-fast: false
matrix: ${{ fromJSON(needs.test-metadata.outputs.matrix) }}
matrix:
os: [ubuntu-22.04]
python-version: ["3.9", "3.10", "3.11", "3.12"]
include:
- os: macos-14
python-version: "3.9"
- os: windows-2022
python-version: "3.9"
env:
TOXENV: integration-${{ matrix.adapter }}
PYTEST_ADDOPTS: "-v --color=yes -n4 --csv integration_results.csv"
DBT_INVOCATION_ENV: github-actions
DD_CIVISIBILITY_AGENTLESS_ENABLED: true
DD_API_KEY: ${{ secrets.DATADOG_API_KEY }}
@@ -190,24 +136,19 @@ jobs:
with:
python-version: ${{ matrix.python-version }}
- name: Update Adapters and Core branches (update dev_requirements.txt)
- name: Update Adapters and Core branches (update hatch.toml)
if: ${{ github.event_name == 'workflow_dispatch' }}
run: |
./.github/scripts/update_dev_dependency_branches.sh \
${{ inputs.dbt_adapters_branch }} \
${{ inputs.dbt_core_branch }} \
${{ inputs.dbt_common_branch }}
cat dev-requirements.txt
cat hatch.toml
- name: Install python dependencies
run: |
python -m pip install --user --upgrade pip
python -m pip install tox
python -m pip --version
tox --version
uses: pypa/hatch@install
- name: Run tox (snowflake)
if: matrix.adapter == 'snowflake'
- run: hatch run integration-tests tests/functional --ddtrace
env:
SNOWFLAKE_TEST_ACCOUNT: ${{ secrets.SNOWFLAKE_TEST_ACCOUNT }}
SNOWFLAKE_TEST_PASSWORD: ${{ secrets.SNOWFLAKE_TEST_PASSWORD }}
@@ -226,26 +167,6 @@ jobs:
DBT_TEST_USER_1: dbt_test_role_1
DBT_TEST_USER_2: dbt_test_role_2
DBT_TEST_USER_3: dbt_test_role_3
run: tox -- --ddtrace
- name: Get current date
if: always()
id: date
run: echo "date=$(date +'%Y-%m-%dT%H_%M_%S')" >> $GITHUB_OUTPUT #no colons allowed for artifacts
- uses: actions/upload-artifact@v4
if: always()
with:
name: logs_${{ matrix.python-version }}_${{ matrix.os }}_${{ matrix.adapter }}-${{ steps.date.outputs.date }}
path: ./logs
overwrite: true
- uses: actions/upload-artifact@v4
if: always()
with:
name: integration_results_${{ matrix.python-version }}_${{ matrix.os }}_${{ matrix.adapter }}-${{ steps.date.outputs.date }}.csv
path: integration_results.csv
overwrite: true
require-label-comment:
runs-on: ubuntu-latest

View File

@@ -52,16 +52,8 @@ jobs:
with:
python-version: '3.9'
- name: Install python dependencies
run: |
python -m pip install --user --upgrade pip
python -m pip install -r dev-requirements.txt
python -m pip --version
pre-commit --version
dbt --version
- name: Run pre-commit hooks
run: pre-commit run --all-files --show-diff-on-failure
uses: pre-commit/action@v3.0.1
unit:
name: unit test / python ${{ matrix.python-version }}
@@ -73,10 +65,6 @@ jobs:
matrix:
python-version: ['3.9', '3.10', '3.11', '3.12']
env:
TOXENV: "unit"
PYTEST_ADDOPTS: "-v --color=yes --csv unit_results.csv"
steps:
- name: Check out the repository
uses: actions/checkout@v4
@@ -89,26 +77,9 @@ jobs:
python-version: ${{ matrix.python-version }}
- name: Install python dependencies
run: |
python -m pip install --user --upgrade pip
python -m pip install tox
python -m pip --version
tox --version
uses: pypa/hatch@install
- name: Run tox
run: tox
- name: Get current date
if: always()
id: date
run: echo "date=$(date +'%Y-%m-%dT%H_%M_%S')" >> $GITHUB_OUTPUT #no colons allowed for artifacts
- uses: actions/upload-artifact@v4
if: always()
with:
name: unit_results_${{ matrix.python-version }}-${{ steps.date.outputs.date }}.csv
path: unit_results.csv
overwrite: true
- run: hatch run unit-tests
build:
name: build packages
@@ -129,25 +100,16 @@ jobs:
with:
python-version: '3.9'
- name: Install python dependencies
run: |
python -m pip install --user --upgrade pip
python -m pip install --upgrade setuptools wheel twine check-wheel-contents
python -m pip --version
- uses: pypa/hatch@install
- name: Build distributions
run: ./scripts/build-dist.sh
run: hatch build
- name: Show distributions
run: ls -lh dist/
- name: Check distribution descriptions
run: |
twine check dist/*
- name: Check wheel contents
run: |
check-wheel-contents dist/*.whl --ignore W007,W008
run: hatch run build:check-all
- name: Check if this is an alpha version
id: check-is-alpha
@@ -184,12 +146,6 @@ jobs:
with:
python-version: ${{ matrix.python-version }}
- name: Install python dependencies
run: |
python -m pip install --user --upgrade pip
python -m pip install --upgrade wheel
python -m pip --version
- uses: actions/download-artifact@v4
with:
name: dist

View File

@@ -50,11 +50,15 @@ jobs:
commit_sha=$(git rev-parse HEAD)
echo "release_commit=$commit_sha" >> $GITHUB_OUTPUT
- name: "Get Current Version Number"
id: version-number-sources
run: |
current_version=`awk -F"current_version = " '{print $2}' .bumpversion.cfg | tr '\n' ' '`
echo "current_version=$current_version" >> $GITHUB_OUTPUT
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: '3.9'
- uses: pypa/hatch@install
- id: version-number-sources
run: echo "current_version=$(hatch version)" >> $GITHUB_OUTPUT
- name: "Audit Version And Parse Into Parts"
id: semver

View File

@@ -119,86 +119,70 @@ defaults:
shell: bash
jobs:
log-inputs:
name: Log Inputs
runs-on: ubuntu-latest
steps:
- name: "[DEBUG] Print Variables"
run: |
echo The last commit sha in the release: ${{ inputs.sha }}
echo The branch to release from: ${{ inputs.target_branch }}
echo The release version number: ${{ inputs.version_number }}
echo Build script path: ${{ inputs.build_script_path }}
echo Environment setup script path: ${{ inputs.env_setup_script_path }}
echo AWS S3 bucket name: ${{ inputs.s3_bucket_name }}
echo Package test command: ${{ inputs.package_test_command }}
echo Test run: ${{ inputs.test_run }}
echo Nightly release: ${{ inputs.nightly_release }}
echo Only Docker: ${{ inputs.only_docker }}
bump-version-generate-changelog:
name: Bump package version, Generate changelog
uses: dbt-labs/dbt-release/.github/workflows/release-prep.yml@main
release-prep:
name: "Release prep: generate changelog, bump version"
uses: ./.github/workflows/release_prep_hatch.yml
with:
sha: ${{ inputs.sha }}
version_number: ${{ inputs.version_number }}
target_branch: ${{ inputs.target_branch }}
env_setup_script_path: ${{ inputs.env_setup_script_path }}
test_run: ${{ inputs.test_run }}
nightly_release: ${{ inputs.nightly_release }}
branch: ${{ inputs.branch }}
version: ${{ inputs.version }}
deploy-to: ${{ inputs.deploy-to }}
secrets: inherit
log-outputs-bump-version-generate-changelog:
name: "[Log output] Bump package version, Generate changelog"
if: ${{ !failure() && !cancelled() && !inputs.only_docker }}
needs: [bump-version-generate-changelog]
build-release:
name: "Build release"
needs: release-prep
runs-on: ubuntu-latest
outputs:
archive-name: ${{ steps.archive.outputs.name }}
steps:
- name: Print variables
- uses: actions/checkout@v4
with:
ref: ${{ needs.release-prep.outputs.release-branch }}
persist-credentials: false
- uses: actions/setup-python@v5
with:
python-version: ${{ inputs.python_version }}
- uses: pypa/hatch@install
- id: archive
run: |
echo Final SHA : ${{ needs.bump-version-generate-changelog.outputs.final_sha }}
echo Changelog path: ${{ needs.bump-version-generate-changelog.outputs.changelog_path }}
build-test-package:
name: Build, Test, Package
if: ${{ !failure() && !cancelled() && !inputs.only_docker }}
needs: [bump-version-generate-changelog]
uses: dbt-labs/dbt-release/.github/workflows/build.yml@main
with:
sha: ${{ needs.bump-version-generate-changelog.outputs.final_sha }}
version_number: ${{ inputs.version_number }}
changelog_path: ${{ needs.bump-version-generate-changelog.outputs.changelog_path }}
build_script_path: ${{ inputs.build_script_path }}
s3_bucket_name: ${{ inputs.s3_bucket_name }}
package_test_command: ${{ inputs.package_test_command }}
test_run: ${{ inputs.test_run }}
nightly_release: ${{ inputs.nightly_release }}
secrets:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
archive_name=${{ github.event.repository.name }}-${{ inputs.version }}-${{ inputs.deploy-to }}
echo "name=$archive_name" >> $GITHUB_OUTPUT
- run: hatch build && hatch run build:check-all
- uses: actions/upload-artifact@v4
with:
name: ${{ steps.archive.outputs.name }}
path: dist/
retention-days: 3
github-release:
name: GitHub Release
if: ${{ !failure() && !cancelled() && !inputs.only_docker }}
needs: [bump-version-generate-changelog, build-test-package]
uses: dbt-labs/dbt-release/.github/workflows/github-release.yml@main
needs: [build-release, release-prep]
uses: dbt-labs/dbt-adapters/.github/workflows/github-release.yml@main
with:
sha: ${{ needs.bump-version-generate-changelog.outputs.final_sha }}
version_number: ${{ inputs.version_number }}
changelog_path: ${{ needs.bump-version-generate-changelog.outputs.changelog_path }}
test_run: ${{ inputs.test_run }}
archive_name: ${{ needs.build-release.outputs.archive-name }}
pypi-release:
name: PyPI Release
if: ${{ !failure() && !cancelled() && !inputs.only_docker }}
needs: [bump-version-generate-changelog, build-test-package]
uses: dbt-labs/dbt-release/.github/workflows/pypi-release.yml@main
with:
version_number: ${{ inputs.version_number }}
test_run: ${{ inputs.test_run }}
secrets:
PYPI_API_TOKEN: ${{ secrets.PYPI_API_TOKEN }}
TEST_PYPI_API_TOKEN: ${{ secrets.TEST_PYPI_API_TOKEN }}
needs: build-release
runs-on: ubuntu-latest
environment:
name: ${{ inputs.deploy-to }}
url: ${{ vars.PYPI_PROJECT_URL }}
permissions:
# this permission is required for trusted publishing
# see https://github.com/marketplace/actions/pypi-publish
id-token: write
steps:
- uses: dbt-labs/dbt-adapters/.github/actions/publish-pypi@main
with:
repository-url: ${{ vars.PYPI_REPOSITORY_URL }}
archive-name: ${{ needs.build-release.outputs.archive-name }}
docker-release:
name: "Docker Release"
@@ -206,7 +190,7 @@ jobs:
# what we need to release but draft releases don't actually tag the commit so it
# finds nothing to release
if: ${{ !failure() && !cancelled() && (!inputs.test_run || inputs.only_docker) }}
needs: [bump-version-generate-changelog, build-test-package, github-release]
needs: github-release
permissions:
packages: write
uses: dbt-labs/dbt-release/.github/workflows/release-docker.yml@main

440
.github/workflows/release_prep_hatch.yml vendored Normal file
View File

@@ -0,0 +1,440 @@
# **what?**
# Perform the version bump, generate the changelog and run tests.
#
# Inputs:
# branch: The branch that we will release from
# version: The release version number (i.e. 1.0.0b1, 1.2.3rc2, 1.0.0)
# deploy-to: If we are deploying to prod or test, if test then release from branch
# is-nightly-release: Identifier that this is nightly release
#
# Outputs:
# release-sha: The sha that will actually be released. This can differ from the
# input sha if adding a version bump and/or changelog
# changelog-path: Path to the changelog file (ex .changes/1.2.3-rc1.md)
#
# Branching strategy:
# - During execution workflow execution the temp branch will be generated.
# - For normal runs the temp branch will be removed once changes were merged to target branch;
# - For test runs we will keep temp branch and will use it for release;
# Naming strategy:
# - For normal runs: prep-release/${{ inputs.deploy-to}}/${{ inputs.version }}_$GITHUB_RUN_ID
# - For nightly releases: prep-release/nightly-release/${{ inputs.version }}_$GITHUB_RUN_ID
#
# **why?**
# Reusable and consistent GitHub release process.
#
# **when?**
# Call when ready to kick off a build and release
#
# Validation Checks
#
# 1. Bump the version if it has not been bumped
# 2. Generate the changelog (via changie) if there is no markdown file for this version
name: "Release prep"
run-name: "Release prep: Generate changelog and bump to ${{ inputs.version }} for release to ${{ inputs.deploy-to }}"
on:
workflow_call:
inputs:
branch:
description: "The branch to release from"
type: string
default: "main"
version:
description: "The version to release"
required: true
type: string
deploy-to:
description: "Deploy to test or prod"
type: string
default: "prod"
is-nightly-release:
description: "Identify if this is a nightly release"
type: boolean
default: false
outputs:
release-branch:
description: "The branch to be released from"
value: ${{ jobs.release.outputs.branch }}
release-sha:
description: "The SHA to be released"
value: ${{ jobs.release.outputs.sha }}
changelog-path:
description: "The path to the changelog from the repo root for this version, e.g. .changes/1.8.0-b1.md"
value: ${{ jobs.release-inputs.outputs.changelog-path }}
secrets:
FISHTOWN_BOT_PAT:
description: "Token to commit/merge changes into branches"
required: true
IT_TEAM_MEMBERSHIP:
description: "Token that can view org level teams"
required: true
permissions:
contents: write
defaults:
run:
shell: bash
env:
PYTHON_TARGET_VERSION: 3.12
NOTIFICATION_PREFIX: "[Release Prep]"
jobs:
release-inputs:
runs-on: ubuntu-latest
outputs:
changelog-path: ${{ steps.changelog.outputs.path }}
changelog-exists: ${{ steps.changelog.outputs.exists }}
base-version: ${{ steps.semver.outputs.base-version }}
pre-release: ${{ steps.semver.outputs.pre-release }}
is-pre-release: ${{ steps.semver.outputs.is-pre-release }}
version-is-current: ${{ steps.version.outputs.is-current }}
steps:
- name: "[DEBUG] Log inputs"
run: |
# WORKFLOW INPUTS
echo Branch: ${{ inputs.branch }}
echo Release version: ${{ inputs.version }}
echo Deploy to: ${{ inputs.deploy-to }}
echo Nightly release: ${{ inputs.is-nightly-release }}
# ENVIRONMENT VARIABLES
echo Python version: ${{ env.PYTHON_TARGET_VERSION }}
echo Notification prefix: ${{ env.NOTIFICATION_PREFIX }}
- name: "Checkout ${{ github.event.repository.name }}@${{ inputs.branch }}"
uses: actions/checkout@v4
with:
ref: ${{ inputs.branch }}
- name: "Setup `hatch`"
uses: dbt-labs/dbt-adapters/.github/actions/setup-hatch@main
- name: "Parse input version"
id: semver
uses: dbt-labs/actions/parse-semver@v1.1.1
with:
version: ${{ inputs.version }}
- name: "Audit version"
id: version
run: |
is_current=false
current_version=$(hatch version)
if test "$current_version" = "${{ inputs.version }}"
then
is_current=true
fi
echo "is-current=$is_current" >> $GITHUB_OUTPUT
- name: "[INFO] Skip version bump"
if: steps.version.outputs.is-current == 'true'
run: |
title="Skip version bump"
message="The version matches the input version ${{ inputs.version }}, skipping version bump"
echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message"
- name: "Audit changelog"
id: changelog
run: |
path=".changes/"
if [[ ${{ steps.semver.outputs.is-pre-release }} -eq 1 ]]
then
path+="${{ steps.semver.outputs.base-version }}-${{ steps.semver.outputs.pre-release }}.md"
else
path+="${{ steps.semver.outputs.base-version }}.md"
fi
echo "path=$path" >> $GITHUB_OUTPUT
does_exist=false
if test -f $path
then
does_exist=true
fi
echo "exists=$does_exist">> $GITHUB_OUTPUT
- name: "[INFO] Skip changelog generation"
if: steps.changelog.outputs.exists == 'true'
run: |
title="Skip changelog generation"
message="A changelog already exists at ${{ steps.changelog.outputs.path }}, skipping generating changelog"
echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message"
release-branch:
runs-on: ubuntu-latest
needs: release-inputs
if: |
needs.release-inputs.outputs.changelog-exists == 'false' ||
needs.release-inputs.outputs.version-is-current == 'false'
outputs:
name: ${{ steps.release-branch.outputs.name }}
steps:
- name: "Checkout ${{ github.event.repository.name }}@${{ inputs.branch }}"
uses: actions/checkout@v4
with:
ref: ${{ inputs.branch }}
- name: "Set release branch"
id: release-branch
run: |
name="prep-release/"
if [[ ${{ inputs.is-nightly-release }} == true ]]
then
name+="nightly-release/"
else
name+="${{ inputs.deploy-to }}/"
fi
name+="${{ inputs.version }}_$GITHUB_RUN_ID"
echo "name=$name" >> $GITHUB_OUTPUT
- name: "Create release branch ${{ steps.release-branch.outputs.name }}"
run: |
git checkout -b ${{ steps.release-branch.outputs.name }}
git push -u origin ${{ steps.release-branch.outputs.name }}
- name: "[INFO] Create release branch"
run: |
title="Create release branch"
message="Create release branch: ${{ steps.release-branch.outputs.name }}"
echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message"
core-team:
if: needs.release-inputs.outputs.changelog-exists == 'false'
needs: release-inputs
uses: dbt-labs/actions/.github/workflows/determine-team-membership.yml@main
with:
github_team: "core-group"
secrets: inherit
generate-changelog:
runs-on: ubuntu-latest
if: needs.release-inputs.outputs.changelog-exists == 'false'
# only runs if we need to make changes, determined by not skipping release-branch
needs:
- release-inputs
- release-branch
- core-team
steps:
- name: "Checkout ${{ github.event.repository.name }}@${{ needs.release-branch.outputs.name }}"
uses: actions/checkout@v4
with:
ref: ${{ needs.release-branch.outputs.name }}
- name: "Setup `hatch`"
uses: dbt-labs/dbt-adapters/.github/actions/setup-hatch@main
- name: "Install `changie`"
run: |
brew tap miniscruff/changie https://github.com/miniscruff/changie
brew install changie
- name: "Generate changelog at ${{ needs.release-inputs.outputs.changelog-path }}"
run: |
if [[ ${{ needs.release-inputs.outputs.is-pre-release }} -eq 1 ]]
then
changie batch ${{ needs.release-inputs.outputs.base-version }} \
--move-dir '${{ needs.release-inputs.outputs.base-version }}' \
--prerelease ${{ needs.release-inputs.outputs.pre-release }}
elif [[ -d ".changes/${{ needs.release-inputs.outputs.base-version }}" ]]
then
changie batch ${{ needs.release-inputs.outputs.base-version }} \
--include '${{ needs.release-inputs.outputs.base-version }}' \
--remove-prereleases
else # releasing a final patch with no pre-releases
changie batch ${{ needs.release-inputs.outputs.base-version }}
fi
changie merge
env:
CHANGIE_CORE_TEAM: ${{ needs.core-team.outputs.team_membership }}
- name: "Remove trailing whitespace and missing new lines"
# this step will fail on whitespace errors but also correct them
continue-on-error: true
run: hatch run code-quality
- name: "Commit & push changes"
run: |
git config user.name "$USER"
git config user.email "$EMAIL"
git pull
git add .
git commit -m "$COMMIT_MESSAGE"
git push
env:
USER: "GitHub Build Bot"
EMAIL: "buildbot@fishtownanalytics.com"
COMMIT_MESSAGE: "Generate changelog at ${{ needs.release-inputs.outputs.changelog-path }}"
- name: "[INFO] Generated changelog at ${{ needs.release-inputs.outputs.changelog-path }}"
run: |
title="Changelog generation"
if [[ -f ${{ needs.release-inputs.outputs.changelog-path }} ]]
then
message="Generated changelog file successfully"
echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message"
else
message="Failed to generate changelog file"
echo "::error title=${{ env.NOTIFICATION_PREFIX }}: $title::$message"
exit 1
fi
bump-version:
runs-on: ubuntu-latest
if: needs.release-inputs.outputs.version-is-current == 'false'
# only runs if we need to make changes, determined by not skipping release-branch
needs:
- release-inputs
- release-branch
- generate-changelog
steps:
- name: "Checkout ${{ github.event.repository.name }}@${{ needs.release-branch.outputs.name }}"
uses: actions/checkout@v4
with:
ref: ${{ needs.release-branch.outputs.name }}
- name: "Setup `hatch`"
uses: dbt-labs/dbt-adapters/.github/actions/setup-hatch@main
- name: "Bump version to ${{ inputs.version }}"
run: hatch version ${{ inputs.version }}
- name: "Commit & push changes"
run: |
git config user.name "$USER"
git config user.email "$EMAIL"
git pull
git add .
git commit -m "$COMMIT_MESSAGE"
git push
env:
USER: "GitHub Build Bot"
EMAIL: "buildbot@fishtownanalytics.com"
COMMIT_MESSAGE: "Bump version to ${{ inputs.version }}"
- name: "[INFO] Bumped version to ${{ inputs.version }}"
run: |
title="Version bump"
message="Bumped version to ${{ inputs.version }}"
echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message"
unit-tests:
runs-on: ubuntu-latest
# only run unit tests if we created a release branch and already bumped the version and generated the changelog
if: |
!failure() && !cancelled() &&
needs.release-branch.outputs.name != ''
needs:
- release-branch
- generate-changelog
- bump-version
steps:
- name: "Checkout ${{ github.event.repository.name }}@${{ needs.release-branch.outputs.name }}"
uses: actions/checkout@v4
with:
ref: ${{ needs.release-branch.outputs.name }}
- name: "Setup `hatch`"
uses: dbt-labs/dbt-adapters/.github/actions/setup-hatch@main
- name: "Run unit tests"
run: hatch run unit-tests
integration-tests:
runs-on: ubuntu-latest
# only run integration tests if we created a release branch and already bumped the version and generated the changelog
if: |
!failure() && !cancelled() &&
needs.release-branch.outputs.name != ''
needs:
- release-branch
- generate-changelog
- bump-version
steps:
- name: "Checkout ${{ github.event.repository.name }}@${{ needs.release-branch.outputs.name }}"
uses: actions/checkout@v4
with:
ref: ${{ needs.release-branch.outputs.name }}\
- name: "Set up `hatch`"
uses: dbt-labs/dbt-adapters/.github/actions/setup-hatch@main
- name: "Run integration tests"
run: hatch run integration-tests
merge-release-branch:
runs-on: ubuntu-latest
needs:
- unit-tests
- integration-tests
- release-branch
- release-inputs
if: |
!failure() && !cancelled() &&
needs.release-branch.result == 'success' &&
inputs.deploy-to == 'prod'
steps:
- name: "Checkout ${{ github.event.repository.name }}"
uses: actions/checkout@v4
- name: "Merge changes into ${{ inputs.branch }}"
uses: everlytic/branch-merge@1.1.5
with:
source_ref: ${{ needs.release-branch.outputs.name }}
target_branch: ${{ inputs.branch }}
github_token: ${{ secrets.FISHTOWN_BOT_PAT }}
commit_message_template: "[Automated] Merged {source_ref} into target {target_branch} during release process"
- name: "[INFO] Merge changes into ${{ inputs.branch }}"
run: |
title="Merge changes"
message="Merge ${{ needs.release-branch.outputs.name }} into ${{ inputs.branch }}"
echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message"
release:
runs-on: ubuntu-latest
needs:
- release-branch
- merge-release-branch
if: ${{ !failure() && !cancelled() }}
# Get the SHA that will be released.
# If the changelog already exists and the version was already current on the input branch, then release from there.
# Otherwise, we generated a changelog and/or did the version bump in this workflow and there is a
# new sha to use from the merge we just did. Grab that here instead.
outputs:
branch: ${{ steps.branch.outputs.name }}
sha: ${{ steps.sha.outputs.sha }}
steps:
- name: "Set release branch"
id: branch
# If a release branch was created and not merged, use the release branch
# Otherwise, use the input branch because either nothing was done, or the changes were merged back in
run: |
if [[ ${{ needs.release-branch.result == 'success' }} && ${{ needs.merge-release-branch.result == 'skipped' }} ]]; then
branch="${{ needs.release-branch.outputs.name }}"
else
branch="${{ inputs.branch }}"
fi
echo "name=$branch" >> $GITHUB_OUTPUT
- name: "Checkout ${{ github.event.repository.name }}@${{ steps.branch.outputs.name }}"
uses: actions/checkout@v4
with:
ref: ${{ steps.branch.outputs.name }}
- name: "Set release SHA"
id: sha
run: echo "sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT
# if this is a real release and a release branch was created, delete it
- name: "Delete release branch: ${{ needs.release-branch.outputs.name }}"
if: ${{ inputs.deploy-to == 'prod' && inputs.is-nightly-release == 'false' && needs.release-branch.outputs.name != '' }}
run: git push origin -d ${{ needs.release-branch.outputs.name }}

View File

@@ -1 +0,0 @@
recursive-include dbt/include *.sql *.yml *.md

View File

@@ -1,48 +0,0 @@
.DEFAULT_GOAL:=help
.PHONY: dev
dev: ## Installs adapter in develop mode along with development dependencies
@\
pip install -e . -r dev-requirements.txt && pre-commit install
.PHONY: dev-uninstall
dev-uninstall: ## Uninstalls all packages while maintaining the virtual environment
## Useful when updating versions, or if you accidentally installed into the system interpreter
pip freeze | grep -v "^-e" | cut -d "@" -f1 | xargs pip uninstall -y
pip uninstall -y dbt-snowflake
.PHONY: unit
unit: ## Runs unit tests with py39.
@\
tox -e py39
.PHONY: test
test: ## Runs unit tests with py39 and code checks against staged changes.
@\
tox -p -e py39; \
pre-commit run --all-files
.PHONY: integration
integration: ## Runs snowflake integration tests with py38.
@\
tox -e py39-snowflake --
.PHONY: clean
@echo "cleaning repo"
@git clean -f -X
.PHONY: help
help: ## Show this help message.
@echo 'usage: make [target]'
@echo
@echo 'targets:'
@grep -E '^[7+a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
.PHONY: docker-dev
docker-dev:
docker build -f docker/dev.Dockerfile -t dbt-snowflake-dev .
docker run --rm -it --name dbt-snowflake-dev -v $(shell pwd):/opt/code dbt-snowflake-dev
.PHONY: docker-prod
docker-prod:
docker build -f docker/Dockerfile -t dbt-snowflake .

View File

@@ -1,23 +0,0 @@
# install latest changes in dbt-core
git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-core&subdirectory=core
git+https://github.com/dbt-labs/dbt-adapters.git
git+https://github.com/dbt-labs/dbt-adapters.git#subdirectory=dbt-tests-adapter
git+https://github.com/dbt-labs/dbt-common.git
# dev
ipdb~=0.13.13
pre-commit~=3.7.0
# test
ddtrace==2.3.0
pytest~=7.4
pytest-csv~=3.0
pytest-dotenv~=0.5.2
pytest-logbook~=1.2
pytest-xdist~=3.6
tox~=4.16
# build
bumpversion~=0.6.0
twine~=5.1
wheel~=0.43

62
hatch.toml Normal file
View File

@@ -0,0 +1,62 @@
[version]
path = "dbt/adapters/snowflake/__version__.py"
[build.targets.sdist]
packages = ["src/dbt"]
sources = ["src"]
[build.targets.wheel]
packages = ["src/dbt"]
sources = ["src"]
[envs.default]
dependencies = [
"dbt-adapters @ git+https://github.com/dbt-labs/dbt-adapters.git",
"dbt-common @ git+https://github.com/dbt-labs/dbt-common.git",
"dbt-tests-adapter @ git+https://github.com/dbt-labs/dbt-adapters.git#subdirectory=dbt-tests-adapter",
"dbt-core @ git+https://github.com/dbt-labs/dbt-core.git#subdirectory=core",
"ddtrace==2.3.0",
"ipdb~=0.13.13",
"pre-commit~=3.7.0",
"pytest~=7.4",
"pytest-csv~=3.0",
"pytest-dotenv~=0.5.2",
"pytest-logbook~=1.2",
"pytest-xdist~=3.6",
"tox~=4.16", # does this pin deps transitively?
]
[envs.default.scripts]
setup = "pre-commit install"
code-quality = "pre-commit run --all-files"
unit-tests = "python -m pytest {args:tests/unit}"
integration-tests = "- python -m pytest {args:tests/functional}"
docker-dev = [
"docker build -f docker/dev.Dockerfile -t dbt-snowflake-dev .",
"docker run --rm -it --name dbt-snowflake-dev -v $(pwd):/opt/code dbt-snowflake-dev",
]
[envs.build]
detached = true
dependencies = [
"wheel",
"twine",
"check-wheel-contents",
]
[envs.build.scripts]
check-all = [
"- check-wheel",
"- check-sdist",
]
check-wheel = [
"twine check dist/*",
"find ./dist/dbt_snowflake-*.whl -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/",
"pip freeze | grep dbt-snowflake",
]
check-sdist = [
"check-wheel-contents dist/*.whl --ignore W007,W008",
"find ./dist/dbt_snowflake-*.gz -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/",
"pip freeze | grep dbt-snowflake",
]
docker-prod = "docker build -f docker/Dockerfile -t dbt-snowflake ."

48
pyproject.toml Normal file
View File

@@ -0,0 +1,48 @@
[build-system]
requires = ["hatchling"]
build-backend = "hatchling.build"
[project]
dynamic = ["version"]
name = "dbt-snowflake"
description = "The Snowflake adapter plugin for dbt"
readme = "README.md"
keywords = ["dbt", "adapter", "adapters", "database", "elt", "dbt-core", "dbt Core", "dbt Cloud", "dbt Labs", "snowflake"]
requires-python = ">=3.9.0"
authors = [{ name = "dbt Labs", email = "info@dbtlabs.com" }]
maintainers = [{ name = "dbt Labs", email = "info@dbtlabs.com" }]
classifiers = [
"Development Status :: 5 - Production/Stable",
"License :: OSI Approved :: Apache Software License",
"Operating System :: MacOS :: MacOS X",
"Operating System :: Microsoft :: Windows",
"Operating System :: POSIX :: Linux",
"Programming Language :: Python :: 3.9",
"Programming Language :: Python :: 3.10",
"Programming Language :: Python :: 3.11",
"Programming Language :: Python :: 3.12",
]
dependencies = [
"dbt-common>=1.10,<2.0",
"dbt-adapters>=1.10.4,<2.0",
"snowflake-connector-python[secure-local-storage]>=3.0.0,<3.12.4",
# add dbt-core to ensure backwards compatibility of installation, this is not a functional dependency
"dbt-core>=1.8.0",
# installed via dbt-core but referenced directly; don't pin to avoid version conflicts with dbt-core
"agate",
]
[project.urls]
Homepage = "https://github.com/dbt-labs/dbt-snowflake"
Documentation = "https://docs.getdbt.com"
Repository = "https://github.com/dbt-labs/dbt-snowflake.git"
Issues = "https://github.com/dbt-labs/dbt-snowflake/issues"
Changelog = "https://github.com/dbt-labs/dbt-snowflake/blob/main/CHANGELOG.md"
[tool.pytest.ini_options]
testpaths = ["tests/functional", "tests/unit"]
env_files = ["test.env"]
addopts = "-v --color=yes -n auto"
filterwarnings = [
"ignore:datetime.datetime.utcnow:DeprecationWarning",
]

View File

@@ -1,10 +0,0 @@
[pytest]
filterwarnings =
ignore:.*'soft_unicode' has been renamed to 'soft_str'*:DeprecationWarning
ignore:unclosed file .*:ResourceWarning
env_files =
test.env
testpaths =
tests/unit
tests/integration
tests/functional

View File

@@ -1,81 +0,0 @@
#!/usr/bin/env python
import os
from pathlib import Path
import sys
# require python 3.8 or newer
if sys.version_info < (3, 9):
print("Error: dbt does not support this version of Python.")
print("Please upgrade to Python 3.9 or higher.")
sys.exit(1)
# require version of setuptools that supports find_namespace_packages
from setuptools import setup
try:
from setuptools import find_namespace_packages
except ImportError:
# the user has a downlevel version of setuptools.
print("Error: dbt requires setuptools v40.1.0 or higher.")
print('Please upgrade setuptools with "pip install --upgrade setuptools" ' "and try again")
sys.exit(1)
# pull long description from README
this_directory = os.path.abspath(os.path.dirname(__file__))
with open(os.path.join(this_directory, "README.md")) as f:
long_description = f.read()
# used for this adapter's version
VERSION = Path(__file__).parent / "dbt/adapters/snowflake/__version__.py"
def _plugin_version() -> str:
"""
Pull the package version from the main package version file
"""
attributes = {}
exec(VERSION.read_text(), attributes)
return attributes["version"]
package_name = "dbt-snowflake"
description = """The Snowflake adapter plugin for dbt"""
setup(
name=package_name,
version=_plugin_version(),
description=description,
long_description=long_description,
long_description_content_type="text/markdown",
author="dbt Labs",
author_email="info@dbtlabs.com",
url="https://github.com/dbt-labs/dbt-snowflake",
packages=find_namespace_packages(include=["dbt", "dbt.*"]),
include_package_data=True,
install_requires=[
"dbt-common>=1.10,<2.0",
"dbt-adapters>=1.10.4,<2.0",
"snowflake-connector-python[secure-local-storage]~=3.0",
# add dbt-core to ensure backwards compatibility of installation, this is not a functional dependency
"dbt-core>=1.8.0",
# installed via dbt-core but referenced directly; don't pin to avoid version conflicts with dbt-core
"agate",
],
zip_safe=False,
classifiers=[
"Development Status :: 5 - Production/Stable",
"License :: OSI Approved :: Apache Software License",
"Operating System :: Microsoft :: Windows",
"Operating System :: MacOS :: MacOS X",
"Operating System :: POSIX :: Linux",
"Programming Language :: Python :: 3.9",
"Programming Language :: Python :: 3.10",
"Programming Language :: Python :: 3.11",
"Programming Language :: Python :: 3.12",
],
python_requires=">=3.9",
)

32
tox.ini
View File

@@ -1,32 +0,0 @@
[tox]
skipsdist = True
envlist = py39,py310,py311,py312
[testenv:{unit,py39,py310,py311,py312,py}]
description = unit testing
skip_install = true
passenv =
DBT_*
PYTEST_ADDOPTS
commands = {envpython} -m pytest {posargs} tests/unit
deps =
-rdev-requirements.txt
-e.
[testenv:{integration,py39,py310,py311,py312,py}-{snowflake}]
description = adapter plugin integration testing
skip_install = true
passenv =
DBT_*
SNOWFLAKE_TEST_*
PYTEST_ADDOPTS
DD_CIVISIBILITY_AGENTLESS_ENABLED
DD_API_KEY
DD_SITE
DD_ENV
DD_SERVICE
commands =
snowflake: {envpython} -m pytest {posargs} tests/functional
deps =
-rdev-requirements.txt
-e.