Compare commits

..

6 Commits

Author SHA1 Message Date
Gerda Shank
0baf0c8dd2 Refactor normalize_warn_error_options 2024-07-17 14:53:36 -04:00
Gerda Shank
7310b79088 Replace conversion method 2024-07-17 09:42:20 -04:00
Gerda Shank
89c01dab09 Fix unit test 2024-07-16 18:41:27 -04:00
Gerda Shank
b47ac8e1d3 Changie 2024-07-16 17:14:33 -04:00
Gerda Shank
254b06f1b8 Add test 2024-07-16 17:13:39 -04:00
Gerda Shank
5108a4cf2e Fix exclusive_primary_alt_value_setting to set warn_error_options correctly 2024-07-16 17:13:23 -04:00
4731 changed files with 77054 additions and 76010 deletions

37
.bumpversion.cfg Normal file
View File

@@ -0,0 +1,37 @@
[bumpversion]
current_version = 1.9.0a1
parse = (?P<major>[\d]+) # major version number
\.(?P<minor>[\d]+) # minor version number
\.(?P<patch>[\d]+) # patch version number
(?P<prerelease> # optional pre-release - ex: a1, b2, rc25
(?P<prekind>a|b|rc) # pre-release type
(?P<num>[\d]+) # pre-release version number
)?
( # optional nightly release indicator
\.(?P<nightly>dev[0-9]+) # ex: .dev02142023
)? # expected matches: `1.15.0`, `1.5.0a11`, `1.5.0a1.dev123`, `1.5.0.dev123457`, expected failures: `1`, `1.5`, `1.5.2-a1`, `text1.5.0`
serialize =
{major}.{minor}.{patch}{prekind}{num}.{nightly}
{major}.{minor}.{patch}.{nightly}
{major}.{minor}.{patch}{prekind}{num}
{major}.{minor}.{patch}
commit = False
tag = False
[bumpversion:part:prekind]
first_value = a
optional_value = final
values =
a
b
rc
final
[bumpversion:part:num]
first_value = 1
[bumpversion:part:nightly]
[bumpversion:file:core/setup.py]
[bumpversion:file:core/dbt/version.py]

View File

@@ -3,9 +3,6 @@
For information on prior major and minor releases, see their changelogs:
* [1.10](https://github.com/dbt-labs/dbt-core/blob/1.10.latest/CHANGELOG.md)
* [1.9](https://github.com/dbt-labs/dbt-core/blob/1.9.latest/CHANGELOG.md)
* [1.8](https://github.com/dbt-labs/dbt-core/blob/1.8.latest/CHANGELOG.md)
* [1.7](https://github.com/dbt-labs/dbt-core/blob/1.7.latest/CHANGELOG.md)
* [1.6](https://github.com/dbt-labs/dbt-core/blob/1.6.latest/CHANGELOG.md)
* [1.5](https://github.com/dbt-labs/dbt-core/blob/1.5.latest/CHANGELOG.md)

View File

@@ -0,0 +1,6 @@
kind: Dependencies
body: Remove logbook dependency
time: 2024-05-09T09:37:17.745129-05:00
custom:
Author: emmyoop
Issue: "8027"

View File

@@ -1,6 +0,0 @@
kind: Dependencies
body: Use EventCatcher from dbt-common instead of maintaining a local copy
time: 2025-11-18T15:53:54.284561+05:30
custom:
Author: 3loka
Issue: "12124"

View File

@@ -0,0 +1,6 @@
kind: Docs
body: Enable display of unit tests
time: 2024-03-11T14:03:44.490834-04:00
custom:
Author: gshank
Issue: "501"

View File

@@ -0,0 +1,6 @@
kind: Docs
body: Unit tests not rendering
time: 2024-05-01T02:10:50.987412+02:00
custom:
Author: aranke
Issue: "506"

View File

@@ -0,0 +1,6 @@
kind: Docs
body: Add support for Saved Query node
time: 2024-05-16T22:30:36.206492-07:00
custom:
Author: ChenyuLInx
Issue: "486"

View File

@@ -0,0 +1,6 @@
kind: Docs
body: Fix npm security vulnerabilities as of June 2024
time: 2024-06-13T15:10:48.301989+01:00
custom:
Author: aranke
Issue: "513"

View File

@@ -0,0 +1,6 @@
kind: Features
body: serialize inferred primary key
time: 2024-05-06T17:56:42.757673-05:00
custom:
Author: dave-connors-3
Issue: "9824"

View File

@@ -0,0 +1,6 @@
kind: Features
body: 'Add unit_test: selection method'
time: 2024-05-07T16:27:17.047585-04:00
custom:
Author: michelleark
Issue: "10053"

View File

@@ -0,0 +1,7 @@
kind: Features
body: Maximally parallelize dbt clone
in clone command"
time: 2024-05-22T00:03:09.765977-04:00
custom:
Author: michelleark
Issue: "7914"

View File

@@ -0,0 +1,6 @@
kind: Features
body: Add --host flag to dbt docs serve, defaulting to '127.0.0.1'
time: 2024-05-27T12:44:05.040843-04:00
custom:
Author: michelleark
Issue: "10229"

View File

@@ -0,0 +1,6 @@
kind: Features
body: Update data_test to accept arbitrary config options
time: 2024-05-31T15:08:16.431966-05:00
custom:
Author: McKnight-42
Issue: "10197"

View File

@@ -0,0 +1,6 @@
kind: Features
body: add pre_model and post_model hook calls to data and unit tests to be able to provide extra config options
time: 2024-06-06T11:23:34.758675-05:00
custom:
Author: McKnight-42
Issue: "10198"

View File

@@ -0,0 +1,6 @@
kind: Features
body: add --empty value to jinja context as flags.EMPTY
time: 2024-06-17T10:39:48.275801-04:00
custom:
Author: michelleark
Issue: "10317"

View File

@@ -0,0 +1,6 @@
kind: Features
body: Support cumulative_type_params & sub-daily granularities in semantic manifest.
time: 2024-06-25T09:51:07.983248-07:00
custom:
Author: courtneyholcomb
Issue: "10360"

View File

@@ -0,0 +1,6 @@
kind: Features
body: Add time_granularity to metric spec.
time: 2024-06-27T16:29:53.500917-07:00
custom:
Author: courtneyholcomb
Issue: "10376"

View File

@@ -1,6 +0,0 @@
kind: Features
body: Support partial parsing for function nodes
time: 2025-10-06T14:03:52.258104-05:00
custom:
Author: QMalcolm
Issue: "12072"

View File

@@ -1,6 +0,0 @@
kind: Features
body: Allow for defining funciton arguments with default values
time: 2025-11-17T14:10:53.860178-06:00
custom:
Author: QMalcolm
Issue: "12044"

View File

@@ -1,6 +0,0 @@
kind: Features
body: Raise jsonschema-based deprecation warnings by default
time: 2025-12-01T16:52:09.354436-05:00
custom:
Author: michelleark
Issue: 12240

View File

@@ -1,6 +0,0 @@
kind: Features
body: ':bug: :snowman: Disable unit tests whose model is disabled'
time: 2025-12-03T12:29:26.209248-05:00
custom:
Author: michelleark
Issue: "10540"

View File

@@ -1,6 +0,0 @@
kind: Features
body: Implement config.meta_get and config.meta_require
time: 2025-12-10T20:20:01.354288-05:00
custom:
Author: gshank
Issue: "12012"

View File

@@ -0,0 +1,6 @@
kind: Fixes
body: Remove unused check_new method
time: 2023-06-01T20:41:57.556342+02:00
custom:
Author: kevinneville
Issue: "7586"

View File

@@ -0,0 +1,6 @@
kind: Fixes
body: Convert "Skipping model due to fail_fast" message to DEBUG level
time: 2024-01-13T07:36:15.836294-00:00
custom:
Author: scottgigante,nevdelap
Issue: "8774"

View File

@@ -0,0 +1,7 @@
kind: Fixes
body: 'Restore previous behavior for --favor-state: only favor defer_relation if not
selected in current command"'
time: 2024-05-08T15:11:27.510912+02:00
custom:
Author: jtcohen6
Issue: "10107"

View File

@@ -0,0 +1,6 @@
kind: Fixes
body: Unit test fixture (csv) returns null for empty value
time: 2024-05-09T09:14:11.772709-04:00
custom:
Author: michelleark
Issue: "9881"

View File

@@ -0,0 +1,7 @@
kind: Fixes
body: Fix json format log and --quiet for ls and jinja print by converting print call
to fire events
time: 2024-05-16T15:39:13.896723-07:00
custom:
Author: ChenyuLInx
Issue: "8756"

View File

@@ -0,0 +1,6 @@
kind: Fixes
body: Add resource type to saved_query
time: 2024-05-16T22:35:10.287514-07:00
custom:
Author: ChenyuLInx
Issue: "10168"

View File

@@ -0,0 +1,7 @@
kind: Fixes
body: 'Fix: Order-insensitive unit test equality assertion for expected/actual with
multiple nulls'
time: 2024-05-22T18:28:55.91733-04:00
custom:
Author: michelleark
Issue: "10167"

View File

@@ -0,0 +1,6 @@
kind: Fixes
body: Renaming or removing a contracted model should raise a BreakingChange warning/error
time: 2024-05-23T20:42:51.033946-04:00
custom:
Author: michelleark
Issue: "10116"

View File

@@ -0,0 +1,6 @@
kind: Fixes
body: prefer disabled project nodes to external node
time: 2024-05-24T13:11:35.440443-04:00
custom:
Author: michelleark
Issue: "10224"

View File

@@ -0,0 +1,6 @@
kind: Fixes
body: Fix issues with selectors and inline nodes
time: 2024-06-05T11:16:52.187667-04:00
custom:
Author: gshank
Issue: 8943 9269

View File

@@ -0,0 +1,6 @@
kind: Fixes
body: Fix snapshot config to work in yaml files
time: 2024-06-07T13:46:48.383215-04:00
custom:
Author: gshank
Issue: "4000"

View File

@@ -0,0 +1,6 @@
kind: Fixes
body: Improve handling of error when loading schema file list
time: 2024-06-10T13:21:30.963371-04:00
custom:
Author: gshank
Issue: "10284"

View File

@@ -0,0 +1,6 @@
kind: Fixes
body: Saved Query node fail during skip
time: 2024-06-12T12:42:56.329073-07:00
custom:
Author: ChenyuLInx
Issue: "10029"

View File

@@ -0,0 +1,6 @@
kind: Fixes
body: Implement state:modified for saved queries
time: 2024-06-12T15:21:39.851426-04:00
custom:
Author: gshank
Issue: "10294"

View File

@@ -0,0 +1,6 @@
kind: Fixes
body: DOn't warn on `unit_test` config paths that are properly used
time: 2024-06-13T18:31:17.486497-07:00
custom:
Author: QMalcolm
Issue: "10311"

View File

@@ -0,0 +1,6 @@
kind: Fixes
body: Fix setting `silence` of `warn_error_options` via `dbt_project.yaml` flags
time: 2024-06-24T17:17:29.464865-07:00
custom:
Author: QMalcolm
Issue: "10160"

View File

@@ -0,0 +1,6 @@
kind: Fixes
body: Limit data_tests deprecation to root_project
time: 2024-06-27T15:44:48.579869-04:00
custom:
Author: gshank
Issue: "9835"

View File

@@ -0,0 +1,6 @@
kind: Fixes
body: CLI flags should take precedence over env var flags
time: 2024-07-09T17:24:40.918977-04:00
custom:
Author: gshank
Issue: "10304"

View File

@@ -0,0 +1,6 @@
kind: Fixes
body: Fix error constructing warn_error_options
time: 2024-07-16T17:14:27.837171-04:00
custom:
Author: gshank
Issue: "10452"

View File

@@ -1,6 +0,0 @@
kind: Fixes
body: Address Click 8.2+ deprecation warning
time: 2025-09-22T15:17:26.983151-06:00
custom:
Author: edgarrmondragon
Issue: "12038"

View File

@@ -1,6 +0,0 @@
kind: Fixes
body: Include macros in unit test parsing
time: 2025-11-17T14:06:49.518566-05:00
custom:
Author: michelleark nathanskone
Issue: "10157"

View File

@@ -1,6 +0,0 @@
kind: Fixes
body: Allow dbt deps to run when vars lack defaults in dbt_project.yml
time: 2025-11-17T18:50:25.759091+05:30
custom:
Author: 3loka
Issue: "8913"

View File

@@ -1,6 +0,0 @@
kind: Fixes
body: Restore DuplicateResourceNameError for intra-project node name duplication, behind behavior flag `require_unique_project_resource_names`
time: 2025-11-18T17:11:06.454784-05:00
custom:
Author: michelleark
Issue: "12152"

View File

@@ -1,6 +0,0 @@
kind: Fixes
body: Allow the usage of `function` with `--exclude-resource-type` flag
time: 2025-11-19T19:50:34.703236-06:00
custom:
Author: QMalcolm
Issue: "12143"

View File

@@ -1,6 +0,0 @@
kind: Fixes
body: Fix bug where schemas of functions weren't guaranteed to exist
time: 2025-11-24T15:56:29.467004-06:00
custom:
Author: QMalcolm
Issue: "12142"

View File

@@ -1,6 +0,0 @@
kind: Fixes
body: Fix generation of deprecations summary
time: 2025-11-24T15:57:56.544123-08:00
custom:
Author: asiunov
Issue: "12146"

View File

@@ -1,6 +0,0 @@
kind: Fixes
body: ':bug: :snowman: Correctly reference foreign key references when --defer and --state provided'
time: 2025-11-24T17:08:55.387946-05:00
custom:
Author: michellark
Issue: "11885"

View File

@@ -1,7 +0,0 @@
kind: Fixes
body: ':bug: :snowman: Add exception when using --state and referring to a removed
test'
time: 2025-11-25T12:02:46.635026-05:00
custom:
Author: emmyoop
Issue: "10630"

View File

@@ -1,6 +0,0 @@
kind: Fixes
body: ':bug: :snowman: Stop emitting `NoNodesForSelectionCriteria` three times during `build` command'
time: 2025-11-25T12:20:20.132379-06:00
custom:
Author: QMalcolm
Issue: "11627"

View File

@@ -1,6 +0,0 @@
kind: Fixes
body: ":bug: :snowman: Fix long Python stack traces appearing when package dependencies have incompatible version requirements"
time: 2025-11-27T14:13:08.082542-05:00
custom:
Author: emmyoop
Issue: "12049"

View File

@@ -1,7 +0,0 @@
kind: Fixes
body: ':bug: :snowman: Fixed issue where changing data type size/precision/scale (e.g.,
varchar(3) to varchar(10)) incorrectly triggered a breaking change error fo'
time: 2025-11-27T14:59:29.256274-05:00
custom:
Author: emmyoop
Issue: "11186"

View File

@@ -1,6 +0,0 @@
kind: Fixes
body: ':bug: :snowman: Support unit testing models that depend on sources with the same name'
time: 2025-11-27T17:01:24.193516-05:00
custom:
Author: michelleark
Issue: 11975 10433

View File

@@ -1,6 +0,0 @@
kind: Fixes
body: Fix bug in partial parsing when updating a model with a schema file that is referenced by a singular test
time: 2025-11-28T10:21:29.911147Z
custom:
Author: mattogburke
Issue: "12223"

View File

@@ -1,6 +0,0 @@
kind: Fixes
body: ':bug: :snowman: Avoid retrying successful run-operation commands'
time: 2025-11-28T12:28:38.546261-05:00
custom:
Author: michelleark
Issue: "11850"

View File

@@ -1,7 +0,0 @@
kind: Fixes
body: ':bug: :snowman: Fix `dbt deps --add-package` crash when packages.yml contains `warn-unpinned:
false`'
time: 2025-11-28T16:19:37.608722-05:00
custom:
Author: emmyoop
Issue: "9104"

View File

@@ -1,7 +0,0 @@
kind: Fixes
body: ':bug: :snowman: Improve `dbt deps --add-package` duplicate detection with better
cross-source matching and word boundaries'
time: 2025-11-28T16:31:44.344099-05:00
custom:
Author: emmyoop
Issue: "12239"

View File

@@ -1,6 +0,0 @@
kind: Fixes
body: ':bug: :snowman: Fix false positive deprecation warning of pre/post-hook SQL configs'
time: 2025-12-02T13:37:05.012112-05:00
custom:
Author: michelleark
Issue: "12244"

View File

@@ -1,6 +0,0 @@
kind: Fixes
body: Ensure recent deprecation warnings include event name in message
time: 2025-12-09T17:50:31.334618-06:00
custom:
Author: QMalcolm
Issue: "12264"

View File

@@ -1,6 +0,0 @@
kind: Fixes
body: Improve error message clarity when detecting nodes with space in name
time: 2025-12-10T14:39:35.107841-08:00
custom:
Author: michelleark
Issue: "11835"

View File

@@ -0,0 +1,6 @@
kind: Security
body: Explicitly bind to localhost in docs serve
time: 2024-05-22T09:45:40.748185-04:00
custom:
Author: ChenyuLInx michelleark
Issue: "10209"

View File

@@ -0,0 +1,6 @@
kind: Under the Hood
body: Clear error message for Private package in dbt-core
time: 2024-05-02T15:44:30.713097-07:00
custom:
Author: ChenyuLInx
Issue: "10083"

View File

@@ -0,0 +1,6 @@
kind: Under the Hood
body: Enable use of context in serialization
time: 2024-05-06T14:55:11.1812-04:00
custom:
Author: gshank
Issue: "10093"

View File

@@ -0,0 +1,6 @@
kind: Under the Hood
body: Make RSS high water mark measurement more accurate on Linux
time: 2024-05-19T15:59:46.700842315-04:00
custom:
Author: peterallenwebb
Issue: "10177"

View File

@@ -0,0 +1,6 @@
kind: Under the Hood
body: Enable record filtering by type.
time: 2024-05-29T10:28:14.547624-05:00
custom:
Author: emmyoop
Issue: "10240"

View File

@@ -0,0 +1,6 @@
kind: Under the Hood
body: Remove IntermediateSnapshotNode
time: 2024-06-18T14:06:52.618602-04:00
custom:
Author: gshank
Issue: "10326"

View File

@@ -0,0 +1,6 @@
kind: Under the Hood
body: Additional logging for skipped ephemeral models
time: 2024-07-01T13:17:50.827788-04:00
custom:
Author: gshank
Issue: "10389"

View File

@@ -1,6 +0,0 @@
kind: Under the Hood
body: Update jsonschemas for schema.yml and dbt_project.yml deprecations
time: 2025-11-19T11:01:10.616676-05:00
custom:
Author: michelleark
Issue: "12180"

View File

@@ -1,6 +0,0 @@
kind: Under the Hood
body: Replace setuptools and tox with hatch for build, test, and environment management.
time: 2025-11-21T14:05:15.838252-05:00
custom:
Author: emmyoop
Issue: "12151"

View File

@@ -1,6 +0,0 @@
kind: Under the Hood
body: Add add_catalog_integration call even if we have a pre-existing manifest
time: 2025-12-09T13:18:57.043254-08:00
custom:
Author: colin-rogers-dbt
Issue: "12262"

View File

@@ -41,26 +41,32 @@ newlines:
endOfVersion: 1
custom:
- key: Author
label: GitHub Username(s) (separated by a single space if multiple)
type: string
minLength: 3
- key: Issue
label: GitHub Issue Number (separated by a single space if multiple)
type: string
minLength: 1
- key: Author
label: GitHub Username(s) (separated by a single space if multiple)
type: string
minLength: 3
- key: Issue
label: GitHub Issue Number (separated by a single space if multiple)
type: string
minLength: 1
footerFormat: |
{{- $contributorDict := dict }}
{{- /* ensure we always skip snyk and dependabot */}}
{{- $bots := list "dependabot[bot]" "snyk-bot"}}
{{- /* ensure all names in this list are all lowercase for later matching purposes */}}
{{- $core_team := splitList " " .Env.CORE_TEAM }}
{{- /* ensure we always skip snyk and dependabot in addition to the core team */}}
{{- $maintainers := list "dependabot[bot]" "snyk-bot"}}
{{- range $team_member := $core_team }}
{{- $team_member_lower := lower $team_member }}
{{- $maintainers = append $maintainers $team_member_lower }}
{{- end }}
{{- range $change := .Changes }}
{{- $authorList := splitList " " $change.Custom.Author }}
{{- /* loop through all authors for a single changelog */}}
{{- range $author := $authorList }}
{{- $authorLower := lower $author }}
{{- /* we only want to include non-bot contributors */}}
{{- if not (has $authorLower $bots)}}
{{- /* we only want to include non-core team contributors */}}
{{- if not (has $authorLower $maintainers)}}
{{- $changeList := splitList " " $change.Custom.Author }}
{{- $IssueList := list }}
{{- $changeLink := $change.Kind }}

View File

@@ -7,8 +7,8 @@ ignore =
W503 # makes Flake8 work like black
W504
E203 # makes Flake8 work like black
E704 # makes Flake8 work like black
E741
E501 # long line checking is done in black
exclude = test/
per-file-ignores =
*/__init__.py: F401

View File

@@ -61,8 +61,8 @@ body:
label: Environment
description: |
examples:
- **OS**: Ubuntu 24.04
- **Python**: 3.10.12 (`python3 --version`)
- **OS**: Ubuntu 20.04
- **Python**: 3.9.12 (`python3 --version`)
- **dbt-core**: 1.1.1 (`dbt --version`)
value: |
- OS:

View File

@@ -12,6 +12,15 @@ contact_links:
- name: Participate in Discussions
url: https://github.com/dbt-labs/dbt-core/discussions
about: Do you have a Big Idea for dbt? Read open discussions, or start a new one
- name: Create an issue for adapters
url: https://github.com/dbt-labs/dbt-adapters/issues/new/choose
about: Report a bug or request a feature for an adapter
- name: Create an issue for dbt-redshift
url: https://github.com/dbt-labs/dbt-redshift/issues/new/choose
about: Report a bug or request a feature for dbt-redshift
- name: Create an issue for dbt-bigquery
url: https://github.com/dbt-labs/dbt-bigquery/issues/new/choose
about: Report a bug or request a feature for dbt-bigquery
- name: Create an issue for dbt-snowflake
url: https://github.com/dbt-labs/dbt-snowflake/issues/new/choose
about: Report a bug or request a feature for dbt-snowflake
- name: Create an issue for dbt-spark
url: https://github.com/dbt-labs/dbt-spark/issues/new/choose
about: Report a bug or request a feature for dbt-spark

View File

@@ -55,8 +55,8 @@ body:
label: Environment
description: |
examples:
- **OS**: Ubuntu 24.04
- **Python**: 3.10.12 (`python3 --version`)
- **OS**: Ubuntu 20.04
- **Python**: 3.9.12 (`python3 --version`)
- **dbt-core (working version)**: 1.1.1 (`dbt --version`)
- **dbt-core (regression version)**: 1.2.0 (`dbt --version`)
value: |

8
.github/_README.md vendored
View File

@@ -120,7 +120,7 @@ Some triggers of note that we use:
```yaml
jobs:
dependency_changelog:
runs-on: ${{ vars.UBUNTU_LATEST }}
runs-on: ubuntu-latest
steps:
- name: Get File Name Timestamp
@@ -188,12 +188,6 @@ ___
- The [GitHub CLI](https://cli.github.com/) is available in the default runners
- Actions run in your context. ie, using an action from the marketplace that uses the GITHUB_TOKEN uses the GITHUB_TOKEN generated by your workflow run.
### Runners
- We dynamically set runners based on repository vars. Admins can view repository vars and reset them. Current values are the following but are subject to change:
- `vars.UBUNTU_LATEST` -> `ubuntu-latest`
- `vars.WINDOWS_LATEST` -> `windows-latest`
- `vars.MACOS_LATEST` -> `macos-14`
### Actions from the Marketplace
- Dont use external actions for things that can easily be accomplished manually.
- Always read through what an external action does before using it! Often an action in the GitHub Actions Marketplace can be replaced with a few lines in bash. This is much more maintainable (and wont change under us) and clear as to whats actually happening. It also prevents any

View File

@@ -33,7 +33,7 @@ on:
jobs:
build:
runs-on: ${{ vars.UBUNTU_LATEST }}
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- name: Wrangle latest tag

View File

@@ -3,24 +3,24 @@ on:
workflow_dispatch:
inputs:
package:
description: The package to publish
required: true
description: The package to publish
required: true
version_number:
description: The version number
required: true
description: The version number
required: true
jobs:
build:
runs-on: ${{ vars.UBUNTU_LATEST }}
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- name: Wrangle latest tag
id: is_latest
uses: ./.github/actions/latest-wrangler
with:
package: ${{ github.event.inputs.package }}
new_version: ${{ github.event.inputs.new_version }}
gh_token: ${{ secrets.GITHUB_TOKEN }}
- name: Print the results
run: |
echo "Is it latest? Survey says: ${{ steps.is_latest.outputs.latest }} !"
- uses: actions/checkout@v3
- name: Wrangle latest tag
id: is_latest
uses: ./.github/actions/latest-wrangler
with:
package: ${{ github.event.inputs.package }}
new_version: ${{ github.event.inputs.new_version }}
gh_token: ${{ secrets.GITHUB_TOKEN }}
- name: Print the results
run: |
echo "Is it latest? Survey says: ${{ steps.is_latest.outputs.latest }} !"

View File

@@ -1,10 +1,9 @@
import os
from packaging.version import Version, parse
import requests
import sys
from typing import List
import requests
from packaging.version import Version, parse
def main():
package_name: str = os.environ["INPUT_PACKAGE_NAME"]

View File

@@ -0,0 +1,10 @@
name: "Set up postgres (linux)"
description: "Set up postgres service on linux vm for dbt integration tests"
runs:
using: "composite"
steps:
- shell: bash
run: |
sudo systemctl start postgresql.service
pg_isready
sudo -u postgres bash ${{ github.action_path }}/setup_db.sh

View File

@@ -0,0 +1 @@
../../../test/setup_db.sh

View File

@@ -0,0 +1,24 @@
name: "Set up postgres (macos)"
description: "Set up postgres service on macos vm for dbt integration tests"
runs:
using: "composite"
steps:
- shell: bash
run: |
brew services start postgresql
echo "Check PostgreSQL service is running"
i=10
COMMAND='pg_isready'
while [ $i -gt -1 ]; do
if [ $i == 0 ]; then
echo "PostgreSQL service not ready, all attempts exhausted"
exit 1
fi
echo "Check PostgreSQL service status"
eval $COMMAND && break
echo "PostgreSQL service not ready, wait 10 more sec, attempts left: $i"
sleep 10
((i--))
done
createuser -s postgres
bash ${{ github.action_path }}/setup_db.sh

View File

@@ -0,0 +1 @@
../../../test/setup_db.sh

View File

@@ -5,22 +5,8 @@ runs:
steps:
- shell: pwsh
run: |
Write-Host -Object "Installing PostgreSQL 16 as windows service..."
$installerArgs = @("--install_runtimes 0", "--superpassword root", "--enable_acledit 1", "--unattendedmodeui none", "--mode unattended")
$filePath = Invoke-DownloadWithRetry -Url "https://get.enterprisedb.com/postgresql/postgresql-16.1-1-windows-x64.exe" -Path "$env:PGROOT/postgresql-16.1-1-windows-x64.exe"
Start-Process -FilePath $filePath -ArgumentList $installerArgs -Wait -PassThru
Write-Host -Object "Validating PostgreSQL 16 Install..."
Get-Service -Name postgresql*
$pgReady = Start-Process -FilePath "$env:PGBIN\pg_isready" -Wait -PassThru
$exitCode = $pgReady.ExitCode
if ($exitCode -ne 0) {
Write-Host -Object "PostgreSQL is not ready. Exitcode: $exitCode"
exit $exitCode
}
Write-Host -Object "Starting PostgreSQL 16 Service..."
$pgService = Get-Service -Name postgresql-x64-16
$pgService = Get-Service -Name postgresql*
Set-Service -InputObject $pgService -Status running -StartupType automatic
Start-Process -FilePath "$env:PGBIN\pg_isready" -Wait -PassThru
$env:Path += ";$env:PGBIN"
bash ${{ github.action_path }}/setup_db.sh

View File

@@ -1 +1 @@
../../../scripts/setup_db.sh
../../../test/setup_db.sh

View File

@@ -1,169 +0,0 @@
# **what?**
# Runs all tests in dbt-postgres with this branch of dbt-core to ensure nothing is broken
# **why?**
# Ensure dbt-core changes do not break dbt-postgres, as a basic proxy for other adapters
# **when?**
# This will run when trying to merge a PR into main.
# It can also be manually triggered.
# This workflow can be skipped by adding the "Skip Postgres Testing" label to the PR. This is
# useful when making a change in both `dbt-postgres` and `dbt-core` where the changes are dependant
# and cause the other repository to break.
name: "dbt-postgres Tests"
run-name: >-
${{ (github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call')
&& format('dbt-postgres@{0} with dbt-core@{1}', inputs.dbt-postgres-ref, inputs.dbt-core-ref)
|| 'dbt-postgres@main with dbt-core branch' }}
on:
push:
branches:
- "main"
- "*.latest"
- "releases/*"
pull_request:
merge_group:
types: [checks_requested]
workflow_dispatch:
inputs:
dbt-postgres-ref:
description: "The branch of dbt-postgres to test against"
default: "main"
dbt-core-ref:
description: "The branch of dbt-core to test against"
default: "main"
workflow_call:
inputs:
dbt-postgres-ref:
description: "The branch of dbt-postgres to test against"
type: string
required: true
default: "main"
dbt-core-ref:
description: "The branch of dbt-core to test against"
type: string
required: true
default: "main"
permissions: read-all
# will cancel previous workflows triggered by the same event
# and for the same ref for PRs/merges or same SHA otherwise
# and for the same inputs on workflow_dispatch or workflow_call
concurrency:
group: ${{ github.workflow }}-${{ github.event_name }}-${{ contains(fromJson('["pull_request", "merge_group"]'), github.event_name) && github.event.pull_request.head.ref || github.sha }}-${{ contains(fromJson('["workflow_call", "workflow_dispatch"]'), github.event_name) && github.event.inputs.dbt-postgres-ref && github.event.inputs.dbt-core-ref || github.sha }}
cancel-in-progress: true
defaults:
run:
shell: bash
jobs:
job-prep:
# This allow us to run the workflow on pull_requests as well so we can always run unit tests
# and only run integration tests on merge for time purposes
name: Setup Repo Refs
runs-on: ubuntu-latest
outputs:
dbt-postgres-ref: ${{ steps.core-ref.outputs.ref }}
dbt-core-ref: ${{ steps.common-ref.outputs.ref }}
steps:
- name: "Input Refs"
id: job-inputs
run: |
echo "inputs.dbt-postgres-ref=${{ inputs.dbt-postgres-ref }}"
echo "inputs.dbt-core-ref=${{ inputs.dbt-core-ref }}"
- name: "Determine dbt-postgres ref"
id: core-ref
run: |
if [[ -z "${{ inputs.dbt-postgres-ref }}" ]]; then
REF="main"
else
REF=${{ inputs.dbt-postgres-ref }}
fi
echo "ref=$REF" >> $GITHUB_OUTPUT
- name: "Determine dbt-core ref"
id: common-ref
run: |
if [[ -z "${{ inputs.dbt-core-ref }}" ]]; then
# these will be commits instead of branches
if [[ "${{ github.event_name }}" == "merge_group" ]]; then
REF=${{ github.event.merge_group.head_sha }}
else
REF=${{ github.event.pull_request.base.sha }}
fi
else
REF=${{ inputs.dbt-core-ref }}
fi
echo "ref=$REF" >> $GITHUB_OUTPUT
- name: "Final Refs"
run: |
echo "dbt-postgres-ref=${{ steps.core-ref.outputs.ref }}"
echo "dbt-core-ref=${{ steps.common-ref.outputs.ref }}"
integration-tests-postgres:
name: "dbt-postgres integration tests"
needs: [job-prep]
runs-on: ubuntu-latest
defaults:
run:
working-directory: "./dbt-postgres"
environment:
name: "dbt-postgres"
env:
POSTGRES_TEST_HOST: ${{ vars.POSTGRES_TEST_HOST }}
POSTGRES_TEST_PORT: ${{ vars.POSTGRES_TEST_PORT }}
POSTGRES_TEST_USER: ${{ vars.POSTGRES_TEST_USER }}
POSTGRES_TEST_PASS: ${{ secrets.POSTGRES_TEST_PASS }}
POSTGRES_TEST_DATABASE: ${{ vars.POSTGRES_TEST_DATABASE }}
POSTGRES_TEST_THREADS: ${{ vars.POSTGRES_TEST_THREADS }}
services:
postgres:
image: postgres
env:
POSTGRES_PASSWORD: postgres
options: >-
--health-cmd pg_isready
--health-interval 10s
--health-timeout 5s
--health-retries 5
ports:
- ${{ vars.POSTGRES_TEST_PORT }}:5432
steps:
- name: "Check out dbt-adapters@${{ needs.job-prep.outputs.dbt-postgres-ref }}"
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # actions/checkout@v4
with:
repository: dbt-labs/dbt-adapters
ref: ${{ needs.job-prep.outputs.dbt-postgres-ref }}
- name: "Set up Python"
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # actions/setup-python@v5
with:
python-version: ${{ inputs.python-version }}
- name: "Set environment variables"
run: |
echo "HATCH_PYTHON=${{ inputs.python-version }}" >> $GITHUB_ENV
echo "PIP_ONLY_BINARY=psycopg2-binary" >> $GITHUB_ENV
- name: "Setup test database"
run: psql -f ./scripts/setup_test_database.sql
env:
PGHOST: ${{ vars.POSTGRES_TEST_HOST }}
PGPORT: ${{ vars.POSTGRES_TEST_PORT }}
PGUSER: postgres
PGPASSWORD: postgres
PGDATABASE: postgres
- name: "Install hatch"
uses: pypa/hatch@257e27e51a6a5616ed08a39a408a21c35c9931bc # pypa/hatch@install
- name: "Run integration tests"
run: hatch run ${{ inputs.hatch-env }}:integration-tests

View File

@@ -1,7 +1,7 @@
Resolves #
resolves #
<!---
Include the number of the issue addressed by this PR above, if applicable.
Include the number of the issue addressed by this PR above if applicable.
PRs for code changes without an associated issue *will not be merged*.
See CONTRIBUTING.md for more information.
@@ -26,8 +26,8 @@ Resolves #
### Checklist
- [ ] I have read [the contributing guide](https://github.com/dbt-labs/dbt-core/blob/main/CONTRIBUTING.md) and understand what's expected of me.
- [ ] I have run this code in development, and it appears to resolve the stated issue.
- [ ] This PR includes tests, or tests are not required or relevant for this PR.
- [ ] This PR has no interface changes (e.g., macros, CLI, logs, JSON artifacts, config files, adapter interface, etc.) or this PR has already received feedback and approval from Product or DX.
- [ ] This PR includes [type annotations](https://docs.python.org/3/library/typing.html) for new and modified functions.
- [ ] I have read [the contributing guide](https://github.com/dbt-labs/dbt-core/blob/main/CONTRIBUTING.md) and understand what's expected of me
- [ ] I have run this code in development and it appears to resolve the stated issue
- [ ] This PR includes tests, or tests are not required/relevant for this PR
- [ ] This PR has no interface changes (e.g. macros, cli, logs, json artifacts, config files, adapter interface, etc) or this PR has already received feedback and approval from Product or DX
- [ ] This PR includes [type annotations](https://docs.python.org/3/library/typing.html) for new and modified functions

View File

@@ -1,186 +0,0 @@
# **what?**
# Enforces 2 reviews when artifact or validation files are modified.
# **why?**
# Ensure artifact changes receive proper review from designated team members. GitHub doesn't support
# multiple reviews on a single PR based on files changed, so we need to enforce this manually.
# **when?**
# This will run when reviews are submitted and dismissed.
name: "Enforce Additional Reviews on Artifact and Validations Changes"
permissions:
checks: write
pull-requests: write
contents: read
on:
# trigger check on review events. use pull_request_target for forks.
pull_request_target:
types: [opened, reopened, ready_for_review, synchronize, review_requested]
pull_request_review:
types: [submitted, edited, dismissed]
# only run this once per PR at a time
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
cancel-in-progress: true
env:
required_approvals: 2
team: "core-group"
jobs:
check-reviews:
name: "Validate Additional Reviews"
runs-on: ubuntu-latest
steps:
- name: "Get list of changed files"
id: changed_files
run: |
# Fetch files as JSON and process with jq to sanitize output
gh api repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}/files \
| jq -r '.[].filename' \
| while IFS= read -r file; do
# Sanitize the filename by removing any special characters and command injection attempts
clean_file=$(echo "$file" | sed 's/[^a-zA-Z0-9\.\/\-_]//g')
echo "$clean_file"
done > changed_files.txt
echo "CHANGED_FILES<<EOF" >> $GITHUB_OUTPUT
cat changed_files.txt >> $GITHUB_OUTPUT
echo "EOF" >> $GITHUB_OUTPUT
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: "Check if any artifact files changed"
id: artifact_files_changed
run: |
artifact_changes=false
while IFS= read -r file; do
# Only process if file path looks legitimate
if [[ "$file" =~ ^[a-zA-Z0-9\.\/\-_]+$ ]]; then
if [[ "$file" == "core/dbt/artifacts/"* ]] ; then
artifact_changes=true
break
fi
fi
done < changed_files.txt
echo "artifact_changes=$artifact_changes" >> $GITHUB_OUTPUT
- name: "Get Core Team Members"
if: steps.artifact_files_changed.outputs.artifact_changes == 'true'
id: core_members
run: |
gh api -H "Accept: application/vnd.github+json" \
/orgs/dbt-labs/teams/${{ env.team }}/members > core_members.json
# Extract usernames and set as multiline output
echo "membership<<EOF" >> $GITHUB_OUTPUT
jq -r '.[].login' core_members.json >> $GITHUB_OUTPUT
echo "EOF" >> $GITHUB_OUTPUT
env:
GH_TOKEN: ${{ secrets.IT_TEAM_MEMBERSHIP }}
- name: "Verify ${{ env.required_approvals }} core team approvals"
if: steps.artifact_files_changed.outputs.artifact_changes == 'true'
id: check_approvals
run: |
# Get all reviews
REVIEWS=$(gh api repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}/reviews)
echo "All reviews:"
echo "$REVIEWS"
# Count approved reviews from core team members (only most recent review per user)
CORE_APPROVALS=0
while IFS= read -r member; do
echo "Checking member: $member"
APPROVED=$(echo "$REVIEWS" | jq --arg user "$member" '
group_by(.user.login) |
map(select(.[0].user.login == $user) |
sort_by(.submitted_at) |
last) |
map(select(.state == "APPROVED" and (.state != "DISMISSED"))) |
length')
echo "Latest review state for $member: $APPROVED"
CORE_APPROVALS=$((CORE_APPROVALS + APPROVED))
echo "Running total: $CORE_APPROVALS"
done <<< "${{ steps.core_members.outputs.membership }}"
echo "CORE_APPROVALS=$CORE_APPROVALS" >> $GITHUB_OUTPUT
echo "CORE_APPROVALS=$CORE_APPROVALS"
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: "Find Comment"
if: steps.artifact_files_changed.outputs.artifact_changes == 'true' && steps.check_approvals.outputs.CORE_APPROVALS < env.required_approvals
uses: peter-evans/find-comment@a54c31d7fa095754bfef525c0c8e5e5674c4b4b1 # peter-evans/find-comment@v2
id: find-comment
with:
issue-number: ${{ github.event.pull_request.number }}
comment-author: 'github-actions[bot]'
body-includes: "### Additional Artifact Review Required"
- name: "Create Comment"
if: steps.artifact_files_changed.outputs.artifact_changes == 'true' && steps.find-comment.outputs.comment-id == '' && steps.check_approvals.outputs.CORE_APPROVALS < env.required_approvals
uses: peter-evans/create-or-update-comment@23ff15729ef2fc348714a3bb66d2f655ca9066f2 # peter-evans/create-or-update-comment@v3
with:
issue-number: ${{ github.event.pull_request.number }}
body: |
### Additional Artifact Review Required
Changes to artifact directory files requires at least ${{ env.required_approvals }} approvals from core team members.
- name: "Notify if not enough approvals"
if: steps.artifact_files_changed.outputs.artifact_changes == 'true'
run: |
if [[ "${{ steps.check_approvals.outputs.CORE_APPROVALS }}" -ge "${{ env.required_approvals }}" ]]; then
title="Extra requirements met"
message="Changes to artifact directory files requires at least ${{ env.required_approvals }} approvals from core team members. Current number of core team approvals: ${{ steps.check_approvals.outputs.CORE_APPROVALS }} "
echo "::notice title=$title::$message"
echo "REVIEW_STATUS=success" >> $GITHUB_OUTPUT
else
title="PR Approval Requirements Not Met"
message="Changes to artifact directory files requires at least ${{ env.required_approvals }} approvals from core team members. Current number of core team approvals: ${{ steps.check_approvals.outputs.CORE_APPROVALS }} "
echo "::notice title=$title::$message"
echo "REVIEW_STATUS=neutral" >> $GITHUB_OUTPUT
fi
id: review_check
- name: "Set check status"
id: status_check
run: |
if [[ "${{ steps.artifact_files_changed.outputs.artifact_changes }}" == 'false' ]]; then
# no extra review required
echo "current_status=success" >> $GITHUB_OUTPUT
elif [[ "${{ steps.review_check.outputs.REVIEW_STATUS }}" == "success" ]]; then
# we have all the required reviews
echo "current_status=success" >> $GITHUB_OUTPUT
else
# neutral exit - neither success nor failure
# we can't fail here because we use multiple triggers for this workflow and they won't reset the check
# workaround is to use a neutral exit to skip the check run until it's actually successful
echo "current_status=neutral" >> $GITHUB_OUTPUT
fi
- name: "Post Event"
# This step posts the status of the check because the workflow is triggered by multiple events
# and we need to ensure the check is always updated. Otherwise we would end up with duplicate
# checks in the GitHub UI.
run: |
if [[ "${{ steps.status_check.outputs.current_status }}" == "success" ]]; then
state="success"
else
state="failure"
fi
gh api \
--method POST \
-H "Accept: application/vnd.github+json" \
/repos/${{ github.repository }}/statuses/${{ github.event.pull_request.base.sha }} \
-f state="$state" \
-f description="Artifact Review Check" \
-f context="Artifact Review Check" \
-f target_url="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
env:
GH_TOKEN: ${{ secrets.FISHTOWN_BOT_PAT }}

View File

@@ -1,50 +0,0 @@
# **what?**
# Check if the an issue is opened near or during an extended holiday period.
# If so, post an automatically-generated comment about the holiday for bug reports.
# Also provide specific information to customers of dbt Cloud.
# **why?**
# Explain why responses will be delayed during our holiday period.
# **when?**
# This will run when new issues are opened.
name: Auto-Respond to Bug Reports During Holiday Period
on:
issues:
types:
- opened
permissions:
contents: read
issues: write
jobs:
auto-response:
runs-on: ${{ vars.UBUNTU_LATEST }}
steps:
- name: Check if current date is within holiday period
id: date-check
run: |
current_date=$(date -u +"%Y-%m-%d")
start_date="2024-12-23"
end_date="2025-01-05"
if [[ "$current_date" < "$start_date" || "$current_date" > "$end_date" ]]; then
echo "outside_holiday=true" >> $GITHUB_ENV
else
echo "outside_holiday=false" >> $GITHUB_ENV
fi
- name: Post comment
if: ${{ env.outside_holiday == 'false' && contains(github.event.issue.labels.*.name, 'bug') }}
run: |
gh issue comment ${{ github.event.issue.number }} --repo ${{ github.repository }} --body "Thank you for your bug report! Our team is will be out of the office for [Christmas and our Global Week of Rest](https://handbook.getdbt.com/docs/time_off#2024-us-holidays), from December 25, 2024, through January 3, 2025.
We will review your issue as soon as possible after returning.
Thank you for your understanding, and happy holidays! 🎄🎉
If you are a customer of dbt Cloud, please contact our Customer Support team via the dbt Cloud web interface or email **support@dbtlabs.com**."
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

View File

@@ -28,13 +28,13 @@ permissions:
jobs:
backport:
name: Backport
runs-on: ${{ vars.UBUNTU_LATEST }}
runs-on: ubuntu-latest
# Only react to merged PRs for security reasons.
# See https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request_target.
if: >
github.event.pull_request.merged
&& contains(github.event.label.name, 'backport')
steps:
- uses: tibdex/backport@9565281eda0731b1d20c4025c43339fb0a23812e # tibdex/backport@v2.0.4
- uses: tibdex/backport@v2.0.4
with:
github_token: ${{ secrets.GITHUB_TOKEN }}

View File

@@ -41,14 +41,14 @@ jobs:
include:
- label: "dependencies"
changie_kind: "Dependencies"
runs-on: ${{ vars.UBUNTU_LATEST }}
runs-on: ubuntu-latest
steps:
- name: Create and commit changelog on bot PR
if: ${{ contains(github.event.pull_request.labels.*.name, matrix.label) }}
id: bot_changelog
uses: emmyoop/changie_bot@22b70618b13d0d1c64ea95212bafca2d2bf6b764 # emmyoop/changie_bot@v1.1.0
uses: emmyoop/changie_bot@v1.1.0
with:
GITHUB_TOKEN: ${{ secrets.FISHTOWN_BOT_PAT }}
commit_author_name: "Github Build Bot"

View File

@@ -4,26 +4,22 @@ on:
pull_request:
types: [ opened, reopened, labeled, unlabeled, synchronize ]
paths-ignore: [ '.changes/**', '.github/**', 'tests/**', '**.md', '**.yml' ]
merge_group:
types: [checks_requested]
workflow_dispatch:
permissions:
contents: read
workflow_dispatch:
jobs:
check-artifact-changes:
runs-on: ${{ vars.UBUNTU_LATEST }}
runs-on: ubuntu-latest
if: ${{ !contains(github.event.pull_request.labels.*.name, 'artifact_minor_upgrade') }}
steps:
- name: Checkout code
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # actions/checkout@v4
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Check for changes in core/dbt/artifacts
# https://github.com/marketplace/actions/paths-changes-filter
uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # dorny/paths-filter@v3
uses: dorny/paths-filter@v3
id: check_artifact_changes
with:
filters: |

View File

@@ -7,6 +7,7 @@
# **when?**
# When a PR is opened, not in draft or moved from draft to ready for review
name: Label community PRs
on:
@@ -28,15 +29,9 @@ jobs:
# If this PR is opened and not draft, determine if it needs to be labeled
# if the PR is converted out of draft, determine if it needs to be labeled
if: |
(
!contains(github.event.pull_request.labels.*.name, 'community')
&& (
(github.event.action == 'opened' && github.event.pull_request.draft == false)
|| github.event.action == 'ready_for_review'
)
&& github.event.pull_request.user.type != 'Bot'
&& github.event.pull_request.user.login != 'dependabot[bot]'
)
(!contains(github.event.pull_request.labels.*.name, 'community') &&
(github.event.action == 'opened' && github.event.pull_request.draft == false ) ||
github.event.action == 'ready_for_review' )
uses: dbt-labs/actions/.github/workflows/label-community.yml@main
with:
github_team: 'core-group'

View File

@@ -1,44 +1,25 @@
# **what?**
# Cuts the `*.latest` branch, bumps dependencies on it, cleans up all files in `.changes/unreleased`
# and `.changes/previous verion on main and bumps main to the input version.
# Cuts a new `*.latest` branch
# Also cleans up all files in `.changes/unreleased` and `.changes/previous verion on
# `main` and bumps `main` to the input version.
# **why?**
# Clean up the main branch after a release branch is cut and automate cutting the release branch.
# Generally reduces the workload of engineers and reducing error.
# Generally reduces the workload of engineers and reduces error. Allow automation.
# **when?**
# This will run when called manually or when triggered in another workflow.
# Example Usage including required permissions: TODO: update once finalized
# permissions:
# contents: read
# pull-requests: write
#
# name: Cut Release Branch
# jobs:
# changelog:
# uses: dbt-labs/actions/.github/workflows/cut-release-branch.yml@main
# with:
# new_branch_name: 1.7.latest
# PR_title: "Cleanup main after cutting new 1.7.latest branch"
# PR_body: "All adapter PRs will fail CI until the dbt-core PR has been merged due to release version conflicts."
# secrets:
# FISHTOWN_BOT_PAT: ${{ secrets.FISHTOWN_BOT_PAT }}
# TODOs
# add note to eventually commit changes directly and bypass checks - same as release - when we move to this model run test action after merge
# This will run when called manually.
name: Cut new release branch
run-name: "Cutting New Branch: ${{ inputs.new_branch_name }}"
on:
workflow_dispatch:
inputs:
new_branch_name:
description: "The full name of the new branch (ex. 1.5.latest)"
version_to_bump_main:
description: 'The alpha version main should bump to (ex. 1.6.0a1)'
required: true
new_branch_name:
description: 'The full name of the new branch (ex. 1.5.latest)'
required: true
type: string
defaults:
run:
@@ -46,346 +27,15 @@ defaults:
permissions:
contents: write
pull-requests: write
env:
PYTHON_TARGET_VERSION: "3.10"
PR_TITLE: "Cleanup main after cutting new ${{ inputs.new_branch_name }} branch"
PR_BODY: "All adapter PRs will fail CI until the dbt-core PR has been merged due to release version conflicts."
jobs:
prep_work:
name: "Prep Work"
runs-on: ubuntu-latest
steps:
- name: "[DEBUG] Print Inputs"
run: |
echo "new_branch_name: ${{ inputs.new_branch_name }}"
echo "PR_title: ${{ env.PR_TITLE }}"
echo "PR_body: ${{ env.PR_BODY }}"
create_temp_branch:
name: "Create Temp branch off main"
runs-on: ubuntu-latest
outputs:
temp_branch_name: ${{ steps.variables.outputs.BRANCH_NAME }}
steps:
- name: "Set Branch Value"
id: variables
run: |
echo "BRANCH_NAME=cutting_release_branch/main_cleanup_$GITHUB_RUN_ID" >> $GITHUB_OUTPUT
- name: "Checkout ${{ github.repository }}"
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # actions/checkout@v4
with:
ref: "main"
token: ${{ secrets.FISHTOWN_BOT_PAT }}
- name: "Create PR Branch"
run: |
user="Github Build Bot"
email="buildbot@fishtownanalytics.com"
git config user.name "$user"
git config user.email "$email"
git checkout -b ${{ steps.variables.outputs.BRANCH_NAME }}
git push --set-upstream origin ${{ steps.variables.outputs.BRANCH_NAME }}
- name: "[Notification] Temp branch created"
run: |
message="Temp branch ${{ steps.variables.outputs.BRANCH_NAME }} created"
echo "::notice title="Temporary branch created": $title::$message"
cleanup_changelog:
name: "Clean Up Changelog"
needs: ["create_temp_branch"]
runs-on: ubuntu-latest
outputs:
next-version: ${{ steps.semver-current.outputs.next-minor-alpha-version }}
steps:
- name: "Checkout ${{ github.repository }}"
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # actions/checkout@v4
with:
ref: ${{ needs.create_temp_branch.outputs.temp_branch_name }}
token: ${{ secrets.FISHTOWN_BOT_PAT }}
- name: "Add Homebrew To PATH"
run: |
echo "/home/linuxbrew/.linuxbrew/bin:/home/linuxbrew/.linuxbrew/sbin" >> $GITHUB_PATH
- name: "Install Homebrew Packages"
run: |
brew install pre-commit
brew tap miniscruff/changie https://github.com/miniscruff/changie
brew install changie
- name: "Check Current Version In Code"
id: determine_version
run: |
current_version=$(grep '^version = ' core/pyproject.toml | sed 's/version = "\(.*\)"/\1/')
echo "current_version=$current_version" >> $GITHUB_OUTPUT
- name: "[Notification] Check Current Version In Code"
run: |
message="The current version is ${{ steps.determine_version.outputs.current_version }}"
echo "::notice title="Version Bump Check": $title::$message"
- name: "Parse Current Version Into Parts for Changelog Directories"
id: semver-current
uses: dbt-labs/actions/parse-semver@main
with:
version: ${{ steps.determine_version.outputs.current_version }}
- name: "[Notification] Next Alpha Version"
run: |
message="The next alpha version is ${{ steps.semver-current.outputs.next-minor-alpha-version }}"
echo "::notice title="Version Bump Check": $title::$message"
- name: "Delete Unreleased Changelog YAMLs"
# removal fails if no files exist. OK to continue since we're just cleaning up the files.
continue-on-error: true
run: |
rm .changes/unreleased/*.yaml || true
- name: "Delete Pre Release Changelogs and YAMLs"
# removal fails if no files exist. OK to continue since we're just cleaning up the files.
continue-on-error: true
run: |
rm .changes/${{ steps.semver-current.outputs.base-version }}/*.yaml || true
rm .changes/${{ steps.semver-current.outputs.major }}.${{ steps.semver-current.outputs.minor }}.*.md || true
- name: "Cleanup CHANGELOG.md"
run: |
changie merge
- name: "Commit Changelog Cleanup to Branch"
run: |
user="Github Build Bot"
email="buildbot@fishtownanalytics.com"
git config user.name "$user"
git config user.email "$email"
git status
git add .
git commit -m "Clean up changelog on main"
git push
- name: "[Notification] Changelog cleaned up"
run: |
message="Changelog on ${{ needs.create_temp_branch.outputs.temp_branch_name }} cleaned up"
echo "::notice title="Changelog cleaned up": $title::$message"
bump_version:
name: "Bump to next minor version"
needs: ["cleanup_changelog", "create_temp_branch"]
runs-on: ubuntu-latest
steps:
- name: "Checkout ${{ github.repository }}"
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # actions/checkout@v4
with:
ref: ${{ needs.create_temp_branch.outputs.temp_branch_name }}
token: ${{ secrets.FISHTOWN_BOT_PAT }}
- name: "Set up Python - ${{ env.PYTHON_TARGET_VERSION }}"
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # actions/setup-python@v5
with:
python-version: "${{ env.PYTHON_TARGET_VERSION }}"
- name: "Install Spark Dependencies"
if: ${{ contains(github.repository, 'dbt-labs/dbt-spark') }}
run: |
sudo apt-get update
sudo apt-get install libsasl2-dev
- name: "Install Python Dependencies"
run: |
python -m pip install --upgrade pip
python -m pip install hatch
- name: "Bump Version To ${{ needs.cleanup_changelog.outputs.next-version }}"
run: |
cd core
hatch version ${{ needs.cleanup_changelog.outputs.next-version }}
hatch run dev-req
dbt --version
- name: "Commit Version Bump to Branch"
run: |
user="Github Build Bot"
email="buildbot@fishtownanalytics.com"
git config user.name "$user"
git config user.email "$email"
git status
git add .
git commit -m "Bumping version to ${{ needs.cleanup_changelog.outputs.next-version }}"
git push
- name: "[Notification] Version Bump completed"
run: |
message="Version on ${{ needs.create_temp_branch.outputs.temp_branch_name }} bumped to ${{ needs.cleanup_changelog.outputs.next-version }}"
echo "::notice title="Version Bump Completed": $title::$message"
cleanup:
name: "Cleanup Code Quality"
needs: ["create_temp_branch", "bump_version"]
runs-on: ubuntu-latest
steps:
- name: "Checkout ${{ github.repository }}"
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # actions/checkout@v4
with:
ref: ${{ needs.create_temp_branch.outputs.temp_branch_name }}
token: ${{ secrets.FISHTOWN_BOT_PAT }}
- name: "Add Homebrew To PATH"
run: |
echo "/home/linuxbrew/.linuxbrew/bin:/home/linuxbrew/.linuxbrew/sbin" >> $GITHUB_PATH
- name: "brew install pre-commit"
run: |
brew install pre-commit
# this step will fail on whitespace errors but also correct them
- name: "Cleanup - Remove Trailing Whitespace Via Pre-commit"
continue-on-error: true
run: |
pre-commit run trailing-whitespace --files CHANGELOG.md .changes/* || true
# this step will fail on newline errors but also correct them
- name: "Cleanup - Remove Extra Newlines Via Pre-commit"
continue-on-error: true
run: |
pre-commit run end-of-file-fixer --files CHANGELOG.md .changes/* || true
- name: "Commit Version Bump to Branch"
run: |
user="Github Build Bot"
email="buildbot@fishtownanalytics.com"
git config user.name "$user"
git config user.email "$email"
git status
git add .
git commit -m "Code quality cleanup"
git push
open_pr:
name: "Open PR Against main"
needs: ["cleanup_changelog", "create_temp_branch", "cleanup"]
runs-on: ubuntu-latest
outputs:
pr_number: ${{ steps.create_pr.outputs.pull-request-number }}
steps:
- name: "Checkout ${{ github.repository }}"
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # actions/checkout@v4
with:
ref: ${{ needs.create_temp_branch.outputs.temp_branch_name }}
token: ${{ secrets.FISHTOWN_BOT_PAT }}
- name: "Determine PR Title"
id: pr_title
run: |
echo "pr_title=${{ env.PR_TITLE }}" >> $GITHUB_OUTPUT
if [${{ env.PR_TITLE }} == ""]; then
echo "pr_title='Clean up changelogs and bump to version ${{ needs.cleanup_changelog.outputs.next-version }}'" >> $GITHUB_OUTPUT
fi
- name: "Determine PR Body"
id: pr_body
run: |
echo "pr_body=${{ env.PR_BODY }}" >> $GITHUB_OUTPUT
if [${{ env.PR_BODY }} == ""]; then
echo "pr_body='Clean up changelogs and bump to version ${{ needs.cleanup_changelog.outputs.next-version }}'" >> $GITHUB_OUTPUT
fi
- name: "Add Branch Details"
id: pr_body_branch
run: |
branch_details="The workflow that generated this PR also created a new branch: ${{ inputs.new_branch_name }}"
full_body="${{ steps.pr_body.outputs.pr_body }} $branch_details"
echo "pr_full_body=$full_body" >> $GITHUB_OUTPUT
- name: "Open Pull Request"
id: create_pr
run: |
pr_url=$(gh pr create -B main -H ${{ needs.create_temp_branch.outputs.temp_branch_name }} -l "Skip Changelog" -t "${{ steps.pr_title.outputs.pr_title }}" -b "${{ steps.pr_body_branch.outputs.pr_full_body }}")
echo "pr_url=$pr_url" >> $GITHUB_OUTPUT
env:
GH_TOKEN: ${{ secrets.FISHTOWN_BOT_PAT }}
- name: "[Notification] Pull Request Opened"
run: |
message="PR opened at ${{ steps.create_pr.outputs.pr_url }}"
echo "::notice title="Pull Request Opened": $title::$message"
cut_new_branch:
# don't cut the new branch until we're done opening the PR against main
name: "Cut New Branch ${{ inputs.new_branch_name }}"
needs: [open_pr]
runs-on: ubuntu-latest
steps:
- name: "Checkout ${{ github.repository }}"
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # actions/checkout@v4
with:
token: ${{ secrets.FISHTOWN_BOT_PAT }}
fetch-depth: 0
- name: "Ensure New Branch Does Not Exist"
id: check_new_branch
run: |
title="Check New Branch Existence"
if git show-ref --quiet ${{ inputs.new_branch_name }}; then
message="Branch ${{ inputs.new_branch_name }} already exists. Exiting."
echo "::error $title::$message"
exit 1
fi
- name: "Create New Release Branch"
run: |
git checkout -b ${{ inputs.new_branch_name }}
- name: "Push up New Branch"
run: |
#Data for commit
user="Github Build Bot"
email="buildbot@fishtownanalytics.com"
git config user.name "$user"
git config user.email "$email"
git push --set-upstream origin ${{ inputs.new_branch_name }}
- name: "[Notification] New branch created"
run: |
message="New branch ${{ inputs.new_branch_name }} created"
echo "::notice title="New branch created": $title::$message"
- name: "Bump dependencies via script"
# This bumps the dependency on dbt-core in the adapters
if: ${{ !contains(github.repository, 'dbt-core') }}
run: |
echo ${{ github.repository }}
echo "running update_dependencies script"
bash ${GITHUB_WORKSPACE}/.github/scripts/update_dependencies.sh ${{ inputs.new_branch_name }}
commit_message="bumping .latest branch variable in update_dependencies.sh to ${{ inputs.new_branch_name }}"
git status
git add .
git commit -m "$commit_message"
git push
- name: "Bump env variable via script"
# bumps the RELEASE_BRANCH variable in nightly-release.yml in adapters
if: ${{ !contains(github.repository, 'dbt-core') }}
run: |
file="./.github/scripts/update_release_branch.sh"
if test -f "$file"; then
echo ${{ github.repository }}
echo "running some script yet to be written now"
bash $file ${{ inputs.new_branch_name }}
commit_message="updating env variable to ${{ inputs.new_branch_name }} in nightly-release.yml"
git status
git add .
git commit -m "$commit_message"
git push
else
echo "no $file seen skipping step"
fi
cut_branch:
name: "Cut branch and clean up main for dbt-core"
uses: dbt-labs/actions/.github/workflows/cut-release-branch.yml@main
with:
version_to_bump_main: ${{ inputs.version_to_bump_main }}
new_branch_name: ${{ inputs.new_branch_name }}
PR_title: "Cleanup main after cutting new ${{ inputs.new_branch_name }} branch"
PR_body: "All adapter PRs will fail CI until the dbt-core PR has been merged due to release version conflicts."
secrets:
FISHTOWN_BOT_PAT: ${{ secrets.FISHTOWN_BOT_PAT }}

View File

@@ -36,6 +36,6 @@ jobs:
uses: dbt-labs/actions/.github/workflows/open-issue-in-repo.yml@main
with:
issue_repository: "dbt-labs/docs.getdbt.com"
issue_title: "[Core] Docs Changes Needed from ${{ github.event.repository.name }} Issue #${{ github.event.issue.number }}"
issue_body: "At a minimum, update body to include a link to the page on docs.getdbt.com requiring updates and what part(s) of the page you would like to see updated.\n Originating from this issue: https://github.com/dbt-labs/dbt-core/issues/${{ github.event.issue.number }}"
issue_title: "Docs Changes Needed from ${{ github.event.repository.name }} Issue #${{ github.event.issue.number }}"
issue_body: "At a minimum, update body to include a link to the page on docs.getdbt.com requiring updates and what part(s) of the page you would like to see updated."
secrets: inherit

View File

@@ -20,8 +20,6 @@ on:
- "*.latest"
- "releases/*"
pull_request:
merge_group:
types: [checks_requested]
workflow_dispatch:
permissions: read-all
@@ -49,33 +47,26 @@ jobs:
steps:
- name: Check out the repository
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # actions/checkout@v4
uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # actions/setup-python@v6
uses: actions/setup-python@v5
with:
python-version: "3.10"
python-version: '3.8'
- name: Install python dependencies
run: |
python -m pip install --user --upgrade pip
python -m pip --version
python -m pip install hatch
cd core
hatch run setup
- name: Verify dbt installation
run: |
cd core
hatch run dbt --version
make dev
mypy --version
dbt --version
- name: Run pre-commit hooks
run: |
cd core
hatch run code-quality
run: pre-commit run --all-files --show-diff-on-failure
unit:
name: "unit test / python ${{ matrix.python-version }}"
name: unit test / python ${{ matrix.python-version }}
runs-on: ubuntu-latest
timeout-minutes: 10
@@ -83,14 +74,17 @@ jobs:
strategy:
fail-fast: false
matrix:
python-version: ["3.10", "3.11", "3.12", "3.13"]
python-version: [ "3.8", "3.9", "3.10", "3.11", "3.12" ]
env:
TOXENV: "unit"
steps:
- name: Check out the repository
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # actions/checkout@v4
uses: actions/checkout@v4
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # actions/setup-python@v6
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
@@ -98,15 +92,15 @@ jobs:
run: |
python -m pip install --user --upgrade pip
python -m pip --version
python -m pip install hatch
hatch --version
python -m pip install tox
tox --version
- name: Run unit tests
uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # nick-fields/retry@v3
uses: nick-fields/retry@v3
with:
timeout_minutes: 10
max_attempts: 3
command: cd core && hatch run ci:unit-tests
command: tox -e unit
- name: Get current date
if: always()
@@ -117,11 +111,10 @@ jobs:
- name: Upload Unit Test Coverage to Codecov
if: ${{ matrix.python-version == '3.11' }}
uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # codecov/codecov-action@v5
uses: codecov/codecov-action@v4
with:
token: ${{ secrets.CODECOV_TOKEN }}
flags: unit
fail_ci_if_error: false
integration-metadata:
name: integration test metadata generation
@@ -146,7 +139,7 @@ jobs:
- name: generate include
id: generate-include
run: |
INCLUDE=('"python-version":"3.10","os":"windows-latest"' '"python-version":"3.10","os":"macos-14"' )
INCLUDE=('"python-version":"3.8","os":"windows-latest"' '"python-version":"3.8","os":"macos-12"' )
INCLUDE_GROUPS="["
for include in ${INCLUDE[@]}; do
for group in $(seq 1 ${{ env.PYTHON_INTEGRATION_TEST_WORKERS }}); do
@@ -158,102 +151,7 @@ jobs:
echo "include=${INCLUDE_GROUPS}"
echo "include=${INCLUDE_GROUPS}" >> $GITHUB_OUTPUT
integration-postgres:
name: "(${{ matrix.split-group }}) integration test / python ${{ matrix.python-version }} / ${{ matrix.os }}"
runs-on: ${{ matrix.os }}
timeout-minutes: 30
needs:
- integration-metadata
strategy:
fail-fast: false
matrix:
python-version: ["3.10", "3.11", "3.12", "3.13"]
os: ["ubuntu-latest"]
split-group: ${{ fromJson(needs.integration-metadata.outputs.split-groups) }}
env:
DBT_INVOCATION_ENV: github-actions
DBT_TEST_USER_1: dbt_test_user_1
DBT_TEST_USER_2: dbt_test_user_2
DBT_TEST_USER_3: dbt_test_user_3
DD_CIVISIBILITY_AGENTLESS_ENABLED: true
DD_API_KEY: ${{ secrets.DATADOG_API_KEY }}
DD_SITE: datadoghq.com
DD_ENV: ci
DD_SERVICE: ${{ github.event.repository.name }}
services:
# Label used to access the service container
postgres:
# Docker Hub image
image: postgres
# Provide the password for postgres
env:
POSTGRES_PASSWORD: password
POSTGRES_USER: postgres
# Set health checks to wait until postgres has started
options: >-
--health-cmd pg_isready
--health-interval 10s
--health-timeout 5s
--health-retries 5
ports:
- 5432:5432
steps:
- name: Check out the repository
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # actions/checkout@v4
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # actions/setup-python@v6
with:
python-version: ${{ matrix.python-version }}
- name: Run postgres setup script
run: |
./scripts/setup_db.sh
env:
PGHOST: localhost
PGPORT: 5432
PGPASSWORD: password
- name: Install python tools
run: |
python -m pip install --user --upgrade pip
python -m pip --version
python -m pip install hatch
hatch --version
- name: Run integration tests
uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # nick-fields/retry@v3
with:
timeout_minutes: 30
max_attempts: 3
shell: bash
command: cd core && hatch run ci:integration-tests -- --ddtrace --splits ${{ env.PYTHON_INTEGRATION_TEST_WORKERS }} --group ${{ matrix.split-group }}
- name: Get current date
if: always()
id: date
run: |
CURRENT_DATE=$(date +'%Y-%m-%dT%H_%M_%S') # no colons allowed for artifacts
echo "date=$CURRENT_DATE" >> $GITHUB_OUTPUT
- uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # actions/upload-artifact@v4
if: always()
with:
name: logs_${{ matrix.python-version }}_${{ matrix.os }}_${{ matrix.split-group }}_${{ steps.date.outputs.date }}
path: ./logs
- name: Upload Integration Test Coverage to Codecov
if: ${{ matrix.python-version == '3.11' }}
uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # codecov/codecov-action@v5
with:
token: ${{ secrets.CODECOV_TOKEN }}
flags: integration
fail_ci_if_error: false
integration-mac-windows:
integration:
name: (${{ matrix.split-group }}) integration test / python ${{ matrix.python-version }} / ${{ matrix.os }}
runs-on: ${{ matrix.os }}
@@ -263,9 +161,12 @@ jobs:
strategy:
fail-fast: false
matrix:
# already includes split group and runs mac + windows
python-version: [ "3.8", "3.9", "3.10", "3.11", "3.12" ]
os: [ubuntu-20.04]
split-group: ${{ fromJson(needs.integration-metadata.outputs.split-groups) }}
include: ${{ fromJson(needs.integration-metadata.outputs.include) }}
env:
TOXENV: integration
DBT_INVOCATION_ENV: github-actions
DBT_TEST_USER_1: dbt_test_user_1
DBT_TEST_USER_2: dbt_test_user_2
@@ -278,21 +179,20 @@ jobs:
steps:
- name: Check out the repository
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # actions/checkout@v4
uses: actions/checkout@v4
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # actions/setup-python@v6
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
- name: Set up postgres (linux)
if: runner.os == 'Linux'
uses: ./.github/actions/setup-postgres-linux
- name: Set up postgres (macos)
if: runner.os == 'macOS'
uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # nick-fields/retry@v3
with:
timeout_minutes: 10
max_attempts: 3
command: ./scripts/setup_db.sh
uses: ./.github/actions/setup-postgres-macos
- name: Set up postgres (windows)
if: runner.os == 'Windows'
@@ -302,16 +202,17 @@ jobs:
run: |
python -m pip install --user --upgrade pip
python -m pip --version
python -m pip install hatch
hatch --version
python -m pip install tox
tox --version
- name: Run integration tests
uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # nick-fields/retry@v3
uses: nick-fields/retry@v3
with:
timeout_minutes: 30
max_attempts: 3
shell: bash
command: cd core && hatch run ci:integration-tests -- --ddtrace --splits ${{ env.PYTHON_INTEGRATION_TEST_WORKERS }} --group ${{ matrix.split-group }}
command: tox -- --ddtrace
env:
PYTEST_ADDOPTS: ${{ format('--splits {0} --group {1}', env.PYTHON_INTEGRATION_TEST_WORKERS, matrix.split-group) }}
- name: Get current date
if: always()
@@ -320,7 +221,7 @@ jobs:
CURRENT_DATE=$(date +'%Y-%m-%dT%H_%M_%S') # no colons allowed for artifacts
echo "date=$CURRENT_DATE" >> $GITHUB_OUTPUT
- uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # actions/upload-artifact@v4
- uses: actions/upload-artifact@v4
if: always()
with:
name: logs_${{ matrix.python-version }}_${{ matrix.os }}_${{ matrix.split-group }}_${{ steps.date.outputs.date }}
@@ -328,20 +229,19 @@ jobs:
- name: Upload Integration Test Coverage to Codecov
if: ${{ matrix.python-version == '3.11' }}
uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # codecov/codecov-action@v5
uses: codecov/codecov-action@v4
with:
token: ${{ secrets.CODECOV_TOKEN }}
flags: integration
fail_ci_if_error: false
integration-report:
if: ${{ always() }}
name: Integration Test Suite
runs-on: ubuntu-latest
needs: [integration-mac-windows, integration-postgres]
needs: integration
steps:
- name: "Integration Tests Failed"
if: ${{ contains(needs.integration-mac-windows.result, 'failure') || contains(needs.integration-mac-windows.result, 'cancelled') || contains(needs.integration-postgres.result, 'failure') || contains(needs.integration-postgres.result, 'cancelled') }}
if: ${{ contains(needs.integration.result, 'failure') || contains(needs.integration.result, 'cancelled') }}
# when this is true the next step won't execute
run: |
echo "::notice title='Integration test suite failed'"
@@ -358,17 +258,17 @@ jobs:
steps:
- name: Check out the repository
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # actions/checkout@v4
uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # actions/setup-python@v6
uses: actions/setup-python@v5
with:
python-version: "3.10"
python-version: '3.8'
- name: Install python dependencies
run: |
python -m pip install --user --upgrade pip
python -m pip install --upgrade hatch twine check-wheel-contents
python -m pip install --upgrade setuptools wheel twine check-wheel-contents
python -m pip --version
- name: Build distributions
@@ -377,7 +277,27 @@ jobs:
- name: Show distributions
run: ls -lh dist/
- name: Check and verify distributions
- name: Check distribution descriptions
run: |
cd core
hatch run build:check-all
twine check dist/*
- name: Check wheel contents
run: |
check-wheel-contents dist/*.whl --ignore W007,W008
- name: Install wheel distributions
run: |
find ./dist/*.whl -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/
- name: Check wheel distributions
run: |
dbt --version
- name: Install source distributions
# ignore dbt-1.0.0, which intentionally raises an error when installed from source
run: |
find ./dist/*.gz -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/
- name: Check source distributions
run: |
dbt --version

265
.github/workflows/model_performance.yml vendored Normal file
View File

@@ -0,0 +1,265 @@
# **what?**
# This workflow models the performance characteristics of a point in time in dbt.
# It runs specific dbt commands on committed projects multiple times to create and
# commit information about the distribution to the current branch. For more information
# see the readme in the performance module at /performance/README.md.
#
# **why?**
# When developing new features, we can take quick performance samples and compare
# them against the commited baseline measurements produced by this workflow to detect
# some performance regressions at development time before they reach users.
#
# **when?**
# This is only run once directly after each release (for non-prereleases). If for some
# reason the results of a run are not satisfactory, it can also be triggered manually.
name: Model Performance Characteristics
on:
# runs after non-prereleases are published.
release:
types: [released]
# run manually from the actions tab
workflow_dispatch:
inputs:
release_id:
description: 'dbt version to model (must be non-prerelease in Pypi)'
type: string
required: true
env:
RUNNER_CACHE_PATH: performance/runner/target/release/runner
# both jobs need to write
permissions:
contents: write
pull-requests: write
jobs:
set-variables:
name: Setting Variables
runs-on: ubuntu-latest
outputs:
cache_key: ${{ steps.variables.outputs.cache_key }}
release_id: ${{ steps.semver.outputs.base-version }}
release_branch: ${{ steps.variables.outputs.release_branch }}
steps:
# explicitly checkout the performance runner from main regardless of which
# version we are modeling.
- name: Checkout
uses: actions/checkout@v4
with:
ref: main
- name: Parse version into parts
id: semver
uses: dbt-labs/actions/parse-semver@v1
with:
version: ${{ github.event.inputs.release_id || github.event.release.tag_name }}
# collect all the variables that need to be used in subsequent jobs
- name: Set variables
id: variables
run: |
# create a cache key that will be used in the next job. without this the
# next job would have to checkout from main and hash the files itself.
echo "cache_key=${{ runner.os }}-${{ hashFiles('performance/runner/Cargo.toml')}}-${{ hashFiles('performance/runner/src/*') }}" >> $GITHUB_OUTPUT
branch_name="${{steps.semver.outputs.major}}.${{steps.semver.outputs.minor}}.latest"
echo "release_branch=$branch_name" >> $GITHUB_OUTPUT
echo "release branch is inferred to be ${branch_name}"
latest-runner:
name: Build or Fetch Runner
runs-on: ubuntu-latest
needs: [set-variables]
env:
RUSTFLAGS: "-D warnings"
steps:
- name: '[DEBUG] print variables'
run: |
echo "all variables defined in set-variables"
echo "cache_key: ${{ needs.set-variables.outputs.cache_key }}"
echo "release_id: ${{ needs.set-variables.outputs.release_id }}"
echo "release_branch: ${{ needs.set-variables.outputs.release_branch }}"
# explicitly checkout the performance runner from main regardless of which
# version we are modeling.
- name: Checkout
uses: actions/checkout@v4
with:
ref: main
# attempts to access a previously cached runner
- uses: actions/cache@v4
id: cache
with:
path: ${{ env.RUNNER_CACHE_PATH }}
key: ${{ needs.set-variables.outputs.cache_key }}
- name: Fetch Rust Toolchain
if: steps.cache.outputs.cache-hit != 'true'
uses: actions-rs/toolchain@v1
with:
profile: minimal
toolchain: stable
override: true
- name: Add fmt
if: steps.cache.outputs.cache-hit != 'true'
run: rustup component add rustfmt
- name: Cargo fmt
if: steps.cache.outputs.cache-hit != 'true'
uses: actions-rs/cargo@v1
with:
command: fmt
args: --manifest-path performance/runner/Cargo.toml --all -- --check
- name: Test
if: steps.cache.outputs.cache-hit != 'true'
uses: actions-rs/cargo@v1
with:
command: test
args: --manifest-path performance/runner/Cargo.toml
- name: Build (optimized)
if: steps.cache.outputs.cache-hit != 'true'
uses: actions-rs/cargo@v1
with:
command: build
args: --release --manifest-path performance/runner/Cargo.toml
# the cache action automatically caches this binary at the end of the job
model:
# depends on `latest-runner` as a separate job so that failures in this job do not prevent
# a successfully tested and built binary from being cached.
needs: [set-variables, latest-runner]
name: Model a release
runs-on: ubuntu-latest
steps:
- name: '[DEBUG] print variables'
run: |
echo "all variables defined in set-variables"
echo "cache_key: ${{ needs.set-variables.outputs.cache_key }}"
echo "release_id: ${{ needs.set-variables.outputs.release_id }}"
echo "release_branch: ${{ needs.set-variables.outputs.release_branch }}"
- name: Setup Python
uses: actions/setup-python@v5
with:
python-version: "3.8"
- name: Install dbt
run: pip install dbt-postgres==${{ needs.set-variables.outputs.release_id }}
- name: Install Hyperfine
run: wget https://github.com/sharkdp/hyperfine/releases/download/v1.11.0/hyperfine_1.11.0_amd64.deb && sudo dpkg -i hyperfine_1.11.0_amd64.deb
# explicitly checkout main to get the latest project definitions
- name: Checkout
uses: actions/checkout@v4
with:
ref: main
# this was built in the previous job so it will be there.
- name: Fetch Runner
uses: actions/cache@v4
id: cache
with:
path: ${{ env.RUNNER_CACHE_PATH }}
key: ${{ needs.set-variables.outputs.cache_key }}
- name: Move Runner
run: mv performance/runner/target/release/runner performance/app
- name: Change Runner Permissions
run: chmod +x ./performance/app
- name: '[DEBUG] ls baseline directory before run'
run: ls -R performance/baselines/
# `${{ github.workspace }}` is used to pass the absolute path
- name: Create directories
run: |
mkdir ${{ github.workspace }}/performance/tmp/
mkdir -p performance/baselines/${{ needs.set-variables.outputs.release_id }}/
# Run modeling with taking 20 samples
- name: Run Measurement
run: |
performance/app model -v ${{ needs.set-variables.outputs.release_id }} -b ${{ github.workspace }}/performance/baselines/ -p ${{ github.workspace }}/performance/projects/ -t ${{ github.workspace }}/performance/tmp/ -n 20
- name: '[DEBUG] ls baseline directory after run'
run: ls -R performance/baselines/
- uses: actions/upload-artifact@v4
with:
name: baseline
path: performance/baselines/${{ needs.set-variables.outputs.release_id }}/
create-pr:
name: Open PR for ${{ matrix.base-branch }}
# depends on `model` as a separate job so that the baseline can be committed to more than one branch
# i.e. release branch and main
needs: [set-variables, latest-runner, model]
runs-on: ubuntu-latest
strategy:
matrix:
include:
- base-branch: refs/heads/main
target-branch: performance-bot/main_${{ needs.set-variables.outputs.release_id }}_${{GITHUB.RUN_ID}}
- base-branch: refs/heads/${{ needs.set-variables.outputs.release_branch }}
target-branch: performance-bot/release_${{ needs.set-variables.outputs.release_id }}_${{GITHUB.RUN_ID}}
steps:
- name: '[DEBUG] print variables'
run: |
echo "all variables defined in set-variables"
echo "cache_key: ${{ needs.set-variables.outputs.cache_key }}"
echo "release_id: ${{ needs.set-variables.outputs.release_id }}"
echo "release_branch: ${{ needs.set-variables.outputs.release_branch }}"
- name: Checkout
uses: actions/checkout@v4
with:
ref: ${{ matrix.base-branch }}
- name: Create PR branch
run: |
git checkout -b ${{ matrix.target-branch }}
git push origin ${{ matrix.target-branch }}
git branch --set-upstream-to=origin/${{ matrix.target-branch }} ${{ matrix.target-branch }}
- uses: actions/download-artifact@v4
with:
name: baseline
path: performance/baselines/${{ needs.set-variables.outputs.release_id }}
- name: '[DEBUG] ls baselines after artifact download'
run: ls -R performance/baselines/
- name: Commit baseline
uses: EndBug/add-and-commit@v9
with:
add: 'performance/baselines/*'
author_name: 'Github Build Bot'
author_email: 'buildbot@fishtownanalytics.com'
message: 'adding performance baseline for ${{ needs.set-variables.outputs.release_id }}'
push: 'origin origin/${{ matrix.target-branch }}'
- name: Create Pull Request
uses: peter-evans/create-pull-request@v6
with:
author: 'Github Build Bot <buildbot@fishtownanalytics.com>'
base: ${{ matrix.base-branch }}
branch: '${{ matrix.target-branch }}'
title: 'Adding performance modeling for ${{needs.set-variables.outputs.release_id}} to ${{ matrix.base-branch }}'
body: 'Committing perf results for tracking for the ${{needs.set-variables.outputs.release_id}}'
labels: |
Skip Changelog
Performance

View File

@@ -31,7 +31,7 @@ env:
jobs:
aggregate-release-data:
runs-on: ${{ vars.UBUNTU_LATEST }}
runs-on: ubuntu-latest
outputs:
version_number: ${{ steps.nightly-release-version.outputs.number }}
@@ -39,14 +39,14 @@ jobs:
steps:
- name: "Checkout ${{ github.repository }} Branch ${{ env.RELEASE_BRANCH }}"
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # actions/checkout@v4
uses: actions/checkout@v4
with:
ref: ${{ env.RELEASE_BRANCH }}
- name: "Get Current Version Number"
id: version-number-sources
run: |
current_version=$(grep '^version = ' core/dbt/__version__.py | sed 's/version = "\(.*\)"/\1/')
current_version=`awk -F"current_version = " '{print $2}' .bumpversion.cfg | tr '\n' ' '`
echo "current_version=$current_version" >> $GITHUB_OUTPUT
- name: "Audit Version And Parse Into Parts"
@@ -76,7 +76,7 @@ jobs:
echo "name=${{ env.RELEASE_BRANCH }}" >> $GITHUB_OUTPUT
log-outputs-aggregate-release-data:
runs-on: ${{ vars.UBUNTU_LATEST }}
runs-on: ubuntu-latest
needs: [aggregate-release-data]
steps:

View File

@@ -72,15 +72,12 @@ defaults:
run:
shell: bash
env:
MIN_HATCH_VERSION: "1.11.0"
jobs:
job-setup:
name: Log Inputs
runs-on: ${{ vars.UBUNTU_LATEST }}
runs-on: ubuntu-latest
outputs:
use_hatch: ${{ steps.use_hatch.outputs.use_hatch }}
starting_sha: ${{ steps.set_sha.outputs.starting_sha }}
steps:
- name: "[DEBUG] Print Variables"
run: |
@@ -91,29 +88,19 @@ jobs:
echo Nightly release: ${{ inputs.nightly_release }}
echo Only Docker: ${{ inputs.only_docker }}
# In version env.HATCH_VERSION we started to use hatch for build tooling. Before that we used setuptools.
# This needs to check if we're using hatch or setuptools based on the version being released. We should
# check if the version is greater than or equal to env.HATCH_VERSION. If it is, we use hatch, otherwise we use setuptools.
- name: "Check if using hatch"
id: use_hatch
run: |
# Extract major.minor from versions like 1.11.0a1 -> 1.11
INPUT_MAJ_MIN=$(echo "${{ inputs.version_number }}" | sed -E 's/^([0-9]+\.[0-9]+).*/\1/')
HATCH_MAJ_MIN=$(echo "${{ env.MIN_HATCH_VERSION }}" | sed -E 's/^([0-9]+\.[0-9]+).*/\1/')
- name: "Checkout target branch"
uses: actions/checkout@v4
with:
ref: ${{ inputs.target_branch }}
if [ $(echo "$INPUT_MAJ_MIN >= $HATCH_MAJ_MIN" | bc) -eq 1 ]; then
echo "use_hatch=true" >> $GITHUB_OUTPUT
else
echo "use_hatch=false" >> $GITHUB_OUTPUT
fi
- name: "Notify if using hatch"
# release-prep.yml really shouldn't take in the sha but since core + all adapters
# depend on it now this workaround lets us not input it manually with risk of error.
# The changes always get merged into the head so we can't use a specific commit for
# releases anyways.
- name: "Capture sha"
id: set_sha
run: |
if [ ${{ steps.use_hatch.outputs.use_hatch }} = "true" ]; then
echo "::notice title="Using Hatch": $title::Using Hatch for release"
else
echo "::notice title="Using Setuptools": $title::Using Setuptools for release"
fi
echo "starting_sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT
bump-version-generate-changelog:
name: Bump package version, Generate changelog
@@ -123,13 +110,12 @@ jobs:
uses: dbt-labs/dbt-release/.github/workflows/release-prep.yml@main
with:
sha: ${{ needs.job-setup.outputs.starting_sha }}
version_number: ${{ inputs.version_number }}
hatch_directory: "core"
target_branch: ${{ inputs.target_branch }}
env_setup_script_path: "scripts/env-setup.sh"
test_run: ${{ inputs.test_run }}
nightly_release: ${{ inputs.nightly_release }}
use_hatch: ${{ needs.job-setup.outputs.use_hatch == 'true' }} # workflow outputs are strings...
secrets: inherit
@@ -139,7 +125,7 @@ jobs:
needs: [bump-version-generate-changelog]
runs-on: ${{ vars.UBUNTU_LATEST }}
runs-on: ubuntu-latest
steps:
- name: Print variables
@@ -157,13 +143,16 @@ jobs:
with:
sha: ${{ needs.bump-version-generate-changelog.outputs.final_sha }}
version_number: ${{ inputs.version_number }}
hatch_directory: "core"
changelog_path: ${{ needs.bump-version-generate-changelog.outputs.changelog_path }}
build_script_path: "scripts/build-dist.sh"
s3_bucket_name: "core-team-artifacts"
package_test_command: "dbt --version"
test_run: ${{ inputs.test_run }}
nightly_release: ${{ inputs.nightly_release }}
use_hatch: ${{ needs.job-setup.outputs.use_hatch == 'true' }} # workflow outputs are strings...
secrets:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
github-release:
name: GitHub Release
@@ -199,7 +188,7 @@ jobs:
# determine if we need to release dbt-core or both dbt-core and dbt-postgres
name: Determine Docker Package
if: ${{ !failure() && !cancelled() }}
runs-on: ${{ vars.UBUNTU_LATEST }}
runs-on: ubuntu-latest
needs: [pypi-release]
outputs:
matrix: ${{ steps.determine-docker-package.outputs.matrix }}

Some files were not shown because too many files have changed in this diff Show More