mirror of
https://github.com/dbt-labs/dbt-core
synced 2025-12-18 22:11:27 +00:00
Compare commits
42 Commits
enable-pos
...
1.11.lates
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
30fdaad7fb | ||
|
|
fa1c26be1f | ||
|
|
d4f32bc31c | ||
|
|
073e0c3495 | ||
|
|
5abcb72861 | ||
|
|
c9a6157982 | ||
|
|
07f02943cd | ||
|
|
f38ed9f2fb | ||
|
|
3b232f63e2 | ||
|
|
f2059529f9 | ||
|
|
4322ff07ec | ||
|
|
4fc9e342f1 | ||
|
|
877f092f00 | ||
|
|
a55663d8b9 | ||
|
|
6fe01dd05b | ||
|
|
c17532f497 | ||
|
|
c54f43986c | ||
|
|
e20863c036 | ||
|
|
fae47a6dce | ||
|
|
58391c8506 | ||
|
|
53e9454c9b | ||
|
|
07ad7aeeb5 | ||
|
|
7a792f5bc8 | ||
|
|
0fae291191 | ||
|
|
9deae6ab8b | ||
|
|
f30e2f65c9 | ||
|
|
6bb8100a02 | ||
|
|
a318d9f3d7 | ||
|
|
ce2cd04019 | ||
|
|
b85794a0dc | ||
|
|
f619002fa6 | ||
|
|
4db02533d6 | ||
|
|
0723c5ecb3 | ||
|
|
829b8b2161 | ||
|
|
51c89d67b3 | ||
|
|
e6a608cd6c | ||
|
|
163674685f | ||
|
|
87d2d87737 | ||
|
|
ee6a69a7c3 | ||
|
|
0d59a3d5a4 | ||
|
|
b52eb6f8e7 | ||
|
|
8666c83f26 |
@@ -1,37 +0,0 @@
|
||||
[bumpversion]
|
||||
current_version = 1.11.0b4
|
||||
parse = (?P<major>[\d]+) # major version number
|
||||
\.(?P<minor>[\d]+) # minor version number
|
||||
\.(?P<patch>[\d]+) # patch version number
|
||||
(?P<prerelease> # optional pre-release - ex: a1, b2, rc25
|
||||
(?P<prekind>a|b|rc) # pre-release type
|
||||
(?P<num>[\d]+) # pre-release version number
|
||||
)?
|
||||
( # optional nightly release indicator
|
||||
\.(?P<nightly>dev[0-9]+) # ex: .dev02142023
|
||||
)? # expected matches: `1.15.0`, `1.5.0a11`, `1.5.0a1.dev123`, `1.5.0.dev123457`, expected failures: `1`, `1.5`, `1.5.2-a1`, `text1.5.0`
|
||||
serialize =
|
||||
{major}.{minor}.{patch}{prekind}{num}.{nightly}
|
||||
{major}.{minor}.{patch}.{nightly}
|
||||
{major}.{minor}.{patch}{prekind}{num}
|
||||
{major}.{minor}.{patch}
|
||||
commit = False
|
||||
tag = False
|
||||
|
||||
[bumpversion:part:prekind]
|
||||
first_value = a
|
||||
optional_value = final
|
||||
values =
|
||||
a
|
||||
b
|
||||
rc
|
||||
final
|
||||
|
||||
[bumpversion:part:num]
|
||||
first_value = 1
|
||||
|
||||
[bumpversion:part:nightly]
|
||||
|
||||
[bumpversion:file:core/pyproject.toml]
|
||||
search = version = "{current_version}"
|
||||
replace = version = "{new_version}"
|
||||
28
.changes/1.11.0-rc1.md
Normal file
28
.changes/1.11.0-rc1.md
Normal file
@@ -0,0 +1,28 @@
|
||||
## dbt-core 1.11.0-rc1 - November 18, 2025
|
||||
|
||||
### Features
|
||||
|
||||
- Allow for defining funciton arguments with default values ([#12044](https://github.com/dbt-labs/dbt-core/issues/12044))
|
||||
|
||||
### Fixes
|
||||
|
||||
- Fix parse error when build_after.count set to 0 ([#12136](https://github.com/dbt-labs/dbt-core/issues/12136))
|
||||
- Stop compiling python udfs like python models ([#12153](https://github.com/dbt-labs/dbt-core/issues/12153))
|
||||
- For metric names, fix bug allowing hyphens (not allowed in metricflow already), make validation throw ValidationErrors (not ParsingErrors), and add tests. ([#n/a](https://github.com/dbt-labs/dbt-core/issues/n/a))
|
||||
- Include macros in unit test parsing ([#10157](https://github.com/dbt-labs/dbt-core/issues/10157))
|
||||
|
||||
### Under the Hood
|
||||
|
||||
- add dbt/jsonschemas to manifest.in ([#12126](https://github.com/dbt-labs/dbt-core/issues/12126))
|
||||
- Move from setup.py to pyproject.toml ([#5696](https://github.com/dbt-labs/dbt-core/issues/5696))
|
||||
- Fixes issue where config isn't propagated to metric from measure when set as create_metric:True ([#None](https://github.com/dbt-labs/dbt-core/issues/None))
|
||||
- Support DBT_ENGINE prefix for record-mode env vars ([#12149](https://github.com/dbt-labs/dbt-core/issues/12149))
|
||||
|
||||
### Dependencies
|
||||
|
||||
- Drop support for python 3.9 ([#12118](https://github.com/dbt-labs/dbt-core/issues/12118))
|
||||
|
||||
### Contributors
|
||||
- [@WilliamDee](https://github.com/WilliamDee) ([#None](https://github.com/dbt-labs/dbt-core/issues/None))
|
||||
- [@nathanskone](https://github.com/nathanskone) ([#10157](https://github.com/dbt-labs/dbt-core/issues/10157))
|
||||
- [@theyostalservice](https://github.com/theyostalservice) ([#n/a](https://github.com/dbt-labs/dbt-core/issues/n/a))
|
||||
29
.changes/1.11.0-rc2.md
Normal file
29
.changes/1.11.0-rc2.md
Normal file
@@ -0,0 +1,29 @@
|
||||
## dbt-core 1.11.0-rc2 - December 01, 2025
|
||||
|
||||
### Features
|
||||
|
||||
- Support partial parsing for function nodes ([#12072](https://github.com/dbt-labs/dbt-core/issues/12072))
|
||||
|
||||
### Fixes
|
||||
|
||||
- Allow dbt deps to run when vars lack defaults in dbt_project.yml ([#8913](https://github.com/dbt-labs/dbt-core/issues/8913))
|
||||
- Restore DuplicateResourceNameError for intra-project node name duplication, behind behavior flag `require_unique_project_resource_names` ([#12152](https://github.com/dbt-labs/dbt-core/issues/12152))
|
||||
- Allow the usage of `function` with `--exclude-resource-type` flag ([#12143](https://github.com/dbt-labs/dbt-core/issues/12143))
|
||||
- Fix bug where schemas of functions weren't guaranteed to exist ([#12142](https://github.com/dbt-labs/dbt-core/issues/12142))
|
||||
- :bug: :snowman: Correctly reference foreign key references when --defer and --state provided ([#11885](https://github.com/dbt-labs/dbt-core/issues/11885))
|
||||
- Fix generation of deprecations summary ([#12146](https://github.com/dbt-labs/dbt-core/issues/12146))
|
||||
- :bug: :snowman: Add exception when using --state and referring to a removed test ([#10630](https://github.com/dbt-labs/dbt-core/issues/10630))
|
||||
- :bug: :snowman: Stop emitting `NoNodesForSelectionCriteria` three times during `build` command ([#11627](https://github.com/dbt-labs/dbt-core/issues/11627))
|
||||
- :bug: :snowman: Fix long Python stack traces appearing when package dependencies have incompatible version requirements ([#12049](https://github.com/dbt-labs/dbt-core/issues/12049))
|
||||
- :bug: :snowman: Fixed issue where changing data type size/precision/scale (e.g., varchar(3) to varchar(10)) incorrectly triggered a breaking change error fo ([#11186](https://github.com/dbt-labs/dbt-core/issues/11186))
|
||||
- :bug: :snowman: Support unit testing models that depend on sources with the same name ([#11975](https://github.com/dbt-labs/dbt-core/issues/11975), [#10433](https://github.com/dbt-labs/dbt-core/issues/10433))
|
||||
- :bug: :snowman: Avoid retrying successful run-operation commands ([#11850](https://github.com/dbt-labs/dbt-core/issues/11850))
|
||||
- :bug: :snowman: Fix `dbt deps --add-package` crash when packages.yml contains `warn-unpinned: false` ([#9104](https://github.com/dbt-labs/dbt-core/issues/9104))
|
||||
|
||||
### Under the Hood
|
||||
|
||||
- Update jsonschemas for schema.yml and dbt_project.yml deprecations ([#12180](https://github.com/dbt-labs/dbt-core/issues/12180))
|
||||
|
||||
### Contributors
|
||||
- [@asiunov](https://github.com/asiunov) ([#12146](https://github.com/dbt-labs/dbt-core/issues/12146))
|
||||
- [@michellark](https://github.com/michellark) ([#11885](https://github.com/dbt-labs/dbt-core/issues/11885))
|
||||
21
.changes/1.11.0-rc3.md
Normal file
21
.changes/1.11.0-rc3.md
Normal file
@@ -0,0 +1,21 @@
|
||||
## dbt-core 1.11.0-rc3 - December 08, 2025
|
||||
|
||||
### Features
|
||||
|
||||
- Raise jsonschema-based deprecation warnings by default ([#12240](https://github.com/dbt-labs/dbt-core/issues/12240))
|
||||
- :bug: :snowman: Disable unit tests whose model is disabled ([#10540](https://github.com/dbt-labs/dbt-core/issues/10540))
|
||||
|
||||
### Fixes
|
||||
|
||||
- Fix bug in partial parsing when updating a model with a schema file that is referenced by a singular test ([#12223](https://github.com/dbt-labs/dbt-core/issues/12223))
|
||||
- :bug: :snowman: Improve `dbt deps --add-package` duplicate detection with better cross-source matching and word boundaries ([#12239](https://github.com/dbt-labs/dbt-core/issues/12239))
|
||||
- :bug: :snowman: Fix false positive deprecation warning of pre/post-hook SQL configs ([#12244](https://github.com/dbt-labs/dbt-core/issues/12244))
|
||||
|
||||
### Under the Hood
|
||||
|
||||
- Replace setuptools and tox with hatch for build, test, and environment management. ([#12151](https://github.com/dbt-labs/dbt-core/issues/12151))
|
||||
|
||||
### Contributors
|
||||
- [@emmyoop](https://github.com/emmyoop) ([#12239](https://github.com/dbt-labs/dbt-core/issues/12239), [#12151](https://github.com/dbt-labs/dbt-core/issues/12151))
|
||||
- [@mattogburke](https://github.com/mattogburke) ([#12223](https://github.com/dbt-labs/dbt-core/issues/12223))
|
||||
- [@michelleark](https://github.com/michelleark) ([#12240](https://github.com/dbt-labs/dbt-core/issues/12240), [#10540](https://github.com/dbt-labs/dbt-core/issues/10540), [#12244](https://github.com/dbt-labs/dbt-core/issues/12244))
|
||||
22
.changes/1.11.0-rc4.md
Normal file
22
.changes/1.11.0-rc4.md
Normal file
@@ -0,0 +1,22 @@
|
||||
## dbt-core 1.11.0-rc4 - December 17, 2025
|
||||
|
||||
### Features
|
||||
|
||||
- Implement config.meta_get and config.meta_require ([#12012](https://github.com/dbt-labs/dbt-core/issues/12012))
|
||||
|
||||
### Fixes
|
||||
|
||||
- :bug: :snowman: Fix ref resolution within package when duplicate nodes exist, behind require_ref_searches_node_package_before_root behavior change flag ([#11351](https://github.com/dbt-labs/dbt-core/issues/11351))
|
||||
- Improve error message clarity when detecting nodes with space in name ([#11835](https://github.com/dbt-labs/dbt-core/issues/11835))
|
||||
- :bug: :snowman:Propagate exceptions for NodeFinished callbacks in dbtRunner ([#11612](https://github.com/dbt-labs/dbt-core/issues/11612))
|
||||
- Adds omitted return statement to RuntimeConfigObject.meta_require method ([#12288](https://github.com/dbt-labs/dbt-core/issues/12288))
|
||||
- Do not raise deprecation warning when encountering dataset or project configs for bigquery ([#12285](https://github.com/dbt-labs/dbt-core/issues/12285))
|
||||
|
||||
### Under the Hood
|
||||
|
||||
- Bump lower bound for dbt-common to 1.37.2 ([#12284](https://github.com/dbt-labs/dbt-core/issues/12284))
|
||||
|
||||
### Contributors
|
||||
- [@gshank](https://github.com/gshank) ([#12012](https://github.com/dbt-labs/dbt-core/issues/12012))
|
||||
- [@michelleark](https://github.com/michelleark) ([#11351](https://github.com/dbt-labs/dbt-core/issues/11351), [#11835](https://github.com/dbt-labs/dbt-core/issues/11835), [#11612](https://github.com/dbt-labs/dbt-core/issues/11612), [#12285](https://github.com/dbt-labs/dbt-core/issues/12285), [#12284](https://github.com/dbt-labs/dbt-core/issues/12284))
|
||||
- [@mjsqu](https://github.com/mjsqu) ([#12288](https://github.com/dbt-labs/dbt-core/issues/12288))
|
||||
6
.changes/1.11.0/Features-20251006-140352.yaml
Normal file
6
.changes/1.11.0/Features-20251006-140352.yaml
Normal file
@@ -0,0 +1,6 @@
|
||||
kind: Features
|
||||
body: Support partial parsing for function nodes
|
||||
time: 2025-10-06T14:03:52.258104-05:00
|
||||
custom:
|
||||
Author: QMalcolm
|
||||
Issue: "12072"
|
||||
6
.changes/1.11.0/Features-20251117-141053.yaml
Normal file
6
.changes/1.11.0/Features-20251117-141053.yaml
Normal file
@@ -0,0 +1,6 @@
|
||||
kind: Features
|
||||
body: Allow for defining funciton arguments with default values
|
||||
time: 2025-11-17T14:10:53.860178-06:00
|
||||
custom:
|
||||
Author: QMalcolm
|
||||
Issue: "12044"
|
||||
6
.changes/1.11.0/Features-20251201-165209.yaml
Normal file
6
.changes/1.11.0/Features-20251201-165209.yaml
Normal file
@@ -0,0 +1,6 @@
|
||||
kind: Features
|
||||
body: Raise jsonschema-based deprecation warnings by default
|
||||
time: 2025-12-01T16:52:09.354436-05:00
|
||||
custom:
|
||||
Author: michelleark
|
||||
Issue: 12240
|
||||
6
.changes/1.11.0/Features-20251203-122926.yaml
Normal file
6
.changes/1.11.0/Features-20251203-122926.yaml
Normal file
@@ -0,0 +1,6 @@
|
||||
kind: Features
|
||||
body: ':bug: :snowman: Disable unit tests whose model is disabled'
|
||||
time: 2025-12-03T12:29:26.209248-05:00
|
||||
custom:
|
||||
Author: michelleark
|
||||
Issue: "10540"
|
||||
6
.changes/1.11.0/Features-20251210-202001.yaml
Normal file
6
.changes/1.11.0/Features-20251210-202001.yaml
Normal file
@@ -0,0 +1,6 @@
|
||||
kind: Features
|
||||
body: Implement config.meta_get and config.meta_require
|
||||
time: 2025-12-10T20:20:01.354288-05:00
|
||||
custom:
|
||||
Author: gshank
|
||||
Issue: "12012"
|
||||
6
.changes/1.11.0/Fixes-20251117-140649.yaml
Normal file
6
.changes/1.11.0/Fixes-20251117-140649.yaml
Normal file
@@ -0,0 +1,6 @@
|
||||
kind: Fixes
|
||||
body: Include macros in unit test parsing
|
||||
time: 2025-11-17T14:06:49.518566-05:00
|
||||
custom:
|
||||
Author: michelleark nathanskone
|
||||
Issue: "10157"
|
||||
6
.changes/1.11.0/Fixes-20251117-185025.yaml
Normal file
6
.changes/1.11.0/Fixes-20251117-185025.yaml
Normal file
@@ -0,0 +1,6 @@
|
||||
kind: Fixes
|
||||
body: Allow dbt deps to run when vars lack defaults in dbt_project.yml
|
||||
time: 2025-11-17T18:50:25.759091+05:30
|
||||
custom:
|
||||
Author: 3loka
|
||||
Issue: "8913"
|
||||
6
.changes/1.11.0/Fixes-20251118-171106.yaml
Normal file
6
.changes/1.11.0/Fixes-20251118-171106.yaml
Normal file
@@ -0,0 +1,6 @@
|
||||
kind: Fixes
|
||||
body: Restore DuplicateResourceNameError for intra-project node name duplication, behind behavior flag `require_unique_project_resource_names`
|
||||
time: 2025-11-18T17:11:06.454784-05:00
|
||||
custom:
|
||||
Author: michelleark
|
||||
Issue: "12152"
|
||||
6
.changes/1.11.0/Fixes-20251119-195034.yaml
Normal file
6
.changes/1.11.0/Fixes-20251119-195034.yaml
Normal file
@@ -0,0 +1,6 @@
|
||||
kind: Fixes
|
||||
body: Allow the usage of `function` with `--exclude-resource-type` flag
|
||||
time: 2025-11-19T19:50:34.703236-06:00
|
||||
custom:
|
||||
Author: QMalcolm
|
||||
Issue: "12143"
|
||||
6
.changes/1.11.0/Fixes-20251124-155629.yaml
Normal file
6
.changes/1.11.0/Fixes-20251124-155629.yaml
Normal file
@@ -0,0 +1,6 @@
|
||||
kind: Fixes
|
||||
body: Fix bug where schemas of functions weren't guaranteed to exist
|
||||
time: 2025-11-24T15:56:29.467004-06:00
|
||||
custom:
|
||||
Author: QMalcolm
|
||||
Issue: "12142"
|
||||
6
.changes/1.11.0/Fixes-20251124-155756.yaml
Normal file
6
.changes/1.11.0/Fixes-20251124-155756.yaml
Normal file
@@ -0,0 +1,6 @@
|
||||
kind: Fixes
|
||||
body: Fix generation of deprecations summary
|
||||
time: 2025-11-24T15:57:56.544123-08:00
|
||||
custom:
|
||||
Author: asiunov
|
||||
Issue: "12146"
|
||||
6
.changes/1.11.0/Fixes-20251124-170855.yaml
Normal file
6
.changes/1.11.0/Fixes-20251124-170855.yaml
Normal file
@@ -0,0 +1,6 @@
|
||||
kind: Fixes
|
||||
body: ':bug: :snowman: Correctly reference foreign key references when --defer and --state provided'
|
||||
time: 2025-11-24T17:08:55.387946-05:00
|
||||
custom:
|
||||
Author: michellark
|
||||
Issue: "11885"
|
||||
7
.changes/1.11.0/Fixes-20251125-120246.yaml
Normal file
7
.changes/1.11.0/Fixes-20251125-120246.yaml
Normal file
@@ -0,0 +1,7 @@
|
||||
kind: Fixes
|
||||
body: ':bug: :snowman: Add exception when using --state and referring to a removed
|
||||
test'
|
||||
time: 2025-11-25T12:02:46.635026-05:00
|
||||
custom:
|
||||
Author: emmyoop
|
||||
Issue: "10630"
|
||||
6
.changes/1.11.0/Fixes-20251125-122020.yaml
Normal file
6
.changes/1.11.0/Fixes-20251125-122020.yaml
Normal file
@@ -0,0 +1,6 @@
|
||||
kind: Fixes
|
||||
body: ':bug: :snowman: Stop emitting `NoNodesForSelectionCriteria` three times during `build` command'
|
||||
time: 2025-11-25T12:20:20.132379-06:00
|
||||
custom:
|
||||
Author: QMalcolm
|
||||
Issue: "11627"
|
||||
6
.changes/1.11.0/Fixes-20251127-141308.yaml
Normal file
6
.changes/1.11.0/Fixes-20251127-141308.yaml
Normal file
@@ -0,0 +1,6 @@
|
||||
kind: Fixes
|
||||
body: ":bug: :snowman: Fix long Python stack traces appearing when package dependencies have incompatible version requirements"
|
||||
time: 2025-11-27T14:13:08.082542-05:00
|
||||
custom:
|
||||
Author: emmyoop
|
||||
Issue: "12049"
|
||||
7
.changes/1.11.0/Fixes-20251127-145929.yaml
Normal file
7
.changes/1.11.0/Fixes-20251127-145929.yaml
Normal file
@@ -0,0 +1,7 @@
|
||||
kind: Fixes
|
||||
body: ':bug: :snowman: Fixed issue where changing data type size/precision/scale (e.g.,
|
||||
varchar(3) to varchar(10)) incorrectly triggered a breaking change error fo'
|
||||
time: 2025-11-27T14:59:29.256274-05:00
|
||||
custom:
|
||||
Author: emmyoop
|
||||
Issue: "11186"
|
||||
6
.changes/1.11.0/Fixes-20251127-170124.yaml
Normal file
6
.changes/1.11.0/Fixes-20251127-170124.yaml
Normal file
@@ -0,0 +1,6 @@
|
||||
kind: Fixes
|
||||
body: ':bug: :snowman: Support unit testing models that depend on sources with the same name'
|
||||
time: 2025-11-27T17:01:24.193516-05:00
|
||||
custom:
|
||||
Author: michelleark
|
||||
Issue: 11975 10433
|
||||
6
.changes/1.11.0/Fixes-20251128-102129.yaml
Normal file
6
.changes/1.11.0/Fixes-20251128-102129.yaml
Normal file
@@ -0,0 +1,6 @@
|
||||
kind: Fixes
|
||||
body: Fix bug in partial parsing when updating a model with a schema file that is referenced by a singular test
|
||||
time: 2025-11-28T10:21:29.911147Z
|
||||
custom:
|
||||
Author: mattogburke
|
||||
Issue: "12223"
|
||||
6
.changes/1.11.0/Fixes-20251128-122838.yaml
Normal file
6
.changes/1.11.0/Fixes-20251128-122838.yaml
Normal file
@@ -0,0 +1,6 @@
|
||||
kind: Fixes
|
||||
body: ':bug: :snowman: Avoid retrying successful run-operation commands'
|
||||
time: 2025-11-28T12:28:38.546261-05:00
|
||||
custom:
|
||||
Author: michelleark
|
||||
Issue: "11850"
|
||||
7
.changes/1.11.0/Fixes-20251128-161937.yaml
Normal file
7
.changes/1.11.0/Fixes-20251128-161937.yaml
Normal file
@@ -0,0 +1,7 @@
|
||||
kind: Fixes
|
||||
body: ':bug: :snowman: Fix `dbt deps --add-package` crash when packages.yml contains `warn-unpinned:
|
||||
false`'
|
||||
time: 2025-11-28T16:19:37.608722-05:00
|
||||
custom:
|
||||
Author: emmyoop
|
||||
Issue: "9104"
|
||||
7
.changes/1.11.0/Fixes-20251128-163144.yaml
Normal file
7
.changes/1.11.0/Fixes-20251128-163144.yaml
Normal file
@@ -0,0 +1,7 @@
|
||||
kind: Fixes
|
||||
body: ':bug: :snowman: Improve `dbt deps --add-package` duplicate detection with better
|
||||
cross-source matching and word boundaries'
|
||||
time: 2025-11-28T16:31:44.344099-05:00
|
||||
custom:
|
||||
Author: emmyoop
|
||||
Issue: "12239"
|
||||
6
.changes/1.11.0/Fixes-20251202-133705.yaml
Normal file
6
.changes/1.11.0/Fixes-20251202-133705.yaml
Normal file
@@ -0,0 +1,6 @@
|
||||
kind: Fixes
|
||||
body: ':bug: :snowman: Fix false positive deprecation warning of pre/post-hook SQL configs'
|
||||
time: 2025-12-02T13:37:05.012112-05:00
|
||||
custom:
|
||||
Author: michelleark
|
||||
Issue: "12244"
|
||||
6
.changes/1.11.0/Fixes-20251204-094753.yaml
Normal file
6
.changes/1.11.0/Fixes-20251204-094753.yaml
Normal file
@@ -0,0 +1,6 @@
|
||||
kind: Fixes
|
||||
body: ':bug: :snowman: Fix ref resolution within package when duplicate nodes exist, behind require_ref_searches_node_package_before_root behavior change flag'
|
||||
time: 2025-12-04T09:47:53.349428-08:00
|
||||
custom:
|
||||
Author: michelleark
|
||||
Issue: "11351"
|
||||
6
.changes/1.11.0/Fixes-20251210-143935.yaml
Normal file
6
.changes/1.11.0/Fixes-20251210-143935.yaml
Normal file
@@ -0,0 +1,6 @@
|
||||
kind: Fixes
|
||||
body: Improve error message clarity when detecting nodes with space in name
|
||||
time: 2025-12-10T14:39:35.107841-08:00
|
||||
custom:
|
||||
Author: michelleark
|
||||
Issue: "11835"
|
||||
6
.changes/1.11.0/Fixes-20251216-120727.yaml
Normal file
6
.changes/1.11.0/Fixes-20251216-120727.yaml
Normal file
@@ -0,0 +1,6 @@
|
||||
kind: Fixes
|
||||
body: ':bug: :snowman:Propagate exceptions for NodeFinished callbacks in dbtRunner'
|
||||
time: 2025-12-16T12:07:27.576087-05:00
|
||||
custom:
|
||||
Author: michelleark
|
||||
Issue: "11612"
|
||||
6
.changes/1.11.0/Fixes-20251217-002813.yaml
Normal file
6
.changes/1.11.0/Fixes-20251217-002813.yaml
Normal file
@@ -0,0 +1,6 @@
|
||||
kind: Fixes
|
||||
body: Adds omitted return statement to RuntimeConfigObject.meta_require method
|
||||
time: 2025-12-17T00:28:13.015416197Z
|
||||
custom:
|
||||
Author: mjsqu
|
||||
Issue: "12288"
|
||||
6
.changes/1.11.0/Fixes-20251217-105918.yaml
Normal file
6
.changes/1.11.0/Fixes-20251217-105918.yaml
Normal file
@@ -0,0 +1,6 @@
|
||||
kind: Fixes
|
||||
body: Do not raise deprecation warning when encountering dataset or project configs for bigquery
|
||||
time: 2025-12-17T10:59:18.372968-05:00
|
||||
custom:
|
||||
Author: michelleark
|
||||
Issue: "12285"
|
||||
6
.changes/1.11.0/Under the Hood-20251119-110110.yaml
Normal file
6
.changes/1.11.0/Under the Hood-20251119-110110.yaml
Normal file
@@ -0,0 +1,6 @@
|
||||
kind: Under the Hood
|
||||
body: Update jsonschemas for schema.yml and dbt_project.yml deprecations
|
||||
time: 2025-11-19T11:01:10.616676-05:00
|
||||
custom:
|
||||
Author: michelleark
|
||||
Issue: "12180"
|
||||
6
.changes/1.11.0/Under the Hood-20251121-140515.yaml
Normal file
6
.changes/1.11.0/Under the Hood-20251121-140515.yaml
Normal file
@@ -0,0 +1,6 @@
|
||||
kind: Under the Hood
|
||||
body: Replace setuptools and tox with hatch for build, test, and environment management.
|
||||
time: 2025-11-21T14:05:15.838252-05:00
|
||||
custom:
|
||||
Author: emmyoop
|
||||
Issue: "12151"
|
||||
6
.changes/1.11.0/Under the Hood-20251215-155046.yaml
Normal file
6
.changes/1.11.0/Under the Hood-20251215-155046.yaml
Normal file
@@ -0,0 +1,6 @@
|
||||
kind: Under the Hood
|
||||
body: Bump lower bound for dbt-common to 1.37.2
|
||||
time: 2025-12-15T15:50:46.857793-05:00
|
||||
custom:
|
||||
Author: michelleark
|
||||
Issue: "12284"
|
||||
@@ -41,32 +41,26 @@ newlines:
|
||||
endOfVersion: 1
|
||||
|
||||
custom:
|
||||
- key: Author
|
||||
label: GitHub Username(s) (separated by a single space if multiple)
|
||||
type: string
|
||||
minLength: 3
|
||||
- key: Issue
|
||||
label: GitHub Issue Number (separated by a single space if multiple)
|
||||
type: string
|
||||
minLength: 1
|
||||
- key: Author
|
||||
label: GitHub Username(s) (separated by a single space if multiple)
|
||||
type: string
|
||||
minLength: 3
|
||||
- key: Issue
|
||||
label: GitHub Issue Number (separated by a single space if multiple)
|
||||
type: string
|
||||
minLength: 1
|
||||
|
||||
footerFormat: |
|
||||
{{- $contributorDict := dict }}
|
||||
{{- /* ensure all names in this list are all lowercase for later matching purposes */}}
|
||||
{{- $core_team := splitList " " .Env.CORE_TEAM }}
|
||||
{{- /* ensure we always skip snyk and dependabot in addition to the core team */}}
|
||||
{{- $maintainers := list "dependabot[bot]" "snyk-bot"}}
|
||||
{{- range $team_member := $core_team }}
|
||||
{{- $team_member_lower := lower $team_member }}
|
||||
{{- $maintainers = append $maintainers $team_member_lower }}
|
||||
{{- end }}
|
||||
{{- /* ensure we always skip snyk and dependabot */}}
|
||||
{{- $bots := list "dependabot[bot]" "snyk-bot"}}
|
||||
{{- range $change := .Changes }}
|
||||
{{- $authorList := splitList " " $change.Custom.Author }}
|
||||
{{- /* loop through all authors for a single changelog */}}
|
||||
{{- range $author := $authorList }}
|
||||
{{- $authorLower := lower $author }}
|
||||
{{- /* we only want to include non-core team contributors */}}
|
||||
{{- if not (has $authorLower $maintainers)}}
|
||||
{{- /* we only want to include non-bot contributors */}}
|
||||
{{- if not (has $authorLower $bots)}}
|
||||
{{- $changeList := splitList " " $change.Custom.Author }}
|
||||
{{- $IssueList := list }}
|
||||
{{- $changeLink := $change.Kind }}
|
||||
|
||||
1
.flake8
1
.flake8
@@ -10,6 +10,5 @@ ignore =
|
||||
E704 # makes Flake8 work like black
|
||||
E741
|
||||
E501 # long line checking is done in black
|
||||
exclude = test/
|
||||
per-file-ignores =
|
||||
*/__init__.py: F401
|
||||
|
||||
@@ -1 +1 @@
|
||||
../../../test/setup_db.sh
|
||||
../../../scripts/setup_db.sh
|
||||
388
.github/workflows/cut-release-branch.yml
vendored
388
.github/workflows/cut-release-branch.yml
vendored
@@ -1,25 +1,44 @@
|
||||
# **what?**
|
||||
# Cuts a new `*.latest` branch
|
||||
# Also cleans up all files in `.changes/unreleased` and `.changes/previous verion on
|
||||
# `main` and bumps `main` to the input version.
|
||||
# Cuts the `*.latest` branch, bumps dependencies on it, cleans up all files in `.changes/unreleased`
|
||||
# and `.changes/previous verion on main and bumps main to the input version.
|
||||
|
||||
# **why?**
|
||||
# Generally reduces the workload of engineers and reduces error. Allow automation.
|
||||
# Clean up the main branch after a release branch is cut and automate cutting the release branch.
|
||||
# Generally reduces the workload of engineers and reducing error.
|
||||
|
||||
# **when?**
|
||||
# This will run when called manually.
|
||||
# This will run when called manually or when triggered in another workflow.
|
||||
|
||||
# Example Usage including required permissions: TODO: update once finalized
|
||||
|
||||
# permissions:
|
||||
# contents: read
|
||||
# pull-requests: write
|
||||
#
|
||||
# name: Cut Release Branch
|
||||
# jobs:
|
||||
# changelog:
|
||||
# uses: dbt-labs/actions/.github/workflows/cut-release-branch.yml@main
|
||||
# with:
|
||||
# new_branch_name: 1.7.latest
|
||||
# PR_title: "Cleanup main after cutting new 1.7.latest branch"
|
||||
# PR_body: "All adapter PRs will fail CI until the dbt-core PR has been merged due to release version conflicts."
|
||||
# secrets:
|
||||
# FISHTOWN_BOT_PAT: ${{ secrets.FISHTOWN_BOT_PAT }}
|
||||
|
||||
# TODOs
|
||||
# add note to eventually commit changes directly and bypass checks - same as release - when we move to this model run test action after merge
|
||||
|
||||
name: Cut new release branch
|
||||
run-name: "Cutting New Branch: ${{ inputs.new_branch_name }}"
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
version_to_bump_main:
|
||||
description: 'The alpha version main should bump to (ex. 1.6.0a1)'
|
||||
required: true
|
||||
new_branch_name:
|
||||
description: 'The full name of the new branch (ex. 1.5.latest)'
|
||||
description: "The full name of the new branch (ex. 1.5.latest)"
|
||||
required: true
|
||||
type: string
|
||||
|
||||
defaults:
|
||||
run:
|
||||
@@ -27,15 +46,346 @@ defaults:
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
pull-requests: write
|
||||
|
||||
env:
|
||||
PYTHON_TARGET_VERSION: "3.10"
|
||||
PR_TITLE: "Cleanup main after cutting new ${{ inputs.new_branch_name }} branch"
|
||||
PR_BODY: "All adapter PRs will fail CI until the dbt-core PR has been merged due to release version conflicts."
|
||||
|
||||
jobs:
|
||||
cut_branch:
|
||||
name: "Cut branch and clean up main for dbt-core"
|
||||
uses: dbt-labs/actions/.github/workflows/cut-release-branch.yml@main
|
||||
with:
|
||||
version_to_bump_main: ${{ inputs.version_to_bump_main }}
|
||||
new_branch_name: ${{ inputs.new_branch_name }}
|
||||
PR_title: "Cleanup main after cutting new ${{ inputs.new_branch_name }} branch"
|
||||
PR_body: "All adapter PRs will fail CI until the dbt-core PR has been merged due to release version conflicts."
|
||||
secrets:
|
||||
FISHTOWN_BOT_PAT: ${{ secrets.FISHTOWN_BOT_PAT }}
|
||||
prep_work:
|
||||
name: "Prep Work"
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: "[DEBUG] Print Inputs"
|
||||
run: |
|
||||
echo "new_branch_name: ${{ inputs.new_branch_name }}"
|
||||
echo "PR_title: ${{ env.PR_TITLE }}"
|
||||
echo "PR_body: ${{ env.PR_BODY }}"
|
||||
|
||||
create_temp_branch:
|
||||
name: "Create Temp branch off main"
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
temp_branch_name: ${{ steps.variables.outputs.BRANCH_NAME }}
|
||||
|
||||
steps:
|
||||
- name: "Set Branch Value"
|
||||
id: variables
|
||||
run: |
|
||||
echo "BRANCH_NAME=cutting_release_branch/main_cleanup_$GITHUB_RUN_ID" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: "Checkout ${{ github.repository }}"
|
||||
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # actions/checkout@v4
|
||||
with:
|
||||
ref: "main"
|
||||
token: ${{ secrets.FISHTOWN_BOT_PAT }}
|
||||
|
||||
- name: "Create PR Branch"
|
||||
run: |
|
||||
user="Github Build Bot"
|
||||
email="buildbot@fishtownanalytics.com"
|
||||
git config user.name "$user"
|
||||
git config user.email "$email"
|
||||
git checkout -b ${{ steps.variables.outputs.BRANCH_NAME }}
|
||||
git push --set-upstream origin ${{ steps.variables.outputs.BRANCH_NAME }}
|
||||
|
||||
- name: "[Notification] Temp branch created"
|
||||
run: |
|
||||
message="Temp branch ${{ steps.variables.outputs.BRANCH_NAME }} created"
|
||||
echo "::notice title="Temporary branch created": $title::$message"
|
||||
|
||||
cleanup_changelog:
|
||||
name: "Clean Up Changelog"
|
||||
needs: ["create_temp_branch"]
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
next-version: ${{ steps.semver-current.outputs.next-minor-alpha-version }}
|
||||
|
||||
steps:
|
||||
- name: "Checkout ${{ github.repository }}"
|
||||
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ needs.create_temp_branch.outputs.temp_branch_name }}
|
||||
token: ${{ secrets.FISHTOWN_BOT_PAT }}
|
||||
|
||||
- name: "Add Homebrew To PATH"
|
||||
run: |
|
||||
echo "/home/linuxbrew/.linuxbrew/bin:/home/linuxbrew/.linuxbrew/sbin" >> $GITHUB_PATH
|
||||
|
||||
- name: "Install Homebrew Packages"
|
||||
run: |
|
||||
brew install pre-commit
|
||||
brew tap miniscruff/changie https://github.com/miniscruff/changie
|
||||
brew install changie
|
||||
|
||||
- name: "Check Current Version In Code"
|
||||
id: determine_version
|
||||
run: |
|
||||
current_version=$(grep '^version = ' core/pyproject.toml | sed 's/version = "\(.*\)"/\1/')
|
||||
echo "current_version=$current_version" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: "[Notification] Check Current Version In Code"
|
||||
run: |
|
||||
message="The current version is ${{ steps.determine_version.outputs.current_version }}"
|
||||
echo "::notice title="Version Bump Check": $title::$message"
|
||||
|
||||
- name: "Parse Current Version Into Parts for Changelog Directories"
|
||||
id: semver-current
|
||||
uses: dbt-labs/actions/parse-semver@main
|
||||
with:
|
||||
version: ${{ steps.determine_version.outputs.current_version }}
|
||||
|
||||
- name: "[Notification] Next Alpha Version"
|
||||
run: |
|
||||
message="The next alpha version is ${{ steps.semver-current.outputs.next-minor-alpha-version }}"
|
||||
echo "::notice title="Version Bump Check": $title::$message"
|
||||
|
||||
- name: "Delete Unreleased Changelog YAMLs"
|
||||
# removal fails if no files exist. OK to continue since we're just cleaning up the files.
|
||||
continue-on-error: true
|
||||
run: |
|
||||
rm .changes/unreleased/*.yaml || true
|
||||
|
||||
- name: "Delete Pre Release Changelogs and YAMLs"
|
||||
# removal fails if no files exist. OK to continue since we're just cleaning up the files.
|
||||
continue-on-error: true
|
||||
run: |
|
||||
rm .changes/${{ steps.semver-current.outputs.base-version }}/*.yaml || true
|
||||
rm .changes/${{ steps.semver-current.outputs.major }}.${{ steps.semver-current.outputs.minor }}.*.md || true
|
||||
|
||||
- name: "Cleanup CHANGELOG.md"
|
||||
run: |
|
||||
changie merge
|
||||
|
||||
- name: "Commit Changelog Cleanup to Branch"
|
||||
run: |
|
||||
user="Github Build Bot"
|
||||
email="buildbot@fishtownanalytics.com"
|
||||
git config user.name "$user"
|
||||
git config user.email "$email"
|
||||
git status
|
||||
git add .
|
||||
git commit -m "Clean up changelog on main"
|
||||
git push
|
||||
|
||||
- name: "[Notification] Changelog cleaned up"
|
||||
run: |
|
||||
message="Changelog on ${{ needs.create_temp_branch.outputs.temp_branch_name }} cleaned up"
|
||||
echo "::notice title="Changelog cleaned up": $title::$message"
|
||||
|
||||
bump_version:
|
||||
name: "Bump to next minor version"
|
||||
needs: ["cleanup_changelog", "create_temp_branch"]
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: "Checkout ${{ github.repository }}"
|
||||
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ needs.create_temp_branch.outputs.temp_branch_name }}
|
||||
token: ${{ secrets.FISHTOWN_BOT_PAT }}
|
||||
|
||||
- name: "Set up Python - ${{ env.PYTHON_TARGET_VERSION }}"
|
||||
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # actions/setup-python@v5
|
||||
with:
|
||||
python-version: "${{ env.PYTHON_TARGET_VERSION }}"
|
||||
|
||||
- name: "Install Spark Dependencies"
|
||||
if: ${{ contains(github.repository, 'dbt-labs/dbt-spark') }}
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install libsasl2-dev
|
||||
|
||||
- name: "Install Python Dependencies"
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
python -m pip install hatch
|
||||
|
||||
- name: "Bump Version To ${{ needs.cleanup_changelog.outputs.next-version }}"
|
||||
run: |
|
||||
cd core
|
||||
hatch version ${{ needs.cleanup_changelog.outputs.next-version }}
|
||||
hatch run dev-req
|
||||
dbt --version
|
||||
|
||||
- name: "Commit Version Bump to Branch"
|
||||
run: |
|
||||
user="Github Build Bot"
|
||||
email="buildbot@fishtownanalytics.com"
|
||||
git config user.name "$user"
|
||||
git config user.email "$email"
|
||||
git status
|
||||
git add .
|
||||
git commit -m "Bumping version to ${{ needs.cleanup_changelog.outputs.next-version }}"
|
||||
git push
|
||||
|
||||
- name: "[Notification] Version Bump completed"
|
||||
run: |
|
||||
message="Version on ${{ needs.create_temp_branch.outputs.temp_branch_name }} bumped to ${{ needs.cleanup_changelog.outputs.next-version }}"
|
||||
echo "::notice title="Version Bump Completed": $title::$message"
|
||||
|
||||
cleanup:
|
||||
name: "Cleanup Code Quality"
|
||||
needs: ["create_temp_branch", "bump_version"]
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: "Checkout ${{ github.repository }}"
|
||||
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ needs.create_temp_branch.outputs.temp_branch_name }}
|
||||
token: ${{ secrets.FISHTOWN_BOT_PAT }}
|
||||
|
||||
- name: "Add Homebrew To PATH"
|
||||
run: |
|
||||
echo "/home/linuxbrew/.linuxbrew/bin:/home/linuxbrew/.linuxbrew/sbin" >> $GITHUB_PATH
|
||||
|
||||
- name: "brew install pre-commit"
|
||||
run: |
|
||||
brew install pre-commit
|
||||
|
||||
# this step will fail on whitespace errors but also correct them
|
||||
- name: "Cleanup - Remove Trailing Whitespace Via Pre-commit"
|
||||
continue-on-error: true
|
||||
run: |
|
||||
pre-commit run trailing-whitespace --files CHANGELOG.md .changes/* || true
|
||||
|
||||
# this step will fail on newline errors but also correct them
|
||||
- name: "Cleanup - Remove Extra Newlines Via Pre-commit"
|
||||
continue-on-error: true
|
||||
run: |
|
||||
pre-commit run end-of-file-fixer --files CHANGELOG.md .changes/* || true
|
||||
|
||||
- name: "Commit Version Bump to Branch"
|
||||
run: |
|
||||
user="Github Build Bot"
|
||||
email="buildbot@fishtownanalytics.com"
|
||||
git config user.name "$user"
|
||||
git config user.email "$email"
|
||||
git status
|
||||
git add .
|
||||
git commit -m "Code quality cleanup"
|
||||
git push
|
||||
|
||||
open_pr:
|
||||
name: "Open PR Against main"
|
||||
needs: ["cleanup_changelog", "create_temp_branch", "cleanup"]
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
pr_number: ${{ steps.create_pr.outputs.pull-request-number }}
|
||||
|
||||
steps:
|
||||
- name: "Checkout ${{ github.repository }}"
|
||||
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ needs.create_temp_branch.outputs.temp_branch_name }}
|
||||
token: ${{ secrets.FISHTOWN_BOT_PAT }}
|
||||
|
||||
- name: "Determine PR Title"
|
||||
id: pr_title
|
||||
run: |
|
||||
echo "pr_title=${{ env.PR_TITLE }}" >> $GITHUB_OUTPUT
|
||||
if [${{ env.PR_TITLE }} == ""]; then
|
||||
echo "pr_title='Clean up changelogs and bump to version ${{ needs.cleanup_changelog.outputs.next-version }}'" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
|
||||
- name: "Determine PR Body"
|
||||
id: pr_body
|
||||
run: |
|
||||
echo "pr_body=${{ env.PR_BODY }}" >> $GITHUB_OUTPUT
|
||||
if [${{ env.PR_BODY }} == ""]; then
|
||||
echo "pr_body='Clean up changelogs and bump to version ${{ needs.cleanup_changelog.outputs.next-version }}'" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
|
||||
- name: "Add Branch Details"
|
||||
id: pr_body_branch
|
||||
run: |
|
||||
branch_details="The workflow that generated this PR also created a new branch: ${{ inputs.new_branch_name }}"
|
||||
full_body="${{ steps.pr_body.outputs.pr_body }} $branch_details"
|
||||
echo "pr_full_body=$full_body" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: "Open Pull Request"
|
||||
id: create_pr
|
||||
run: |
|
||||
pr_url=$(gh pr create -B main -H ${{ needs.create_temp_branch.outputs.temp_branch_name }} -l "Skip Changelog" -t "${{ steps.pr_title.outputs.pr_title }}" -b "${{ steps.pr_body_branch.outputs.pr_full_body }}")
|
||||
echo "pr_url=$pr_url" >> $GITHUB_OUTPUT
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.FISHTOWN_BOT_PAT }}
|
||||
|
||||
- name: "[Notification] Pull Request Opened"
|
||||
run: |
|
||||
message="PR opened at ${{ steps.create_pr.outputs.pr_url }}"
|
||||
echo "::notice title="Pull Request Opened": $title::$message"
|
||||
|
||||
cut_new_branch:
|
||||
# don't cut the new branch until we're done opening the PR against main
|
||||
name: "Cut New Branch ${{ inputs.new_branch_name }}"
|
||||
needs: [open_pr]
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: "Checkout ${{ github.repository }}"
|
||||
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # actions/checkout@v4
|
||||
with:
|
||||
token: ${{ secrets.FISHTOWN_BOT_PAT }}
|
||||
fetch-depth: 0
|
||||
|
||||
- name: "Ensure New Branch Does Not Exist"
|
||||
id: check_new_branch
|
||||
run: |
|
||||
title="Check New Branch Existence"
|
||||
if git show-ref --quiet ${{ inputs.new_branch_name }}; then
|
||||
message="Branch ${{ inputs.new_branch_name }} already exists. Exiting."
|
||||
echo "::error $title::$message"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: "Create New Release Branch"
|
||||
run: |
|
||||
git checkout -b ${{ inputs.new_branch_name }}
|
||||
|
||||
- name: "Push up New Branch"
|
||||
run: |
|
||||
#Data for commit
|
||||
user="Github Build Bot"
|
||||
email="buildbot@fishtownanalytics.com"
|
||||
git config user.name "$user"
|
||||
git config user.email "$email"
|
||||
git push --set-upstream origin ${{ inputs.new_branch_name }}
|
||||
|
||||
- name: "[Notification] New branch created"
|
||||
run: |
|
||||
message="New branch ${{ inputs.new_branch_name }} created"
|
||||
echo "::notice title="New branch created": $title::$message"
|
||||
|
||||
- name: "Bump dependencies via script"
|
||||
# This bumps the dependency on dbt-core in the adapters
|
||||
if: ${{ !contains(github.repository, 'dbt-core') }}
|
||||
run: |
|
||||
echo ${{ github.repository }}
|
||||
echo "running update_dependencies script"
|
||||
bash ${GITHUB_WORKSPACE}/.github/scripts/update_dependencies.sh ${{ inputs.new_branch_name }}
|
||||
commit_message="bumping .latest branch variable in update_dependencies.sh to ${{ inputs.new_branch_name }}"
|
||||
git status
|
||||
git add .
|
||||
git commit -m "$commit_message"
|
||||
git push
|
||||
|
||||
- name: "Bump env variable via script"
|
||||
# bumps the RELEASE_BRANCH variable in nightly-release.yml in adapters
|
||||
if: ${{ !contains(github.repository, 'dbt-core') }}
|
||||
run: |
|
||||
file="./.github/scripts/update_release_branch.sh"
|
||||
if test -f "$file"; then
|
||||
echo ${{ github.repository }}
|
||||
echo "running some script yet to be written now"
|
||||
bash $file ${{ inputs.new_branch_name }}
|
||||
commit_message="updating env variable to ${{ inputs.new_branch_name }} in nightly-release.yml"
|
||||
git status
|
||||
git add .
|
||||
git commit -m "$commit_message"
|
||||
git push
|
||||
else
|
||||
echo "no $file seen skipping step"
|
||||
fi
|
||||
|
||||
92
.github/workflows/main.yml
vendored
92
.github/workflows/main.yml
vendored
@@ -54,22 +54,28 @@ jobs:
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # actions/setup-python@v6
|
||||
with:
|
||||
python-version: '3.10'
|
||||
python-version: "3.10"
|
||||
|
||||
- name: Install python dependencies
|
||||
run: |
|
||||
python -m pip install --user --upgrade pip
|
||||
python -m pip --version
|
||||
make dev
|
||||
make dev_req
|
||||
mypy --version
|
||||
dbt --version
|
||||
python -m pip install hatch
|
||||
cd core
|
||||
hatch -v run setup
|
||||
|
||||
- name: Verify dbt installation
|
||||
run: |
|
||||
cd core
|
||||
hatch run dbt --version
|
||||
|
||||
- name: Run pre-commit hooks
|
||||
run: pre-commit run --all-files --show-diff-on-failure
|
||||
run: |
|
||||
cd core
|
||||
hatch run code-quality
|
||||
|
||||
unit:
|
||||
name: unit test / python ${{ matrix.python-version }}
|
||||
name: "unit test / python ${{ matrix.python-version }}"
|
||||
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
@@ -77,10 +83,7 @@ jobs:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
python-version: ["3.10", "3.11", "3.12", "3.13" ]
|
||||
|
||||
env:
|
||||
TOXENV: "unit"
|
||||
python-version: ["3.10", "3.11", "3.12", "3.13"]
|
||||
|
||||
steps:
|
||||
- name: Check out the repository
|
||||
@@ -95,15 +98,15 @@ jobs:
|
||||
run: |
|
||||
python -m pip install --user --upgrade pip
|
||||
python -m pip --version
|
||||
python -m pip install tox
|
||||
tox --version
|
||||
python -m pip install hatch
|
||||
hatch --version
|
||||
|
||||
- name: Run unit tests
|
||||
uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # nick-fields/retry@v3
|
||||
with:
|
||||
timeout_minutes: 10
|
||||
max_attempts: 3
|
||||
command: tox -e unit
|
||||
command: cd core && hatch -v run ci:unit-tests
|
||||
|
||||
- name: Get current date
|
||||
if: always()
|
||||
@@ -118,6 +121,7 @@ jobs:
|
||||
with:
|
||||
token: ${{ secrets.CODECOV_TOKEN }}
|
||||
flags: unit
|
||||
fail_ci_if_error: false
|
||||
|
||||
integration-metadata:
|
||||
name: integration test metadata generation
|
||||
@@ -155,7 +159,7 @@ jobs:
|
||||
echo "include=${INCLUDE_GROUPS}" >> $GITHUB_OUTPUT
|
||||
|
||||
integration-postgres:
|
||||
name: (${{ matrix.split-group }}) integration test / python ${{ matrix.python-version }} / ${{ matrix.os }}
|
||||
name: "(${{ matrix.split-group }}) integration test / python ${{ matrix.python-version }} / ${{ matrix.os }}"
|
||||
|
||||
runs-on: ${{ matrix.os }}
|
||||
timeout-minutes: 30
|
||||
@@ -164,11 +168,10 @@ jobs:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
python-version: [ "3.10", "3.11", "3.12", "3.13" ]
|
||||
python-version: ["3.10", "3.11", "3.12", "3.13"]
|
||||
os: ["ubuntu-latest"]
|
||||
split-group: ${{ fromJson(needs.integration-metadata.outputs.split-groups) }}
|
||||
env:
|
||||
TOXENV: integration
|
||||
DBT_INVOCATION_ENV: github-actions
|
||||
DBT_TEST_USER_1: dbt_test_user_1
|
||||
DBT_TEST_USER_2: dbt_test_user_2
|
||||
@@ -208,7 +211,7 @@ jobs:
|
||||
|
||||
- name: Run postgres setup script
|
||||
run: |
|
||||
./test/setup_db.sh
|
||||
./scripts/setup_db.sh
|
||||
env:
|
||||
PGHOST: localhost
|
||||
PGPORT: 5432
|
||||
@@ -218,17 +221,16 @@ jobs:
|
||||
run: |
|
||||
python -m pip install --user --upgrade pip
|
||||
python -m pip --version
|
||||
python -m pip install tox
|
||||
tox --version
|
||||
python -m pip install hatch
|
||||
hatch --version
|
||||
|
||||
- name: Run integration tests
|
||||
uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # nick-fields/retry@v3
|
||||
with:
|
||||
timeout_minutes: 30
|
||||
max_attempts: 3
|
||||
command: tox -- --ddtrace
|
||||
env:
|
||||
PYTEST_ADDOPTS: ${{ format('--splits {0} --group {1}', env.PYTHON_INTEGRATION_TEST_WORKERS, matrix.split-group) }}
|
||||
shell: bash
|
||||
command: cd core && hatch -v run ci:integration-tests -- --ddtrace --splits ${{ env.PYTHON_INTEGRATION_TEST_WORKERS }} --group ${{ matrix.split-group }}
|
||||
|
||||
- name: Get current date
|
||||
if: always()
|
||||
@@ -249,6 +251,7 @@ jobs:
|
||||
with:
|
||||
token: ${{ secrets.CODECOV_TOKEN }}
|
||||
flags: integration
|
||||
fail_ci_if_error: false
|
||||
|
||||
integration-mac-windows:
|
||||
name: (${{ matrix.split-group }}) integration test / python ${{ matrix.python-version }} / ${{ matrix.os }}
|
||||
@@ -263,7 +266,6 @@ jobs:
|
||||
# already includes split group and runs mac + windows
|
||||
include: ${{ fromJson(needs.integration-metadata.outputs.include) }}
|
||||
env:
|
||||
TOXENV: integration
|
||||
DBT_INVOCATION_ENV: github-actions
|
||||
DBT_TEST_USER_1: dbt_test_user_1
|
||||
DBT_TEST_USER_2: dbt_test_user_2
|
||||
@@ -290,7 +292,7 @@ jobs:
|
||||
with:
|
||||
timeout_minutes: 10
|
||||
max_attempts: 3
|
||||
command: ./test/setup_db.sh
|
||||
command: ./scripts/setup_db.sh
|
||||
|
||||
- name: Set up postgres (windows)
|
||||
if: runner.os == 'Windows'
|
||||
@@ -300,17 +302,16 @@ jobs:
|
||||
run: |
|
||||
python -m pip install --user --upgrade pip
|
||||
python -m pip --version
|
||||
python -m pip install tox
|
||||
tox --version
|
||||
python -m pip install hatch
|
||||
hatch --version
|
||||
|
||||
- name: Run integration tests
|
||||
uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # nick-fields/retry@v3
|
||||
with:
|
||||
timeout_minutes: 30
|
||||
max_attempts: 3
|
||||
command: tox -- --ddtrace
|
||||
env:
|
||||
PYTEST_ADDOPTS: ${{ format('--splits {0} --group {1}', env.PYTHON_INTEGRATION_TEST_WORKERS, matrix.split-group) }}
|
||||
shell: bash
|
||||
command: cd core && hatch -v run ci:integration-tests -- --ddtrace --splits ${{ env.PYTHON_INTEGRATION_TEST_WORKERS }} --group ${{ matrix.split-group }}
|
||||
|
||||
- name: Get current date
|
||||
if: always()
|
||||
@@ -325,12 +326,13 @@ jobs:
|
||||
name: logs_${{ matrix.python-version }}_${{ matrix.os }}_${{ matrix.split-group }}_${{ steps.date.outputs.date }}
|
||||
path: ./logs
|
||||
|
||||
- name: Upload Integration Test Coverage to Codecov
|
||||
- name: Upload Integration Test Coverage
|
||||
if: ${{ matrix.python-version == '3.11' }}
|
||||
uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # codecov/codecov-action@v5
|
||||
with:
|
||||
token: ${{ secrets.CODECOV_TOKEN }}
|
||||
flags: integration
|
||||
fail_ci_if_error: false
|
||||
|
||||
integration-report:
|
||||
if: ${{ always() }}
|
||||
@@ -361,12 +363,12 @@ jobs:
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # actions/setup-python@v6
|
||||
with:
|
||||
python-version: '3.10'
|
||||
python-version: "3.10"
|
||||
|
||||
- name: Install python dependencies
|
||||
run: |
|
||||
python -m pip install --user --upgrade pip
|
||||
python -m pip install --upgrade setuptools wheel twine check-wheel-contents
|
||||
python -m pip install --upgrade hatch twine check-wheel-contents
|
||||
python -m pip --version
|
||||
|
||||
- name: Build distributions
|
||||
@@ -377,25 +379,5 @@ jobs:
|
||||
|
||||
- name: Check distribution descriptions
|
||||
run: |
|
||||
twine check dist/*
|
||||
|
||||
- name: Check wheel contents
|
||||
run: |
|
||||
check-wheel-contents dist/*.whl --ignore W007,W008
|
||||
|
||||
- name: Install wheel distributions
|
||||
run: |
|
||||
find ./dist/*.whl -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/
|
||||
|
||||
- name: Check wheel distributions
|
||||
run: |
|
||||
dbt --version
|
||||
|
||||
- name: Install source distributions
|
||||
# ignore dbt-1.0.0, which intentionally raises an error when installed from source
|
||||
run: |
|
||||
find ./dist/*.gz -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/
|
||||
|
||||
- name: Check source distributions
|
||||
run: |
|
||||
dbt --version
|
||||
cd core
|
||||
hatch -v run build:check-all
|
||||
|
||||
2
.github/workflows/nightly-release.yml
vendored
2
.github/workflows/nightly-release.yml
vendored
@@ -46,7 +46,7 @@ jobs:
|
||||
- name: "Get Current Version Number"
|
||||
id: version-number-sources
|
||||
run: |
|
||||
current_version=`awk -F"current_version = " '{print $2}' .bumpversion.cfg | tr '\n' ' '`
|
||||
current_version=$(grep '^version = ' core/dbt/__version__.py | sed 's/version = "\(.*\)"/\1/')
|
||||
echo "current_version=$current_version" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: "Audit Version And Parse Into Parts"
|
||||
|
||||
22
.github/workflows/schema-check.yml
vendored
22
.github/workflows/schema-check.yml
vendored
@@ -22,7 +22,7 @@ on:
|
||||
target_branch:
|
||||
description: "The branch to check against"
|
||||
type: string
|
||||
default: 'main'
|
||||
default: "main"
|
||||
required: true
|
||||
|
||||
# no special access is needed
|
||||
@@ -48,8 +48,8 @@ jobs:
|
||||
- name: Checkout dbt repo
|
||||
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # actions/checkout@v4
|
||||
with:
|
||||
path: ${{ env.DBT_REPO_DIRECTORY }}
|
||||
ref: ${{ inputs.target_branch }}
|
||||
path: ${{ env.DBT_REPO_DIRECTORY }}
|
||||
ref: ${{ inputs.target_branch }}
|
||||
|
||||
- name: Check for changes in core/dbt/artifacts
|
||||
# https://github.com/marketplace/actions/paths-changes-filter
|
||||
@@ -72,18 +72,16 @@ jobs:
|
||||
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # actions/checkout@v4
|
||||
with:
|
||||
repository: dbt-labs/schemas.getdbt.com
|
||||
ref: 'main'
|
||||
ref: "main"
|
||||
path: ${{ env.SCHEMA_REPO_DIRECTORY }}
|
||||
|
||||
- name: Generate current schema
|
||||
if: steps.check_artifact_changes.outputs.artifacts_changed == 'true'
|
||||
run: |
|
||||
cd ${{ env.DBT_REPO_DIRECTORY }}
|
||||
python3 -m venv env
|
||||
source env/bin/activate
|
||||
pip install --upgrade pip
|
||||
pip install -r dev-requirements.txt -r editable-requirements.txt
|
||||
python scripts/collect-artifact-schema.py --path ${{ env.LATEST_SCHEMA_PATH }}
|
||||
cd ${{ env.DBT_REPO_DIRECTORY }}/core
|
||||
pip install --upgrade pip hatch
|
||||
hatch run setup
|
||||
hatch run json-schema -- --path ${{ env.LATEST_SCHEMA_PATH }}
|
||||
|
||||
# Copy generated schema files into the schemas.getdbt.com repo
|
||||
# Do a git diff to find any changes
|
||||
@@ -99,5 +97,5 @@ jobs:
|
||||
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # actions/upload-artifact@v4
|
||||
if: ${{ failure() && steps.check_artifact_changes.outputs.artifacts_changed == 'true' }}
|
||||
with:
|
||||
name: 'schema_changes.txt'
|
||||
path: '${{ env.SCHEMA_DIFF_ARTIFACT }}'
|
||||
name: "schema_changes.txt"
|
||||
path: "${{ env.SCHEMA_DIFF_ARTIFACT }}"
|
||||
|
||||
@@ -102,12 +102,12 @@ jobs:
|
||||
run: |
|
||||
pip install --user --upgrade pip
|
||||
pip --version
|
||||
pip install tox
|
||||
tox --version
|
||||
pip install hatch
|
||||
hatch --version
|
||||
|
||||
- name: Run postgres setup script
|
||||
run: |
|
||||
./test/setup_db.sh
|
||||
./scripts/setup_db.sh
|
||||
env:
|
||||
PGHOST: localhost
|
||||
PGPORT: 5432
|
||||
@@ -123,7 +123,7 @@ jobs:
|
||||
with:
|
||||
timeout_minutes: 30
|
||||
max_attempts: 3
|
||||
command: tox -e integration -- -nauto
|
||||
command: cd core && hatch -v run ci:integration-tests -- -nauto
|
||||
env:
|
||||
PYTEST_ADDOPTS: ${{ format('--splits {0} --group {1}', env.PYTHON_INTEGRATION_TEST_WORKERS, matrix.split-group) }}
|
||||
|
||||
|
||||
43
.github/workflows/test-repeater.yml
vendored
43
.github/workflows/test-repeater.yml
vendored
@@ -14,33 +14,33 @@ on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
branch:
|
||||
description: 'Branch to check out'
|
||||
description: "Branch to check out"
|
||||
type: string
|
||||
required: true
|
||||
default: 'main'
|
||||
default: "main"
|
||||
test_path:
|
||||
description: 'Path to single test to run (ex: tests/functional/retry/test_retry.py::TestRetry::test_fail_fast)'
|
||||
description: "Path to single test to run (ex: tests/functional/retry/test_retry.py::TestRetry::test_fail_fast)"
|
||||
type: string
|
||||
required: true
|
||||
default: 'tests/functional/...'
|
||||
default: "tests/functional/..."
|
||||
python_version:
|
||||
description: 'Version of Python to Test Against'
|
||||
description: "Version of Python to Test Against"
|
||||
type: choice
|
||||
options:
|
||||
- '3.10'
|
||||
- '3.11'
|
||||
- "3.10"
|
||||
- "3.11"
|
||||
os:
|
||||
description: 'OS to run test in'
|
||||
description: "OS to run test in"
|
||||
type: choice
|
||||
options:
|
||||
- 'ubuntu-latest'
|
||||
- 'macos-14'
|
||||
- 'windows-latest'
|
||||
- "ubuntu-latest"
|
||||
- "macos-14"
|
||||
- "windows-latest"
|
||||
num_runs_per_batch:
|
||||
description: 'Max number of times to run the test per batch. We always run 10 batches.'
|
||||
description: "Max number of times to run the test per batch. We always run 10 batches."
|
||||
type: number
|
||||
required: true
|
||||
default: '50'
|
||||
default: "50"
|
||||
|
||||
permissions: read-all
|
||||
|
||||
@@ -90,12 +90,19 @@ jobs:
|
||||
with:
|
||||
python-version: "${{ inputs.python_version }}"
|
||||
|
||||
- name: "Install hatch"
|
||||
run: python -m pip install --user --upgrade pip hatch
|
||||
|
||||
- name: "Setup Dev Environment"
|
||||
run: make dev
|
||||
run: |
|
||||
cd core
|
||||
hatch run setup
|
||||
|
||||
- name: "Set up postgres (linux)"
|
||||
if: inputs.os == '${{ vars.UBUNTU_LATEST }}'
|
||||
run: make setup-db
|
||||
run: |
|
||||
cd core
|
||||
hatch run setup-db
|
||||
|
||||
# mac and windows don't use make due to limitations with docker with those runners in GitHub
|
||||
- name: Set up postgres (macos)
|
||||
@@ -104,7 +111,7 @@ jobs:
|
||||
with:
|
||||
timeout_minutes: 10
|
||||
max_attempts: 3
|
||||
command: ./test/setup_db.sh
|
||||
command: ./scripts/setup_db.sh
|
||||
|
||||
- name: "Set up postgres (windows)"
|
||||
if: inputs.os == 'windows-latest'
|
||||
@@ -153,5 +160,5 @@ jobs:
|
||||
- name: "Error for Failures"
|
||||
if: ${{ steps.pytest.outputs.failure }}
|
||||
run: |
|
||||
echo "Batch ${{ matrix.batch }} failed ${{ steps.pytest.outputs.failure }} of ${{ inputs.num_runs_per_batch }} tests"
|
||||
exit 1
|
||||
echo "Batch ${{ matrix.batch }} failed ${{ steps.pytest.outputs.failure }} of ${{ inputs.num_runs_per_batch }} tests"
|
||||
exit 1
|
||||
|
||||
2
.gitignore
vendored
2
.gitignore
vendored
@@ -15,6 +15,7 @@ build/
|
||||
!core/dbt/docs/build
|
||||
develop-eggs/
|
||||
dist/
|
||||
dist-*/
|
||||
downloads/
|
||||
eggs/
|
||||
.eggs/
|
||||
@@ -95,6 +96,7 @@ target/
|
||||
# pycharm
|
||||
.idea/
|
||||
venv/
|
||||
.venv*/
|
||||
|
||||
# AWS credentials
|
||||
.aws/
|
||||
|
||||
106
CHANGELOG.md
106
CHANGELOG.md
@@ -5,6 +5,111 @@
|
||||
- "Breaking changes" listed under a version may require action from end users or external maintainers when upgrading to that version.
|
||||
- Do not edit this file directly. This file is auto-generated using [changie](https://github.com/miniscruff/changie). For details on how to document a change, see [the contributing guide](https://github.com/dbt-labs/dbt-core/blob/main/CONTRIBUTING.md#adding-changelog-entry)
|
||||
|
||||
## dbt-core 1.11.0-rc4 - December 17, 2025
|
||||
|
||||
### Features
|
||||
|
||||
- Implement config.meta_get and config.meta_require ([#12012](https://github.com/dbt-labs/dbt-core/issues/12012))
|
||||
|
||||
### Fixes
|
||||
|
||||
- :bug: :snowman: Fix ref resolution within package when duplicate nodes exist, behind require_ref_searches_node_package_before_root behavior change flag ([#11351](https://github.com/dbt-labs/dbt-core/issues/11351))
|
||||
- Improve error message clarity when detecting nodes with space in name ([#11835](https://github.com/dbt-labs/dbt-core/issues/11835))
|
||||
- :bug: :snowman:Propagate exceptions for NodeFinished callbacks in dbtRunner ([#11612](https://github.com/dbt-labs/dbt-core/issues/11612))
|
||||
- Adds omitted return statement to RuntimeConfigObject.meta_require method ([#12288](https://github.com/dbt-labs/dbt-core/issues/12288))
|
||||
- Do not raise deprecation warning when encountering dataset or project configs for bigquery ([#12285](https://github.com/dbt-labs/dbt-core/issues/12285))
|
||||
|
||||
### Under the Hood
|
||||
|
||||
- Bump lower bound for dbt-common to 1.37.2 ([#12284](https://github.com/dbt-labs/dbt-core/issues/12284))
|
||||
|
||||
### Contributors
|
||||
- [@gshank](https://github.com/gshank) ([#12012](https://github.com/dbt-labs/dbt-core/issues/12012))
|
||||
- [@michelleark](https://github.com/michelleark) ([#11351](https://github.com/dbt-labs/dbt-core/issues/11351), [#11835](https://github.com/dbt-labs/dbt-core/issues/11835), [#11612](https://github.com/dbt-labs/dbt-core/issues/11612), [#12285](https://github.com/dbt-labs/dbt-core/issues/12285), [#12284](https://github.com/dbt-labs/dbt-core/issues/12284))
|
||||
- [@mjsqu](https://github.com/mjsqu) ([#12288](https://github.com/dbt-labs/dbt-core/issues/12288))
|
||||
|
||||
|
||||
## dbt-core 1.11.0-rc3 - December 08, 2025
|
||||
|
||||
### Features
|
||||
|
||||
- Raise jsonschema-based deprecation warnings by default ([#12240](https://github.com/dbt-labs/dbt-core/issues/12240))
|
||||
- :bug: :snowman: Disable unit tests whose model is disabled ([#10540](https://github.com/dbt-labs/dbt-core/issues/10540))
|
||||
|
||||
### Fixes
|
||||
|
||||
- Fix bug in partial parsing when updating a model with a schema file that is referenced by a singular test ([#12223](https://github.com/dbt-labs/dbt-core/issues/12223))
|
||||
- :bug: :snowman: Improve `dbt deps --add-package` duplicate detection with better cross-source matching and word boundaries ([#12239](https://github.com/dbt-labs/dbt-core/issues/12239))
|
||||
- :bug: :snowman: Fix false positive deprecation warning of pre/post-hook SQL configs ([#12244](https://github.com/dbt-labs/dbt-core/issues/12244))
|
||||
|
||||
### Under the Hood
|
||||
|
||||
- Replace setuptools and tox with hatch for build, test, and environment management. ([#12151](https://github.com/dbt-labs/dbt-core/issues/12151))
|
||||
|
||||
### Contributors
|
||||
- [@emmyoop](https://github.com/emmyoop) ([#12239](https://github.com/dbt-labs/dbt-core/issues/12239), [#12151](https://github.com/dbt-labs/dbt-core/issues/12151))
|
||||
- [@mattogburke](https://github.com/mattogburke) ([#12223](https://github.com/dbt-labs/dbt-core/issues/12223))
|
||||
- [@michelleark](https://github.com/michelleark) ([#12240](https://github.com/dbt-labs/dbt-core/issues/12240), [#10540](https://github.com/dbt-labs/dbt-core/issues/10540), [#12244](https://github.com/dbt-labs/dbt-core/issues/12244))
|
||||
|
||||
## dbt-core 1.11.0-rc2 - December 01, 2025
|
||||
|
||||
### Features
|
||||
|
||||
- Support partial parsing for function nodes ([#12072](https://github.com/dbt-labs/dbt-core/issues/12072))
|
||||
|
||||
### Fixes
|
||||
|
||||
- Allow dbt deps to run when vars lack defaults in dbt_project.yml ([#8913](https://github.com/dbt-labs/dbt-core/issues/8913))
|
||||
- Restore DuplicateResourceNameError for intra-project node name duplication, behind behavior flag `require_unique_project_resource_names` ([#12152](https://github.com/dbt-labs/dbt-core/issues/12152))
|
||||
- Allow the usage of `function` with `--exclude-resource-type` flag ([#12143](https://github.com/dbt-labs/dbt-core/issues/12143))
|
||||
- Fix bug where schemas of functions weren't guaranteed to exist ([#12142](https://github.com/dbt-labs/dbt-core/issues/12142))
|
||||
- :bug: :snowman: Correctly reference foreign key references when --defer and --state provided ([#11885](https://github.com/dbt-labs/dbt-core/issues/11885))
|
||||
- Fix generation of deprecations summary ([#12146](https://github.com/dbt-labs/dbt-core/issues/12146))
|
||||
- :bug: :snowman: Add exception when using --state and referring to a removed test ([#10630](https://github.com/dbt-labs/dbt-core/issues/10630))
|
||||
- :bug: :snowman: Stop emitting `NoNodesForSelectionCriteria` three times during `build` command ([#11627](https://github.com/dbt-labs/dbt-core/issues/11627))
|
||||
- :bug: :snowman: Fix long Python stack traces appearing when package dependencies have incompatible version requirements ([#12049](https://github.com/dbt-labs/dbt-core/issues/12049))
|
||||
- :bug: :snowman: Fixed issue where changing data type size/precision/scale (e.g., varchar(3) to varchar(10)) incorrectly triggered a breaking change error fo ([#11186](https://github.com/dbt-labs/dbt-core/issues/11186))
|
||||
- :bug: :snowman: Support unit testing models that depend on sources with the same name ([#11975](https://github.com/dbt-labs/dbt-core/issues/11975), [#10433](https://github.com/dbt-labs/dbt-core/issues/10433))
|
||||
- :bug: :snowman: Avoid retrying successful run-operation commands ([#11850](https://github.com/dbt-labs/dbt-core/issues/11850))
|
||||
- :bug: :snowman: Fix `dbt deps --add-package` crash when packages.yml contains `warn-unpinned: false` ([#9104](https://github.com/dbt-labs/dbt-core/issues/9104))
|
||||
|
||||
### Under the Hood
|
||||
|
||||
- Update jsonschemas for schema.yml and dbt_project.yml deprecations ([#12180](https://github.com/dbt-labs/dbt-core/issues/12180))
|
||||
|
||||
### Contributors
|
||||
- [@asiunov](https://github.com/asiunov) ([#12146](https://github.com/dbt-labs/dbt-core/issues/12146))
|
||||
- [@michellark](https://github.com/michellark) ([#11885](https://github.com/dbt-labs/dbt-core/issues/11885))
|
||||
|
||||
## dbt-core 1.11.0-rc1 - November 18, 2025
|
||||
|
||||
### Features
|
||||
|
||||
- Allow for defining funciton arguments with default values ([#12044](https://github.com/dbt-labs/dbt-core/issues/12044))
|
||||
|
||||
### Fixes
|
||||
|
||||
- Fix parse error when build_after.count set to 0 ([#12136](https://github.com/dbt-labs/dbt-core/issues/12136))
|
||||
- Stop compiling python udfs like python models ([#12153](https://github.com/dbt-labs/dbt-core/issues/12153))
|
||||
- For metric names, fix bug allowing hyphens (not allowed in metricflow already), make validation throw ValidationErrors (not ParsingErrors), and add tests. ([#n/a](https://github.com/dbt-labs/dbt-core/issues/n/a))
|
||||
- Include macros in unit test parsing ([#10157](https://github.com/dbt-labs/dbt-core/issues/10157))
|
||||
|
||||
### Under the Hood
|
||||
|
||||
- add dbt/jsonschemas to manifest.in ([#12126](https://github.com/dbt-labs/dbt-core/issues/12126))
|
||||
- Move from setup.py to pyproject.toml ([#5696](https://github.com/dbt-labs/dbt-core/issues/5696))
|
||||
- Fixes issue where config isn't propagated to metric from measure when set as create_metric:True ([#None](https://github.com/dbt-labs/dbt-core/issues/None))
|
||||
- Support DBT_ENGINE prefix for record-mode env vars ([#12149](https://github.com/dbt-labs/dbt-core/issues/12149))
|
||||
|
||||
### Dependencies
|
||||
|
||||
- Drop support for python 3.9 ([#12118](https://github.com/dbt-labs/dbt-core/issues/12118))
|
||||
|
||||
### Contributors
|
||||
- [@WilliamDee](https://github.com/WilliamDee) ([#None](https://github.com/dbt-labs/dbt-core/issues/None))
|
||||
- [@nathanskone](https://github.com/nathanskone) ([#10157](https://github.com/dbt-labs/dbt-core/issues/10157))
|
||||
- [@theyostalservice](https://github.com/theyostalservice) ([#n/a](https://github.com/dbt-labs/dbt-core/issues/n/a))
|
||||
|
||||
## dbt-core 1.11.0-b4 - October 28, 2025
|
||||
|
||||
### Features
|
||||
@@ -29,7 +134,6 @@
|
||||
- [@12030](https://github.com/12030) ([#QMalcolm](https://github.com/dbt-labs/dbt-core/issues/QMalcolm))
|
||||
- [@WilliamDee](https://github.com/WilliamDee) ([#None](https://github.com/dbt-labs/dbt-core/issues/None))
|
||||
|
||||
|
||||
## dbt-core 1.11.0-b3 - October 07, 2025
|
||||
|
||||
### Features
|
||||
|
||||
122
CONTRIBUTING.md
122
CONTRIBUTING.md
@@ -20,9 +20,8 @@
|
||||
- [Testing](#testing)
|
||||
- [Initial setup](#initial-setup)
|
||||
- [Test commands](#test-commands)
|
||||
- [Makefile](#makefile)
|
||||
- [Hatch scripts](#hatch-scripts)
|
||||
- [`pre-commit`](#pre-commit)
|
||||
- [`tox`](#tox)
|
||||
- [`pytest`](#pytest)
|
||||
- [Unit, Integration, Functional?](#unit-integration-functional)
|
||||
- [Debugging](#debugging)
|
||||
@@ -35,7 +34,7 @@
|
||||
|
||||
There are many ways to contribute to the ongoing development of `dbt-core`, such as by participating in discussions and issues. We encourage you to first read our higher-level document: ["Expectations for Open Source Contributors"](https://docs.getdbt.com/docs/contributing/oss-expectations).
|
||||
|
||||
The rest of this document serves as a more granular guide for contributing code changes to `dbt-core` (this repository). It is not intended as a guide for using `dbt-core`, and some pieces assume a level of familiarity with Python development (virtualenvs, `pip`, etc). Specific code snippets in this guide assume you are using macOS or Linux and are comfortable with the command line.
|
||||
The rest of this document serves as a more granular guide for contributing code changes to `dbt-core` (this repository). It is not intended as a guide for using `dbt-core`, and some pieces assume a level of familiarity with Python development and package managers. Specific code snippets in this guide assume you are using macOS or Linux and are comfortable with the command line.
|
||||
|
||||
If you get stuck, we're happy to help! Drop us a line in the `#dbt-core-development` channel in the [dbt Community Slack](https://community.getdbt.com).
|
||||
|
||||
@@ -74,28 +73,22 @@ There are some tools that will be helpful to you in developing locally. While th
|
||||
|
||||
These are the tools used in `dbt-core` development and testing:
|
||||
|
||||
- [`tox`](https://tox.readthedocs.io/en/latest/) to manage virtualenvs across python versions. We currently target the latest patch releases for Python 3.10, 3.11, 3.12, and 3.13
|
||||
- [`hatch`](https://hatch.pypa.io/) for build backend, environment management, and running tests across Python versions (3.10, 3.11, 3.12, and 3.13)
|
||||
- [`pytest`](https://docs.pytest.org/en/latest/) to define, discover, and run tests
|
||||
- [`flake8`](https://flake8.pycqa.org/en/latest/) for code linting
|
||||
- [`black`](https://github.com/psf/black) for code formatting
|
||||
- [`mypy`](https://mypy.readthedocs.io/en/stable/) for static type checking
|
||||
- [`pre-commit`](https://pre-commit.com) to easily run those checks
|
||||
- [`changie`](https://changie.dev/) to create changelog entries, without merge conflicts
|
||||
- [`make`](https://users.cs.duke.edu/~ola/courses/programming/Makefiles/Makefiles.html) to run multiple setup or test steps in combination. Don't worry too much, nobody _really_ understands how `make` works, and our Makefile aims to be super simple.
|
||||
- [GitHub Actions](https://github.com/features/actions) for automating tests and checks, once a PR is pushed to the `dbt-core` repository
|
||||
|
||||
A deep understanding of these tools in not required to effectively contribute to `dbt-core`, but we recommend checking out the attached documentation if you're interested in learning more about each one.
|
||||
|
||||
#### Virtual environments
|
||||
|
||||
We strongly recommend using virtual environments when developing code in `dbt-core`. We recommend creating this virtualenv
|
||||
in the root of the `dbt-core` repository. To create a new virtualenv, run:
|
||||
```sh
|
||||
python3 -m venv env
|
||||
source env/bin/activate
|
||||
```
|
||||
dbt-core uses [Hatch](https://hatch.pypa.io/) for dependency and environment management. Hatch automatically creates and manages isolated environments for development, testing, and building, so you don't need to manually create virtual environments.
|
||||
|
||||
This will create and activate a new Python virtual environment.
|
||||
For more information on how Hatch manages environments, see the [Hatch environment documentation](https://hatch.pypa.io/latest/environment/).
|
||||
|
||||
#### Docker and `docker-compose`
|
||||
|
||||
@@ -114,22 +107,42 @@ brew install postgresql
|
||||
|
||||
### Installation
|
||||
|
||||
First make sure that you set up your `virtualenv` as described in [Setting up an environment](#setting-up-an-environment). Also ensure you have the latest version of pip installed with `pip install --upgrade pip`. Next, install `dbt-core` (and its dependencies):
|
||||
First make sure you have Python 3.10 or later installed. Ensure you have the latest version of pip installed with `pip install --upgrade pip`. Next, install `hatch`. Finally set up `dbt-core` for development:
|
||||
|
||||
```sh
|
||||
make dev
|
||||
cd core
|
||||
hatch run setup
|
||||
```
|
||||
or, alternatively:
|
||||
|
||||
This will install all development dependencies and set up pre-commit hooks.
|
||||
|
||||
By default, hatch will use whatever Python version is active in your environment. To specify a particular Python version, set the `HATCH_PYTHON` environment variable:
|
||||
|
||||
```sh
|
||||
pip install -r dev-requirements.txt -r editable-requirements.txt
|
||||
pre-commit install
|
||||
export HATCH_PYTHON=3.12
|
||||
hatch env create
|
||||
```
|
||||
|
||||
Or add it to your shell profile (e.g., `~/.zshrc` or `~/.bashrc`) for persistence.
|
||||
|
||||
When installed in this way, any changes you make to your local copy of the source code will be reflected immediately in your next `dbt` run.
|
||||
|
||||
#### Building dbt-core
|
||||
|
||||
dbt-core uses [Hatch](https://hatch.pypa.io/) (specifically `hatchling`) as its build backend. To build distribution packages:
|
||||
|
||||
```sh
|
||||
cd core
|
||||
hatch build
|
||||
```
|
||||
|
||||
This will create both wheel (`.whl`) and source distribution (`.tar.gz`) files in the `dist/` directory.
|
||||
|
||||
The build configuration is defined in `core/pyproject.toml`. You can also use the standard `python -m build` command if you prefer.
|
||||
|
||||
### Running `dbt-core`
|
||||
|
||||
With your virtualenv activated, the `dbt` script should point back to the source code you've cloned on your machine. You can verify this by running `which dbt`. This command should show you a path to an executable in your virtualenv.
|
||||
Once you've run `hatch run setup`, the `dbt` command will be available in your PATH. You can verify this by running `which dbt`.
|
||||
|
||||
Configure your [profile](https://docs.getdbt.com/docs/configure-your-profile) as necessary to connect to your target databases. It may be a good idea to add a new profile pointing to a local Postgres instance, or a specific test sandbox within your data warehouse if appropriate. Make sure to create a profile before running integration tests.
|
||||
|
||||
@@ -147,45 +160,78 @@ Although `dbt-core` works with a number of different databases, you won't need t
|
||||
Postgres offers the easiest way to test most `dbt-core` functionality today. They are the fastest to run, and the easiest to set up. To run the Postgres integration tests, you'll have to do one extra step of setting up the test database:
|
||||
|
||||
```sh
|
||||
make setup-db
|
||||
cd core
|
||||
hatch run setup-db
|
||||
```
|
||||
or, alternatively:
|
||||
|
||||
Alternatively, you can run the setup commands directly:
|
||||
|
||||
```sh
|
||||
docker-compose up -d database
|
||||
PGHOST=localhost PGUSER=root PGPASSWORD=password PGDATABASE=postgres bash test/setup_db.sh
|
||||
PGHOST=localhost PGUSER=root PGPASSWORD=password PGDATABASE=postgres bash scripts/setup_db.sh
|
||||
```
|
||||
|
||||
### Test commands
|
||||
|
||||
There are a few methods for running tests locally.
|
||||
|
||||
#### Makefile
|
||||
#### Hatch scripts
|
||||
|
||||
There are multiple targets in the Makefile to run common test suites and code
|
||||
checks, most notably:
|
||||
The primary way to run tests and checks is using hatch scripts (defined in `core/hatch.toml`):
|
||||
|
||||
```sh
|
||||
# Runs unit tests with py38 and code checks in parallel.
|
||||
make test
|
||||
# Runs postgres integration tests with py38 in "fail fast" mode.
|
||||
make integration
|
||||
```
|
||||
> These make targets assume you have a local installation of a recent version of [`tox`](https://tox.readthedocs.io/en/latest/) for unit/integration testing and pre-commit for code quality checks,
|
||||
> unless you use choose a Docker container to run tests. Run `make help` for more info.
|
||||
cd core
|
||||
|
||||
Check out the other targets in the Makefile to see other commonly used test
|
||||
suites.
|
||||
# Run all unit tests
|
||||
hatch run unit-tests
|
||||
|
||||
# Run unit tests and all code quality checks
|
||||
hatch run test
|
||||
|
||||
# Run integration tests
|
||||
hatch run integration-tests
|
||||
|
||||
# Run integration tests in fail-fast mode
|
||||
hatch run integration-tests-fail-fast
|
||||
|
||||
# Run linting checks only
|
||||
hatch run lint
|
||||
hatch run flake8
|
||||
hatch run mypy
|
||||
hatch run black
|
||||
|
||||
# Run all pre-commit hooks
|
||||
hatch run code-quality
|
||||
|
||||
# Clean build artifacts
|
||||
hatch run clean
|
||||
```
|
||||
|
||||
Hatch manages isolated environments and dependencies automatically. The commands above use the `default` environment which is recommended for most local development.
|
||||
|
||||
**Using the `ci` environment (optional)**
|
||||
|
||||
If you need to replicate exactly what runs in GitHub Actions (e.g., with coverage reporting), use the `ci` environment:
|
||||
|
||||
```sh
|
||||
cd core
|
||||
|
||||
# Run unit tests with coverage
|
||||
hatch run ci:unit-tests
|
||||
|
||||
# Run unit tests with a specific Python version
|
||||
hatch run +py=3.11 ci:unit-tests
|
||||
```
|
||||
|
||||
> **Note:** Most developers should use the default environment (`hatch run unit-tests`). The `ci` environment is primarily for debugging CI failures or running tests with coverage.
|
||||
|
||||
#### `pre-commit`
|
||||
[`pre-commit`](https://pre-commit.com) takes care of running all code-checks for formatting and linting. Run `make dev` to install `pre-commit` in your local environment (we recommend running this command with a python virtual environment active). This command installs several pip executables including black, mypy, and flake8. Once this is done you can use any of the linter-based make targets as well as a git pre-commit hook that will ensure proper formatting and linting.
|
||||
|
||||
#### `tox`
|
||||
|
||||
[`tox`](https://tox.readthedocs.io/en/latest/) takes care of managing virtualenvs and install dependencies in order to run tests. You can also run tests in parallel, for example, you can run unit tests for Python 3.8, Python 3.9, Python 3.10 and Python 3.11 checks in parallel with `tox -p`. Also, you can run unit tests for specific python versions with `tox -e py38`. The configuration for these tests in located in `tox.ini`.
|
||||
[`pre-commit`](https://pre-commit.com) takes care of running all code-checks for formatting and linting. Run `hatch run setup` to install `pre-commit` in your local environment (we recommend running this command with a python virtual environment active). This installs several pip executables including black, mypy, and flake8. Once installed, hooks will run automatically on `git commit`, or you can run them manually with `hatch run code-quality`.
|
||||
|
||||
#### `pytest`
|
||||
|
||||
Finally, you can also run a specific test or group of tests using [`pytest`](https://docs.pytest.org/en/latest/) directly. With a virtualenv active and dev dependencies installed you can do things like:
|
||||
Finally, you can also run a specific test or group of tests using [`pytest`](https://docs.pytest.org/en/latest/) directly. After running `hatch run setup`, you can run pytest commands like:
|
||||
|
||||
```sh
|
||||
# run all unit tests in a file
|
||||
|
||||
@@ -47,7 +47,7 @@ RUN curl -LO https://github.com/jwilder/dockerize/releases/download/$DOCKERIZE_V
|
||||
&& tar -C /usr/local/bin -xzvf dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz \
|
||||
&& rm dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz
|
||||
|
||||
RUN pip3 install -U tox wheel six setuptools pre-commit
|
||||
RUN pip3 install -U hatch wheel pre-commit
|
||||
|
||||
# These args are passed in via docker-compose, which reads then from the .env file.
|
||||
# On Linux, run `make .env` to create the .env file for the current user.
|
||||
@@ -62,7 +62,6 @@ RUN if [ ${USER_ID:-0} -ne 0 ] && [ ${GROUP_ID:-0} -ne 0 ]; then \
|
||||
useradd -mU -l dbt_test_user; \
|
||||
fi
|
||||
RUN mkdir /usr/app && chown dbt_test_user /usr/app
|
||||
RUN mkdir /home/tox && chown dbt_test_user /home/tox
|
||||
|
||||
WORKDIR /usr/app
|
||||
VOLUME /usr/app
|
||||
|
||||
163
Makefile
163
Makefile
@@ -1,146 +1,95 @@
|
||||
# ============================================================================
|
||||
# DEPRECATED: This Makefile is maintained for backwards compatibility only.
|
||||
#
|
||||
# dbt-core now uses Hatch for task management and development workflows.
|
||||
# Please migrate to using hatch commands directly:
|
||||
#
|
||||
# make dev → cd core && hatch run setup
|
||||
# make unit → cd core && hatch run unit-tests
|
||||
# make test → cd core && hatch run test
|
||||
# make integration → cd core && hatch run integration-tests
|
||||
# make lint → cd core && hatch run lint
|
||||
# make code_quality → cd core && hatch run code-quality
|
||||
# make setup-db → cd core && hatch run setup-db
|
||||
# make clean → cd core && hatch run clean
|
||||
#
|
||||
# See core/pyproject.toml [tool.hatch.envs.default.scripts] for all available
|
||||
# commands and CONTRIBUTING.md for detailed usage instructions.
|
||||
#
|
||||
# This Makefile will be removed in a future version of dbt-core.
|
||||
# ============================================================================
|
||||
|
||||
.DEFAULT_GOAL:=help
|
||||
|
||||
# Optional flag to run target in a docker container.
|
||||
# (example `make test USE_DOCKER=true`)
|
||||
ifeq ($(USE_DOCKER),true)
|
||||
DOCKER_CMD := docker-compose run --rm test
|
||||
endif
|
||||
|
||||
#
|
||||
# To override CI_flags, create a file at this repo's root dir named `makefile.test.env`. Fill it
|
||||
# with any ENV_VAR overrides required by your test environment, e.g.
|
||||
# DBT_TEST_USER_1=user
|
||||
# LOG_DIR="dir with a space in it"
|
||||
#
|
||||
# Warn: Restrict each line to one variable only.
|
||||
#
|
||||
ifeq (./makefile.test.env,$(wildcard ./makefile.test.env))
|
||||
include ./makefile.test.env
|
||||
endif
|
||||
|
||||
CI_FLAGS =\
|
||||
DBT_TEST_USER_1=$(if $(DBT_TEST_USER_1),$(DBT_TEST_USER_1),dbt_test_user_1)\
|
||||
DBT_TEST_USER_2=$(if $(DBT_TEST_USER_2),$(DBT_TEST_USER_2),dbt_test_user_2)\
|
||||
DBT_TEST_USER_3=$(if $(DBT_TEST_USER_3),$(DBT_TEST_USER_3),dbt_test_user_3)\
|
||||
RUSTFLAGS=$(if $(RUSTFLAGS),$(RUSTFLAGS),"-D warnings")\
|
||||
LOG_DIR=$(if $(LOG_DIR),$(LOG_DIR),./logs)\
|
||||
DBT_LOG_FORMAT=$(if $(DBT_LOG_FORMAT),$(DBT_LOG_FORMAT),json)
|
||||
|
||||
|
||||
.PHONY: dev_req
|
||||
dev_req: ## Installs dbt-* packages in develop mode along with only development dependencies.
|
||||
@\
|
||||
pip install -r dev-requirements.txt -r editable-requirements.txt
|
||||
@cd core && hatch run dev-req
|
||||
|
||||
.PHONY: dev
|
||||
dev: dev_req ## Installs dbt-* packages in develop mode along with development dependencies and pre-commit.
|
||||
@\
|
||||
$(DOCKER_CMD) pre-commit install
|
||||
dev: ## Installs dbt-* packages in develop mode along with development dependencies and pre-commit.
|
||||
@cd core && hatch run setup
|
||||
|
||||
.PHONY: dev-uninstall
|
||||
dev-uninstall: ## Uninstall all packages in venv except for build tools
|
||||
@\
|
||||
pip freeze | grep -v "^-e" | cut -d "@" -f1 | xargs pip uninstall -y; \
|
||||
pip uninstall -y dbt-core
|
||||
@pip freeze | grep -v "^-e" | cut -d "@" -f1 | xargs pip uninstall -y; \
|
||||
pip uninstall -y dbt-core
|
||||
|
||||
.PHONY: mypy
|
||||
mypy: .env ## Runs mypy against staged changes for static type checking.
|
||||
@\
|
||||
$(DOCKER_CMD) pre-commit run --hook-stage manual mypy-check | grep -v "INFO"
|
||||
mypy: ## Runs mypy against staged changes for static type checking.
|
||||
@cd core && hatch run mypy
|
||||
|
||||
.PHONY: flake8
|
||||
flake8: .env ## Runs flake8 against staged changes to enforce style guide.
|
||||
@\
|
||||
$(DOCKER_CMD) pre-commit run --hook-stage manual flake8-check | grep -v "INFO"
|
||||
flake8: ## Runs flake8 against staged changes to enforce style guide.
|
||||
@cd core && hatch run flake8
|
||||
|
||||
.PHONY: black
|
||||
black: .env ## Runs black against staged changes to enforce style guide.
|
||||
@\
|
||||
$(DOCKER_CMD) pre-commit run --hook-stage manual black-check -v | grep -v "INFO"
|
||||
black: ## Runs black against staged changes to enforce style guide.
|
||||
@cd core && hatch run black
|
||||
|
||||
.PHONY: lint
|
||||
lint: .env ## Runs flake8 and mypy code checks against staged changes.
|
||||
@\
|
||||
$(DOCKER_CMD) pre-commit run flake8-check --hook-stage manual | grep -v "INFO"; \
|
||||
$(DOCKER_CMD) pre-commit run mypy-check --hook-stage manual | grep -v "INFO"
|
||||
lint: ## Runs flake8 and mypy code checks against staged changes.
|
||||
@cd core && hatch run lint
|
||||
|
||||
.PHONY: code_quality
|
||||
code_quality: ## Runs all pre-commit hooks against all files.
|
||||
@cd core && hatch run code-quality
|
||||
|
||||
.PHONY: unit
|
||||
unit: .env ## Runs unit tests with py
|
||||
@\
|
||||
$(DOCKER_CMD) tox -e py
|
||||
unit: ## Runs unit tests with py
|
||||
@cd core && hatch run unit-tests
|
||||
|
||||
.PHONY: test
|
||||
test: .env ## Runs unit tests with py and code checks against staged changes.
|
||||
@\
|
||||
$(DOCKER_CMD) tox -e py; \
|
||||
$(DOCKER_CMD) pre-commit run black-check --hook-stage manual | grep -v "INFO"; \
|
||||
$(DOCKER_CMD) pre-commit run flake8-check --hook-stage manual | grep -v "INFO"; \
|
||||
$(DOCKER_CMD) pre-commit run mypy-check --hook-stage manual | grep -v "INFO"
|
||||
test: ## Runs unit tests with py and code checks against staged changes.
|
||||
@cd core && hatch run test
|
||||
|
||||
.PHONY: integration
|
||||
integration: .env ## Runs core integration tests using postgres with py-integration
|
||||
@\
|
||||
$(CI_FLAGS) $(DOCKER_CMD) tox -e py-integration -- -nauto
|
||||
integration: ## Runs core integration tests using postgres with py-integration
|
||||
@cd core && hatch run integration-tests
|
||||
|
||||
.PHONY: integration-fail-fast
|
||||
integration-fail-fast: .env ## Runs core integration tests using postgres with py-integration in "fail fast" mode.
|
||||
@\
|
||||
$(DOCKER_CMD) tox -e py-integration -- -x -nauto
|
||||
|
||||
.PHONY: interop
|
||||
interop: clean
|
||||
@\
|
||||
mkdir $(LOG_DIR) && \
|
||||
$(CI_FLAGS) $(DOCKER_CMD) tox -e py-integration -- -nauto && \
|
||||
LOG_DIR=$(LOG_DIR) cargo run --manifest-path test/interop/log_parsing/Cargo.toml
|
||||
integration-fail-fast: ## Runs core integration tests using postgres with py-integration in "fail fast" mode.
|
||||
@cd core && hatch run integration-tests-fail-fast
|
||||
|
||||
.PHONY: setup-db
|
||||
setup-db: ## Setup Postgres database with docker-compose for system testing.
|
||||
@\
|
||||
docker compose up -d database && \
|
||||
PGHOST=localhost PGUSER=root PGPASSWORD=password PGDATABASE=postgres SKIP_HOMEBREW=true bash test/setup_db.sh
|
||||
|
||||
# This rule creates a file named .env that is used by docker-compose for passing
|
||||
# the USER_ID and GROUP_ID arguments to the Docker image.
|
||||
.env: ## Setup step for using using docker-compose with make target.
|
||||
@touch .env
|
||||
ifneq ($(OS),Windows_NT)
|
||||
ifneq ($(shell uname -s), Darwin)
|
||||
@echo USER_ID=$(shell id -u) > .env
|
||||
@echo GROUP_ID=$(shell id -g) >> .env
|
||||
endif
|
||||
endif
|
||||
@cd core && hatch run setup-db
|
||||
|
||||
.PHONY: clean
|
||||
clean: ## Resets development environment.
|
||||
@echo 'cleaning repo...'
|
||||
@rm -f .coverage
|
||||
@rm -f .coverage.*
|
||||
@rm -rf .eggs/
|
||||
@rm -f .env
|
||||
@rm -rf .tox/
|
||||
@rm -rf build/
|
||||
@rm -rf dbt.egg-info/
|
||||
@rm -f dbt_project.yml
|
||||
@rm -rf dist/
|
||||
@rm -f htmlcov/*.{css,html,js,json,png}
|
||||
@rm -rf logs/
|
||||
@rm -rf target/
|
||||
@find . -type f -name '*.pyc' -delete
|
||||
@find . -type d -name '__pycache__' -depth -delete
|
||||
@echo 'done.'
|
||||
@cd core && hatch run clean
|
||||
|
||||
.PHONY: json_schema
|
||||
json_schema: ## Update generated JSON schema using code changes.
|
||||
@cd core && hatch run json-schema
|
||||
|
||||
.PHONY: help
|
||||
help: ## Show this help message.
|
||||
@echo 'usage: make [target] [USE_DOCKER=true]'
|
||||
@echo 'usage: make [target]'
|
||||
@echo
|
||||
@echo 'DEPRECATED: This Makefile is a compatibility shim.'
|
||||
@echo 'Please use "cd core && hatch run <command>" directly.'
|
||||
@echo
|
||||
@echo 'targets:'
|
||||
@grep -E '^[8+a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
|
||||
@echo
|
||||
@echo 'options:'
|
||||
@echo 'use USE_DOCKER=true to run target in a docker container'
|
||||
|
||||
.PHONY: json_schema
|
||||
json_schema: ## Update generated JSON schema using code changes.
|
||||
scripts/collect-artifact-schema.py --path schemas
|
||||
@echo 'For more information, see CONTRIBUTING.md'
|
||||
|
||||
27
codecov.yml
27
codecov.yml
@@ -2,39 +2,22 @@ ignore:
|
||||
- ".github"
|
||||
- ".changes"
|
||||
|
||||
# Disable all status checks to prevent red X's in CI
|
||||
# Coverage data is still uploaded and PR comments are still posted
|
||||
coverage:
|
||||
status:
|
||||
project:
|
||||
default:
|
||||
target: auto
|
||||
threshold: 0.1% # Reduce noise by ignoring rounding errors in coverage drops
|
||||
informational: true
|
||||
patch:
|
||||
default:
|
||||
target: auto
|
||||
threshold: 80%
|
||||
informational: true
|
||||
project: off
|
||||
patch: off
|
||||
|
||||
comment:
|
||||
layout: "header, diff, flags, components" # show component info in the PR comment
|
||||
layout: "header, diff, flags, components" # show component info in the PR comment
|
||||
|
||||
component_management:
|
||||
default_rules: # default rules that will be inherited by all components
|
||||
statuses:
|
||||
- type: project # in this case every component that doens't have a status defined will have a project type one
|
||||
target: auto
|
||||
threshold: 0.1%
|
||||
- type: patch
|
||||
target: 80%
|
||||
individual_components:
|
||||
- component_id: unittests
|
||||
name: "Unit Tests"
|
||||
flag_regexes:
|
||||
- "unit"
|
||||
statuses:
|
||||
- type: patch
|
||||
target: 80%
|
||||
threshold: 5%
|
||||
- component_id: integrationtests
|
||||
name: "Integration Tests"
|
||||
flag_regexes:
|
||||
|
||||
@@ -1,4 +0,0 @@
|
||||
recursive-include dbt/include *.py *.sql *.yml *.html *.md .gitkeep .gitignore
|
||||
include dbt/py.typed
|
||||
recursive-include dbt/task/docs *.html
|
||||
recursive-include dbt/jsonschemas *.json
|
||||
1
core/dbt/__version__.py
Normal file
1
core/dbt/__version__.py
Normal file
@@ -0,0 +1 @@
|
||||
version = "1.11.0rc4"
|
||||
@@ -1,5 +1,5 @@
|
||||
from dataclasses import dataclass, field
|
||||
from typing import List, Literal, Optional
|
||||
from typing import Any, List, Literal, Optional
|
||||
|
||||
from dbt.artifacts.resources.types import FunctionType, FunctionVolatility, NodeType
|
||||
from dbt.artifacts.resources.v1.components import CompiledResource
|
||||
@@ -32,6 +32,7 @@ class FunctionArgument(dbtClassMixin):
|
||||
name: str
|
||||
data_type: str
|
||||
description: Optional[str] = None
|
||||
default_value: Optional[Any] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
|
||||
@@ -235,6 +235,7 @@ exclude_resource_type = _create_option_and_track_env_var(
|
||||
"exposure",
|
||||
"snapshot",
|
||||
"seed",
|
||||
"function",
|
||||
"default",
|
||||
],
|
||||
case_sensitive=False,
|
||||
|
||||
@@ -291,8 +291,22 @@ def project(func):
|
||||
flags = ctx.obj["flags"]
|
||||
# TODO deprecations warnings fired from loading the project will lack
|
||||
# the project_id in the snowplow event.
|
||||
|
||||
# Determine if vars should be required during project loading.
|
||||
# Commands that don't need vars evaluated (like 'deps', 'clean')
|
||||
# should use lenient mode (require_vars=False) to allow missing vars.
|
||||
# Commands that validate or execute (like 'run', 'compile', 'build', 'debug') should use
|
||||
# strict mode (require_vars=True) to show helpful "Required var X not found" errors.
|
||||
# If adding more commands to lenient mode, update this condition.
|
||||
require_vars = flags.WHICH != "deps"
|
||||
|
||||
project = load_project(
|
||||
flags.PROJECT_DIR, flags.VERSION_CHECK, ctx.obj["profile"], flags.VARS, validate=True
|
||||
flags.PROJECT_DIR,
|
||||
flags.VERSION_CHECK,
|
||||
ctx.obj["profile"],
|
||||
flags.VARS,
|
||||
validate=True,
|
||||
require_vars=require_vars,
|
||||
)
|
||||
ctx.obj["project"] = project
|
||||
|
||||
|
||||
@@ -26,6 +26,7 @@ from dbt.contracts.graph.nodes import (
|
||||
SeedNode,
|
||||
UnitTestDefinition,
|
||||
UnitTestNode,
|
||||
UnitTestSourceDefinition,
|
||||
)
|
||||
from dbt.events.types import FoundStats, WritingInjectedSQLForNode
|
||||
from dbt.exceptions import (
|
||||
@@ -566,7 +567,12 @@ class Compiler:
|
||||
|
||||
_extend_prepended_ctes(prepended_ctes, new_prepended_ctes)
|
||||
|
||||
new_cte_name = self.add_ephemeral_prefix(cte_model.identifier)
|
||||
cte_name = (
|
||||
cte_model.cte_name
|
||||
if isinstance(cte_model, UnitTestSourceDefinition)
|
||||
else cte_model.identifier
|
||||
)
|
||||
new_cte_name = self.add_ephemeral_prefix(cte_name)
|
||||
rendered_sql = cte_model._pre_injected_sql or cte_model.compiled_code
|
||||
sql = f" {new_cte_name} as (\n{rendered_sql}\n)"
|
||||
|
||||
@@ -654,8 +660,15 @@ class Compiler:
|
||||
raise GraphDependencyNotFoundError(node, to_expression)
|
||||
|
||||
adapter = get_adapter(self.config)
|
||||
relation_name = str(adapter.Relation.create_from(self.config, foreign_key_node))
|
||||
return relation_name
|
||||
|
||||
if (
|
||||
hasattr(foreign_key_node, "defer_relation")
|
||||
and foreign_key_node.defer_relation
|
||||
and self.config.args.defer
|
||||
):
|
||||
return str(adapter.Relation.create_from(self.config, foreign_key_node.defer_relation))
|
||||
else:
|
||||
return str(adapter.Relation.create_from(self.config, foreign_key_node))
|
||||
|
||||
# This method doesn't actually "compile" any of the nodes. That is done by the
|
||||
# "compile_node" method. This creates a Linker and builds the networkx graph,
|
||||
|
||||
@@ -101,7 +101,10 @@ class DbtProjectYamlRenderer(BaseRenderer):
|
||||
_KEYPATH_HANDLERS = ProjectPostprocessor()
|
||||
|
||||
def __init__(
|
||||
self, profile: Optional[HasCredentials] = None, cli_vars: Optional[Dict[str, Any]] = None
|
||||
self,
|
||||
profile: Optional[HasCredentials] = None,
|
||||
cli_vars: Optional[Dict[str, Any]] = None,
|
||||
require_vars: bool = True,
|
||||
) -> None:
|
||||
# Generate contexts here because we want to save the context
|
||||
# object in order to retrieve the env_vars. This is almost always
|
||||
@@ -109,10 +112,19 @@ class DbtProjectYamlRenderer(BaseRenderer):
|
||||
# even when we don't have a profile.
|
||||
if cli_vars is None:
|
||||
cli_vars = {}
|
||||
# Store profile and cli_vars for creating strict context later
|
||||
self.profile = profile
|
||||
self.cli_vars = cli_vars
|
||||
|
||||
# By default, require vars (strict mode) for proper error messages.
|
||||
# Commands that don't need vars (like 'deps') should explicitly pass
|
||||
# require_vars=False for lenient loading.
|
||||
if profile:
|
||||
self.ctx_obj = TargetContext(profile.to_target_dict(), cli_vars)
|
||||
self.ctx_obj = TargetContext(
|
||||
profile.to_target_dict(), cli_vars, require_vars=require_vars
|
||||
)
|
||||
else:
|
||||
self.ctx_obj = BaseContext(cli_vars) # type:ignore
|
||||
self.ctx_obj = BaseContext(cli_vars, require_vars=require_vars) # type:ignore
|
||||
context = self.ctx_obj.to_dict()
|
||||
super().__init__(context)
|
||||
|
||||
|
||||
@@ -52,9 +52,10 @@ def load_project(
|
||||
profile: HasCredentials,
|
||||
cli_vars: Optional[Dict[str, Any]] = None,
|
||||
validate: bool = False,
|
||||
require_vars: bool = True,
|
||||
) -> Project:
|
||||
# get the project with all of the provided information
|
||||
project_renderer = DbtProjectYamlRenderer(profile, cli_vars)
|
||||
project_renderer = DbtProjectYamlRenderer(profile, cli_vars, require_vars=require_vars)
|
||||
project = Project.from_project_root(
|
||||
project_root, project_renderer, verify_version=version_check, validate=validate
|
||||
)
|
||||
@@ -267,7 +268,14 @@ class RuntimeConfig(Project, Profile, AdapterRequiredConfig):
|
||||
args,
|
||||
)
|
||||
flags = get_flags()
|
||||
project = load_project(project_root, bool(flags.VERSION_CHECK), profile, cli_vars)
|
||||
# For dbt deps, use lenient var validation to allow missing vars
|
||||
# For all other commands, use strict validation for helpful error messages
|
||||
# If command is not set (e.g., during test setup), default to strict mode
|
||||
# unless the command is explicitly "deps"
|
||||
require_vars = getattr(flags, "WHICH", None) != "deps"
|
||||
project = load_project(
|
||||
project_root, bool(flags.VERSION_CHECK), profile, cli_vars, require_vars=require_vars
|
||||
)
|
||||
return project, profile
|
||||
|
||||
# Called in task/base.py, in BaseTask.from_args
|
||||
|
||||
@@ -152,10 +152,12 @@ class Var:
|
||||
context: Mapping[str, Any],
|
||||
cli_vars: Mapping[str, Any],
|
||||
node: Optional[Resource] = None,
|
||||
require_vars: bool = True,
|
||||
) -> None:
|
||||
self._context: Mapping[str, Any] = context
|
||||
self._cli_vars: Mapping[str, Any] = cli_vars
|
||||
self._node: Optional[Resource] = node
|
||||
self._require_vars: bool = require_vars
|
||||
self._merged: Mapping[str, Any] = self._generate_merged()
|
||||
|
||||
def _generate_merged(self) -> Mapping[str, Any]:
|
||||
@@ -168,9 +170,11 @@ class Var:
|
||||
else:
|
||||
return "<Configuration>"
|
||||
|
||||
def get_missing_var(self, var_name: str) -> NoReturn:
|
||||
# TODO function name implies a non exception resolution
|
||||
raise RequiredVarNotFoundError(var_name, dict(self._merged), self._node)
|
||||
def get_missing_var(self, var_name: str) -> None:
|
||||
# Only raise an error if vars are _required_
|
||||
if self._require_vars:
|
||||
# TODO function name implies a non exception resolution
|
||||
raise RequiredVarNotFoundError(var_name, dict(self._merged), self._node)
|
||||
|
||||
def has_var(self, var_name: str) -> bool:
|
||||
return var_name in self._merged
|
||||
@@ -198,10 +202,11 @@ class BaseContext(metaclass=ContextMeta):
|
||||
_context_attrs_: Dict[str, Any]
|
||||
|
||||
# subclass is TargetContext
|
||||
def __init__(self, cli_vars: Dict[str, Any]) -> None:
|
||||
def __init__(self, cli_vars: Dict[str, Any], require_vars: bool = True) -> None:
|
||||
self._ctx: Dict[str, Any] = {}
|
||||
self.cli_vars: Dict[str, Any] = cli_vars
|
||||
self.env_vars: Dict[str, Any] = {}
|
||||
self.require_vars: bool = require_vars
|
||||
|
||||
def generate_builtins(self) -> Dict[str, Any]:
|
||||
builtins: Dict[str, Any] = {}
|
||||
@@ -307,7 +312,7 @@ class BaseContext(metaclass=ContextMeta):
|
||||
from events
|
||||
where event_type = '{{ var("event_type", "activation") }}'
|
||||
"""
|
||||
return Var(self._ctx, self.cli_vars)
|
||||
return Var(self._ctx, self.cli_vars, require_vars=self.require_vars)
|
||||
|
||||
@contextmember()
|
||||
def env_var(self, var: str, default: Optional[str] = None) -> str:
|
||||
|
||||
@@ -15,8 +15,8 @@ class ConfiguredContext(TargetContext):
|
||||
# subclasses are SchemaYamlContext, MacroResolvingContext, ManifestContext
|
||||
config: AdapterRequiredConfig
|
||||
|
||||
def __init__(self, config: AdapterRequiredConfig) -> None:
|
||||
super().__init__(config.to_target_dict(), config.cli_vars)
|
||||
def __init__(self, config: AdapterRequiredConfig, require_vars: bool = True) -> None:
|
||||
super().__init__(config.to_target_dict(), config.cli_vars, require_vars=require_vars)
|
||||
self.config = config
|
||||
|
||||
@contextproperty()
|
||||
|
||||
@@ -544,9 +544,15 @@ class ParseConfigObject(Config):
|
||||
def require(self, name, validator=None):
|
||||
return ""
|
||||
|
||||
def meta_require(self, name, validator=None):
|
||||
return ""
|
||||
|
||||
def get(self, name, default=None, validator=None):
|
||||
return ""
|
||||
|
||||
def meta_get(self, name, default=None, validator=None):
|
||||
return ""
|
||||
|
||||
def persist_relation_docs(self) -> bool:
|
||||
return False
|
||||
|
||||
@@ -578,6 +584,16 @@ class RuntimeConfigObject(Config):
|
||||
raise MissingConfigError(unique_id=self.model.unique_id, name=name)
|
||||
return result
|
||||
|
||||
def _lookup_meta(self, name, default=_MISSING):
|
||||
# if this is a macro, there might be no `model.config`.
|
||||
if not hasattr(self.model, "config"):
|
||||
result = default
|
||||
else:
|
||||
result = self.model.config.meta_get(name, default)
|
||||
if result is _MISSING:
|
||||
raise MissingConfigError(unique_id=self.model.unique_id, name=name)
|
||||
return result
|
||||
|
||||
def require(self, name, validator=None):
|
||||
to_return = self._lookup(name)
|
||||
|
||||
@@ -586,6 +602,14 @@ class RuntimeConfigObject(Config):
|
||||
|
||||
return to_return
|
||||
|
||||
def meta_require(self, name, validator=None):
|
||||
to_return = self._lookup_meta(name)
|
||||
|
||||
if validator is not None:
|
||||
self._validate(validator, to_return)
|
||||
|
||||
return to_return
|
||||
|
||||
def get(self, name, default=None, validator=None):
|
||||
to_return = self._lookup(name, default)
|
||||
|
||||
@@ -594,6 +618,14 @@ class RuntimeConfigObject(Config):
|
||||
|
||||
return to_return
|
||||
|
||||
def meta_get(self, name, default=None, validator=None):
|
||||
to_return = self._lookup_meta(name, default)
|
||||
|
||||
if validator is not None and default is not None:
|
||||
self._validate(validator, to_return)
|
||||
|
||||
return to_return
|
||||
|
||||
def persist_relation_docs(self) -> bool:
|
||||
persist_docs = self.get("persist_docs", default={})
|
||||
if not isinstance(persist_docs, dict):
|
||||
@@ -854,7 +886,12 @@ class RuntimeUnitTestSourceResolver(BaseSourceResolver):
|
||||
# we just need to set_cte, but skipping it confuses typing. We *do* need
|
||||
# the relation in the "this" property.
|
||||
self.model.set_cte(target_source.unique_id, None)
|
||||
return self.Relation.create_ephemeral_from(target_source)
|
||||
|
||||
identifier = self.Relation.add_ephemeral_prefix(target_source.cte_name)
|
||||
return self.Relation.create(
|
||||
type=self.Relation.CTE,
|
||||
identifier=identifier,
|
||||
).quote(identifier=False)
|
||||
|
||||
|
||||
# metric` implementations
|
||||
@@ -1905,6 +1942,21 @@ def generate_parser_model_context(
|
||||
return ctx.to_dict()
|
||||
|
||||
|
||||
def generate_parser_unit_test_context(
|
||||
unit_test: UnitTestNode, config: RuntimeConfig, manifest: Manifest
|
||||
) -> Dict[str, Any]:
|
||||
context_config = ContextConfig(
|
||||
config,
|
||||
unit_test.fqn,
|
||||
NodeType.Unit,
|
||||
config.project_name,
|
||||
)
|
||||
|
||||
ctx = UnitTestContext(unit_test, config, manifest, ParseProvider(), context_config)
|
||||
|
||||
return ctx.to_dict()
|
||||
|
||||
|
||||
def generate_generate_name_macro_context(
|
||||
macro: Macro,
|
||||
config: RuntimeConfig,
|
||||
|
||||
@@ -5,8 +5,10 @@ from dbt.context.base import BaseContext, contextproperty
|
||||
|
||||
class TargetContext(BaseContext):
|
||||
# subclass is ConfiguredContext
|
||||
def __init__(self, target_dict: Dict[str, Any], cli_vars: Dict[str, Any]):
|
||||
super().__init__(cli_vars=cli_vars)
|
||||
def __init__(
|
||||
self, target_dict: Dict[str, Any], cli_vars: Dict[str, Any], require_vars: bool = True
|
||||
):
|
||||
super().__init__(cli_vars=cli_vars, require_vars=require_vars)
|
||||
self.target_dict = target_dict
|
||||
|
||||
@contextproperty()
|
||||
|
||||
@@ -161,6 +161,7 @@ class SourceFile(BaseSourceFile):
|
||||
docs: List[str] = field(default_factory=list)
|
||||
macros: List[str] = field(default_factory=list)
|
||||
env_vars: List[str] = field(default_factory=list)
|
||||
functions: List[str] = field(default_factory=list)
|
||||
|
||||
@classmethod
|
||||
def big_seed(cls, path: FilePath) -> "SourceFile":
|
||||
|
||||
@@ -558,7 +558,10 @@ def _packages_to_search(
|
||||
elif current_project == node_package:
|
||||
return [current_project, None]
|
||||
else:
|
||||
return [current_project, node_package, None]
|
||||
if get_flags().require_ref_searches_node_package_before_root:
|
||||
return [node_package, current_project, None]
|
||||
else:
|
||||
return [current_project, node_package, None]
|
||||
|
||||
|
||||
def _sort_values(dct):
|
||||
@@ -1715,9 +1718,10 @@ class Manifest(MacroMethods, dbtClassMixin):
|
||||
self.exposures[exposure.unique_id] = exposure
|
||||
source_file.exposures.append(exposure.unique_id)
|
||||
|
||||
def add_function(self, function: FunctionNode):
|
||||
def add_function(self, source_file: SourceFile, function: FunctionNode):
|
||||
_check_duplicates(function, self.functions)
|
||||
self.functions[function.unique_id] = function
|
||||
source_file.functions.append(function.unique_id)
|
||||
|
||||
def add_metric(
|
||||
self, source_file: SchemaSourceFile, metric: Metric, generated_from: Optional[str] = None
|
||||
|
||||
@@ -697,6 +697,36 @@ class ModelNode(ModelResource, CompiledNode):
|
||||
)
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _normalize_data_type_for_comparison(data_type: Optional[str]) -> Optional[str]:
|
||||
"""
|
||||
Normalize a data type string by removing size, precision, and scale parameters.
|
||||
This allows comparison of base types while ignoring non-breaking parameter changes.
|
||||
|
||||
Examples:
|
||||
varchar(10) -> varchar
|
||||
VARCHAR(5) -> varchar
|
||||
numeric(10,2) -> numeric
|
||||
text -> text
|
||||
decimal(5) -> decimal
|
||||
None -> None
|
||||
|
||||
Per dbt documentation, changes to size/precision/scale should not be
|
||||
considered breaking changes for contracts.
|
||||
See: https://docs.getdbt.com/reference/resource-configs/contract#size-precision-and-scale
|
||||
|
||||
Note: Comparison is case-insensitive. Type aliases (e.g., 'varchar' vs
|
||||
'character varying') are not automatically resolved - users should use
|
||||
consistent type names in their contracts to avoid false positives.
|
||||
"""
|
||||
if not data_type:
|
||||
return data_type
|
||||
|
||||
# Split on the first '(' to get the base type without parameters
|
||||
# Convert to lowercase for case-insensitive comparison
|
||||
base_type, _, _ = data_type.partition("(")
|
||||
return base_type.strip().lower()
|
||||
|
||||
def same_contract(self, old, adapter_type=None) -> bool:
|
||||
# If the contract wasn't previously enforced:
|
||||
if old.contract.enforced is False and self.contract.enforced is False:
|
||||
@@ -738,14 +768,24 @@ class ModelNode(ModelResource, CompiledNode):
|
||||
columns_removed.append(old_value.name)
|
||||
# Has this column's data type changed?
|
||||
elif old_value.data_type != self.columns[old_key].data_type:
|
||||
column_type_changes.append(
|
||||
{
|
||||
"column_name": str(old_value.name),
|
||||
"previous_column_type": str(old_value.data_type),
|
||||
"current_column_type": str(self.columns[old_key].data_type),
|
||||
}
|
||||
# Compare normalized data types (without size/precision/scale)
|
||||
# to determine if this is a breaking change
|
||||
old_normalized = self._normalize_data_type_for_comparison(old_value.data_type)
|
||||
new_normalized = self._normalize_data_type_for_comparison(
|
||||
self.columns[old_key].data_type
|
||||
)
|
||||
|
||||
# Only consider it a breaking change if the base types differ
|
||||
# Changes like varchar(3) -> varchar(10) are not breaking
|
||||
if old_normalized != new_normalized:
|
||||
column_type_changes.append(
|
||||
{
|
||||
"column_name": str(old_value.name),
|
||||
"previous_column_type": str(old_value.data_type),
|
||||
"current_column_type": str(self.columns[old_key].data_type),
|
||||
}
|
||||
)
|
||||
|
||||
# track if there are any column level constraints for the materialization check late
|
||||
if old_value.constraints:
|
||||
column_constraints_exist = True
|
||||
@@ -1058,6 +1098,10 @@ class UnitTestSourceDefinition(ModelNode):
|
||||
source_name: str = "undefined"
|
||||
quoting: QuotingResource = field(default_factory=QuotingResource)
|
||||
|
||||
@property
|
||||
def cte_name(self):
|
||||
return self.unique_id.split(".")[-1]
|
||||
|
||||
@property
|
||||
def search_name(self):
|
||||
return f"{self.source_name}.{self.name}"
|
||||
|
||||
@@ -366,6 +366,8 @@ class ProjectFlags(ExtensibleDbtClassMixin):
|
||||
validate_macro_args: bool = False
|
||||
require_all_warnings_handled_by_warn_error: bool = False
|
||||
require_generic_test_arguments_property: bool = True
|
||||
require_unique_project_resource_names: bool = False
|
||||
require_ref_searches_node_package_before_root: bool = False
|
||||
|
||||
@property
|
||||
def project_only_flags(self) -> Dict[str, Any]:
|
||||
@@ -382,6 +384,8 @@ class ProjectFlags(ExtensibleDbtClassMixin):
|
||||
"validate_macro_args": self.validate_macro_args,
|
||||
"require_all_warnings_handled_by_warn_error": self.require_all_warnings_handled_by_warn_error,
|
||||
"require_generic_test_arguments_property": self.require_generic_test_arguments_property,
|
||||
"require_unique_project_resource_names": self.require_unique_project_resource_names,
|
||||
"require_ref_searches_node_package_before_root": self.require_ref_searches_node_package_before_root,
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -230,6 +230,11 @@ class ModulesItertoolsUsageDeprecation(DBTDeprecation):
|
||||
_event = "ModulesItertoolsUsageDeprecation"
|
||||
|
||||
|
||||
class DuplicateNameDistinctNodeTypesDeprecation(DBTDeprecation):
|
||||
_name = "duplicate-name-distinct-node-types-deprecation"
|
||||
_event = "DuplicateNameDistinctNodeTypesDeprecation"
|
||||
|
||||
|
||||
def renamed_env_var(old_name: str, new_name: str):
|
||||
class EnvironmentVariableRenamed(DBTDeprecation):
|
||||
_name = f"environment-variable-renamed:{old_name}"
|
||||
@@ -266,7 +271,7 @@ def show_deprecations_summary() -> None:
|
||||
deprecation_event = deprecations[deprecation].event()
|
||||
summaries.append(
|
||||
DeprecationSummary(
|
||||
event_name=deprecation_event.__name__,
|
||||
event_name=type(deprecation_event).__name__,
|
||||
event_code=deprecation_event.code(),
|
||||
occurrences=occurrences,
|
||||
).to_msg_dict()
|
||||
@@ -316,6 +321,7 @@ deprecations_list: List[DBTDeprecation] = [
|
||||
ArgumentsPropertyInGenericTestDeprecation(),
|
||||
MissingArgumentsPropertyInGenericTestDeprecation(),
|
||||
ModulesItertoolsUsageDeprecation(),
|
||||
DuplicateNameDistinctNodeTypesDeprecation(),
|
||||
]
|
||||
|
||||
deprecations: Dict[str, DBTDeprecation] = {d.name: d for d in deprecations_list}
|
||||
|
||||
@@ -771,6 +771,15 @@ class MissingArgumentsPropertyInGenericTestDeprecation(WarnLevel):
|
||||
return line_wrap_message(deprecation_tag(description, self.__class__.__name__))
|
||||
|
||||
|
||||
class DuplicateNameDistinctNodeTypesDeprecation(WarnLevel):
|
||||
def code(self) -> str:
|
||||
return "D040"
|
||||
|
||||
def message(self) -> str:
|
||||
description = f"Found resources with the same name '{self.resource_name}' in package '{self.package_name}': '{self.unique_id1}' and '{self.unique_id2}'. Please update one of the resources to have a unique name."
|
||||
return line_wrap_message(deprecation_tag(description))
|
||||
|
||||
|
||||
# =======================================================
|
||||
# I - Project parsing
|
||||
# =======================================================
|
||||
@@ -1262,6 +1271,19 @@ class InvalidMacroAnnotation(WarnLevel):
|
||||
return self.msg
|
||||
|
||||
|
||||
class PackageNodeDependsOnRootProjectNode(WarnLevel):
|
||||
def code(self) -> str:
|
||||
return "I077"
|
||||
|
||||
def message(self) -> str:
|
||||
msg = (
|
||||
f"The node '{self.node_name}'in package '{self.package_name}' depends on the root project node '{self.root_project_unique_id}'."
|
||||
"This may lead to unexpected cycles downstream. Please set the 'require_ref_prefers_node_package_to_root' behavior change flag to True to avoid this issue."
|
||||
"For more information, see the documentation at https://docs.getdbt.com/reference/global-configs/behavior-changes#require_ref_prefers_node_package_to_root"
|
||||
)
|
||||
return warning_tag(msg)
|
||||
|
||||
|
||||
# =======================================================
|
||||
# M - Deps generation
|
||||
# =======================================================
|
||||
|
||||
@@ -80,7 +80,7 @@ class AliasError(DbtValidationError):
|
||||
pass
|
||||
|
||||
|
||||
class DependencyError(Exception):
|
||||
class DependencyError(DbtRuntimeError):
|
||||
CODE = 10006
|
||||
MESSAGE = "Dependency Error"
|
||||
|
||||
|
||||
@@ -120,7 +120,9 @@ class NodeSelector(MethodManager):
|
||||
additional.update(self.graph.select_children(selected, depth))
|
||||
return additional
|
||||
|
||||
def select_nodes_recursively(self, spec: SelectionSpec) -> Tuple[Set[UniqueId], Set[UniqueId]]:
|
||||
def select_nodes_recursively(
|
||||
self, spec: SelectionSpec, warn_on_no_nodes: bool = True
|
||||
) -> Tuple[Set[UniqueId], Set[UniqueId]]:
|
||||
"""If the spec is a composite spec (a union, difference, or intersection),
|
||||
recurse into its selections and combine them. If the spec is a concrete
|
||||
selection criteria, resolve that using the given graph.
|
||||
@@ -128,7 +130,10 @@ class NodeSelector(MethodManager):
|
||||
if isinstance(spec, SelectionCriteria):
|
||||
direct_nodes, indirect_nodes = self.get_nodes_from_criteria(spec)
|
||||
else:
|
||||
bundles = [self.select_nodes_recursively(component) for component in spec]
|
||||
bundles = [
|
||||
self.select_nodes_recursively(spec=component, warn_on_no_nodes=warn_on_no_nodes)
|
||||
for component in spec.components
|
||||
]
|
||||
|
||||
direct_sets = []
|
||||
indirect_sets = []
|
||||
@@ -144,19 +149,23 @@ class NodeSelector(MethodManager):
|
||||
initial_direct, indirect_nodes, spec.indirect_selection
|
||||
)
|
||||
|
||||
if spec.expect_exists and len(direct_nodes) == 0:
|
||||
if spec.expect_exists and len(direct_nodes) == 0 and warn_on_no_nodes:
|
||||
warn_or_error(NoNodesForSelectionCriteria(spec_raw=str(spec.raw)))
|
||||
|
||||
return direct_nodes, indirect_nodes
|
||||
|
||||
def select_nodes(self, spec: SelectionSpec) -> Tuple[Set[UniqueId], Set[UniqueId]]:
|
||||
def select_nodes(
|
||||
self, spec: SelectionSpec, warn_on_no_nodes: bool = True
|
||||
) -> Tuple[Set[UniqueId], Set[UniqueId]]:
|
||||
"""Select the nodes in the graph according to the spec.
|
||||
|
||||
This is the main point of entry for turning a spec into a set of nodes:
|
||||
- Recurse through spec, select by criteria, combine by set operation
|
||||
- Return final (unfiltered) selection set
|
||||
"""
|
||||
direct_nodes, indirect_nodes = self.select_nodes_recursively(spec)
|
||||
direct_nodes, indirect_nodes = self.select_nodes_recursively(
|
||||
spec=spec, warn_on_no_nodes=warn_on_no_nodes
|
||||
)
|
||||
indirect_only = indirect_nodes.difference(direct_nodes)
|
||||
return direct_nodes, indirect_only
|
||||
|
||||
@@ -324,7 +333,7 @@ class NodeSelector(MethodManager):
|
||||
|
||||
return selected
|
||||
|
||||
def get_selected(self, spec: SelectionSpec) -> Set[UniqueId]:
|
||||
def get_selected(self, spec: SelectionSpec, warn_on_no_nodes: bool = True) -> Set[UniqueId]:
|
||||
"""get_selected runs through the node selection process:
|
||||
|
||||
- node selection. Based on the include/exclude sets, the set
|
||||
@@ -334,7 +343,9 @@ class NodeSelector(MethodManager):
|
||||
- selectors can filter the nodes after all of them have been
|
||||
selected
|
||||
"""
|
||||
selected_nodes, indirect_only = self.select_nodes(spec)
|
||||
selected_nodes, indirect_only = self.select_nodes(
|
||||
spec=spec, warn_on_no_nodes=warn_on_no_nodes
|
||||
)
|
||||
filtered_nodes = self.filter_selection(selected_nodes)
|
||||
|
||||
return filtered_nodes
|
||||
|
||||
@@ -35,7 +35,7 @@ from dbt.contracts.state import PreviousState
|
||||
from dbt.node_types import NodeType
|
||||
from dbt_common.dataclass_schema import StrEnum
|
||||
from dbt_common.events.contextvars import get_project_root
|
||||
from dbt_common.exceptions import DbtInternalError, DbtRuntimeError
|
||||
from dbt_common.exceptions import CompilationError, DbtInternalError, DbtRuntimeError
|
||||
|
||||
from .graph import UniqueId
|
||||
|
||||
@@ -655,6 +655,16 @@ class StateSelectorMethod(SelectorMethod):
|
||||
continue
|
||||
visited_macros.append(macro_uid)
|
||||
|
||||
# If macro_uid is None, it means the macro/test was removed but is still referenced.
|
||||
# Raise a clear error to match the behavior of regular dbt run.
|
||||
if macro_uid is None:
|
||||
raise CompilationError(
|
||||
f"Node '{node.name}' (in {node.original_file_path}) depends on a macro or test "
|
||||
f"that does not exist. This can happen when a macro or generic test is removed "
|
||||
f"but is still referenced. Check for typos and/or install package dependencies "
|
||||
f"with 'dbt deps'."
|
||||
)
|
||||
|
||||
if macro_uid in self.modified_macros:
|
||||
return True
|
||||
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
from datetime import date, datetime
|
||||
from pathlib import Path
|
||||
@@ -38,6 +37,10 @@ _HIERARCHICAL_CONFIG_KEYS = {
|
||||
"unit_tests",
|
||||
}
|
||||
|
||||
_ADAPTER_TO_CONFIG_ALIASES = {
|
||||
"bigquery": ["dataset", "project"],
|
||||
}
|
||||
|
||||
|
||||
def load_json_from_package(jsonschema_type: str, filename: str) -> Dict[str, Any]:
|
||||
"""Loads a JSON file from within a package."""
|
||||
@@ -107,6 +110,16 @@ def _validate_with_schema(
|
||||
return validator.iter_errors(json)
|
||||
|
||||
|
||||
def _get_allowed_config_key_aliases() -> List[str]:
|
||||
config_aliases = []
|
||||
invocation_context = get_invocation_context()
|
||||
for adapter in invocation_context.adapter_types:
|
||||
if adapter in _ADAPTER_TO_CONFIG_ALIASES:
|
||||
config_aliases.extend(_ADAPTER_TO_CONFIG_ALIASES[adapter])
|
||||
|
||||
return config_aliases
|
||||
|
||||
|
||||
def _get_allowed_config_fields_from_error_path(
|
||||
yml_schema: Dict[str, Any], error_path: List[Union[str, int]]
|
||||
) -> Optional[List[str]]:
|
||||
@@ -136,14 +149,12 @@ def _get_allowed_config_fields_from_error_path(
|
||||
][0]["$ref"].split("/")[-1]
|
||||
|
||||
allowed_config_fields = list(set(yml_schema["definitions"][config_field_name]["properties"]))
|
||||
allowed_config_fields.extend(_get_allowed_config_key_aliases())
|
||||
|
||||
return allowed_config_fields
|
||||
|
||||
|
||||
def _can_run_validations() -> bool:
|
||||
if not os.environ.get("DBT_ENV_PRIVATE_RUN_JSONSCHEMA_VALIDATIONS"):
|
||||
return False
|
||||
|
||||
invocation_context = get_invocation_context()
|
||||
return invocation_context.adapter_types.issubset(_JSONSCHEMA_SUPPORTED_ADAPTERS)
|
||||
|
||||
@@ -168,8 +179,11 @@ def jsonschema_validate(schema: Dict[str, Any], json: Dict[str, Any], file_path:
|
||||
else:
|
||||
key_path = error_path_to_string(error)
|
||||
for key in keys:
|
||||
if key == "overrides" and key_path.startswith("sources"):
|
||||
# Type params are not in the metrics v2 jsonschema from fusion, but dbt-core continues to maintain support for them in v1.
|
||||
if key == "type_params":
|
||||
continue
|
||||
|
||||
if key == "overrides" and key_path.startswith("sources"):
|
||||
deprecations.warn(
|
||||
"source-override-deprecation",
|
||||
source_name=key_path.split(".")[-1],
|
||||
@@ -205,6 +219,9 @@ def jsonschema_validate(schema: Dict[str, Any], json: Dict[str, Any], file_path:
|
||||
keys = _additional_properties_violation_keys(sub_error)
|
||||
key_path = error_path_to_string(error)
|
||||
for key in keys:
|
||||
if key in _get_allowed_config_key_aliases():
|
||||
continue
|
||||
|
||||
deprecations.warn(
|
||||
"custom-key-in-config-deprecation",
|
||||
key=key,
|
||||
@@ -265,6 +282,11 @@ def validate_model_config(config: Dict[str, Any], file_path: str) -> None:
|
||||
if len(error.path) == 0:
|
||||
key_path = error_path_to_string(error)
|
||||
for key in keys:
|
||||
# Special case for pre/post hook keys as they are updated during config parsing
|
||||
# from the user-provided pre_hook/post_hook to pre-hook/post-hook keys.
|
||||
# Avoids false positives as described in https://github.com/dbt-labs/dbt-core/issues/12087
|
||||
if key in ("post-hook", "pre-hook"):
|
||||
continue
|
||||
deprecations.warn(
|
||||
"custom-key-in-config-deprecation",
|
||||
key=key,
|
||||
|
||||
@@ -6,6 +6,16 @@
|
||||
"name"
|
||||
],
|
||||
"properties": {
|
||||
"analyses": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/ProjectAnalysisConfig"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"analysis-paths": {
|
||||
"type": [
|
||||
"array",
|
||||
@@ -98,6 +108,25 @@
|
||||
}
|
||||
]
|
||||
},
|
||||
"function-paths": {
|
||||
"type": [
|
||||
"array",
|
||||
"null"
|
||||
],
|
||||
"items": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"functions": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/ProjectFunctionConfig"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"log-path": {
|
||||
"anyOf": [
|
||||
{
|
||||
@@ -220,7 +249,7 @@
|
||||
"saved-queries": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/ProjectSavedQueriesConfig"
|
||||
"$ref": "#/definitions/ProjectSavedQueryConfig"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
@@ -399,6 +428,17 @@
|
||||
},
|
||||
"additionalProperties": false
|
||||
},
|
||||
"DataLakeObjectCategory": {
|
||||
"description": "See `category` from https://developer.salesforce.com/docs/data/connectapi/references/spec?meta=postDataLakeObject",
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"Profile",
|
||||
"Engagement",
|
||||
"Directory_Table",
|
||||
"Insights",
|
||||
"Other"
|
||||
]
|
||||
},
|
||||
"DbtBatchSize": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
@@ -441,8 +481,7 @@
|
||||
"merge",
|
||||
"delete+insert",
|
||||
"insert_overwrite",
|
||||
"microbatch",
|
||||
"unknown"
|
||||
"microbatch"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -451,6 +490,18 @@
|
||||
"enum": [
|
||||
"replace_where"
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "object",
|
||||
"required": [
|
||||
"custom"
|
||||
],
|
||||
"properties": {
|
||||
"custom": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"additionalProperties": false
|
||||
}
|
||||
]
|
||||
},
|
||||
@@ -459,6 +510,8 @@
|
||||
{
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"snapshot",
|
||||
"seed",
|
||||
"view",
|
||||
"table",
|
||||
"incremental",
|
||||
@@ -467,7 +520,8 @@
|
||||
"test",
|
||||
"ephemeral",
|
||||
"unit",
|
||||
"analysis"
|
||||
"analysis",
|
||||
"function"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -484,6 +538,13 @@
|
||||
"dynamic_table"
|
||||
]
|
||||
},
|
||||
{
|
||||
"description": "for inline SQL compilation",
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"inline"
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "object",
|
||||
"required": [
|
||||
@@ -647,6 +708,15 @@
|
||||
},
|
||||
"additionalProperties": false
|
||||
},
|
||||
"FunctionKind": {
|
||||
"description": "Function kind enum with same values as UDFKind",
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"scalar",
|
||||
"aggregate",
|
||||
"table"
|
||||
]
|
||||
},
|
||||
"GrantAccessToTarget": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
@@ -676,6 +746,12 @@
|
||||
"HookConfig": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"index": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"sql": {
|
||||
"type": [
|
||||
"string",
|
||||
@@ -788,7 +864,7 @@
|
||||
]
|
||||
},
|
||||
"PartitionConfig": {
|
||||
"description": "dbt-core allows either of the variants for the `partition_by` in the model config but the bigquery-adapter throws RunTime error the behaviors are tested from the latest dbt-core + bigquery-adapter as this is written we're conformant to this behavior via here and via the `validate` method",
|
||||
"description": "dbt-core allows either of the variants for the `partition_by` in the model config but the bigquery-adapter throws RunTime error the behaviors are tested from the latest dbt-core + bigquery-adapter as this is written we're conformant to this behavior via here and via the `into_bigquery()` method",
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "string"
|
||||
@@ -849,9 +925,77 @@
|
||||
},
|
||||
"additionalProperties": false
|
||||
},
|
||||
"ProjectAnalysisConfig": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"+docs": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/DocsConfig"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"+enabled": {
|
||||
"type": [
|
||||
"boolean",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"+group": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"+meta": {
|
||||
"type": [
|
||||
"object",
|
||||
"null"
|
||||
],
|
||||
"additionalProperties": {
|
||||
"$ref": "#/definitions/AnyValue"
|
||||
}
|
||||
},
|
||||
"+static_analysis": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/StaticAnalysisKind"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"+tags": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/StringOrArrayOfStrings"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"additionalProperties": {
|
||||
"$ref": "#/definitions/ProjectAnalysisConfig"
|
||||
}
|
||||
},
|
||||
"ProjectDataTestConfig": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"+adapter_properties": {
|
||||
"type": [
|
||||
"object",
|
||||
"null"
|
||||
],
|
||||
"additionalProperties": {
|
||||
"$ref": "#/definitions/AnyValue"
|
||||
}
|
||||
},
|
||||
"+alias": {
|
||||
"type": [
|
||||
"string",
|
||||
@@ -919,6 +1063,12 @@
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"+catalog_name": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"+cluster_by": {
|
||||
"anyOf": [
|
||||
{
|
||||
@@ -1194,11 +1344,10 @@
|
||||
},
|
||||
"+refresh_interval_minutes": {
|
||||
"type": [
|
||||
"integer",
|
||||
"number",
|
||||
"null"
|
||||
],
|
||||
"format": "uint64",
|
||||
"minimum": 0.0
|
||||
"format": "double"
|
||||
},
|
||||
"+refresh_mode": {
|
||||
"type": [
|
||||
@@ -1476,6 +1625,12 @@
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"tenant_hostname": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
}
|
||||
},
|
||||
"additionalProperties": false
|
||||
@@ -1513,6 +1668,150 @@
|
||||
"$ref": "#/definitions/ProjectExposureConfig"
|
||||
}
|
||||
},
|
||||
"ProjectFunctionConfig": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"+access": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/Access"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"+alias": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"+database": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"+description": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"+docs": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/DocsConfig"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"+enabled": {
|
||||
"type": [
|
||||
"boolean",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"+grants": {
|
||||
"type": [
|
||||
"object",
|
||||
"null"
|
||||
],
|
||||
"additionalProperties": {
|
||||
"$ref": "#/definitions/StringOrArrayOfStrings"
|
||||
}
|
||||
},
|
||||
"+group": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"+language": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"+meta": {
|
||||
"type": [
|
||||
"object",
|
||||
"null"
|
||||
],
|
||||
"additionalProperties": {
|
||||
"$ref": "#/definitions/AnyValue"
|
||||
}
|
||||
},
|
||||
"+on_configuration_change": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"+quoting": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/DbtQuoting"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"+schema": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"+static_analysis": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/StaticAnalysisKind"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"+tags": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/StringOrArrayOfStrings"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"+type": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/FunctionKind"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"+volatility": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/Volatility"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"additionalProperties": {
|
||||
"$ref": "#/definitions/ProjectFunctionConfig"
|
||||
}
|
||||
},
|
||||
"ProjectMetricConfigs": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
@@ -1565,6 +1864,15 @@
|
||||
}
|
||||
]
|
||||
},
|
||||
"+adapter_properties": {
|
||||
"type": [
|
||||
"object",
|
||||
"null"
|
||||
],
|
||||
"additionalProperties": {
|
||||
"$ref": "#/definitions/AnyValue"
|
||||
}
|
||||
},
|
||||
"+alias": {
|
||||
"type": [
|
||||
"string",
|
||||
@@ -1654,6 +1962,16 @@
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"+category": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/DataLakeObjectCategory"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"+cluster_by": {
|
||||
"anyOf": [
|
||||
{
|
||||
@@ -1828,6 +2146,16 @@
|
||||
"format": "uint64",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"+imports": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/StringOrArrayOfStrings"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"+include_full_name_in_path": {
|
||||
"type": [
|
||||
"boolean",
|
||||
@@ -2089,6 +2417,12 @@
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"+primary_key": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"+query_tag": {
|
||||
"type": [
|
||||
"string",
|
||||
@@ -2107,11 +2441,10 @@
|
||||
},
|
||||
"+refresh_interval_minutes": {
|
||||
"type": [
|
||||
"integer",
|
||||
"number",
|
||||
"null"
|
||||
],
|
||||
"format": "uint64",
|
||||
"minimum": 0.0
|
||||
"format": "double"
|
||||
},
|
||||
"+refresh_mode": {
|
||||
"type": [
|
||||
@@ -2125,6 +2458,15 @@
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"+resource_tags": {
|
||||
"type": [
|
||||
"object",
|
||||
"null"
|
||||
],
|
||||
"additionalProperties": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"+row_access_policy": {
|
||||
"type": [
|
||||
"string",
|
||||
@@ -2278,13 +2620,13 @@
|
||||
"$ref": "#/definitions/ProjectModelConfig"
|
||||
}
|
||||
},
|
||||
"ProjectSavedQueriesConfig": {
|
||||
"ProjectSavedQueryConfig": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"+cache": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/SavedQueriesConfigCache"
|
||||
"$ref": "#/definitions/SavedQueryCache"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
@@ -2340,12 +2682,21 @@
|
||||
}
|
||||
},
|
||||
"additionalProperties": {
|
||||
"$ref": "#/definitions/ProjectSavedQueriesConfig"
|
||||
"$ref": "#/definitions/ProjectSavedQueryConfig"
|
||||
}
|
||||
},
|
||||
"ProjectSeedConfig": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"+adapter_properties": {
|
||||
"type": [
|
||||
"object",
|
||||
"null"
|
||||
],
|
||||
"additionalProperties": {
|
||||
"$ref": "#/definitions/AnyValue"
|
||||
}
|
||||
},
|
||||
"+alias": {
|
||||
"type": [
|
||||
"string",
|
||||
@@ -2413,6 +2764,12 @@
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"+catalog_name": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"+cluster_by": {
|
||||
"anyOf": [
|
||||
{
|
||||
@@ -2751,11 +3108,10 @@
|
||||
},
|
||||
"+refresh_interval_minutes": {
|
||||
"type": [
|
||||
"integer",
|
||||
"number",
|
||||
"null"
|
||||
],
|
||||
"format": "uint64",
|
||||
"minimum": 0.0
|
||||
"format": "double"
|
||||
},
|
||||
"+refresh_mode": {
|
||||
"type": [
|
||||
@@ -2900,6 +3256,27 @@
|
||||
"ProjectSemanticModelConfig": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"+enabled": {
|
||||
"type": [
|
||||
"boolean",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"+group": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"+meta": {
|
||||
"type": [
|
||||
"object",
|
||||
"null"
|
||||
],
|
||||
"additionalProperties": {
|
||||
"$ref": "#/definitions/AnyValue"
|
||||
}
|
||||
},
|
||||
"+tags": {
|
||||
"anyOf": [
|
||||
{
|
||||
@@ -2909,27 +3286,6 @@
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"enabled": {
|
||||
"type": [
|
||||
"boolean",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"group": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"meta": {
|
||||
"type": [
|
||||
"object",
|
||||
"null"
|
||||
],
|
||||
"additionalProperties": {
|
||||
"$ref": "#/definitions/AnyValue"
|
||||
}
|
||||
}
|
||||
},
|
||||
"additionalProperties": {
|
||||
@@ -2939,6 +3295,15 @@
|
||||
"ProjectSnapshotConfig": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"+adapter_properties": {
|
||||
"type": [
|
||||
"object",
|
||||
"null"
|
||||
],
|
||||
"additionalProperties": {
|
||||
"$ref": "#/definitions/AnyValue"
|
||||
}
|
||||
},
|
||||
"+alias": {
|
||||
"type": [
|
||||
"string",
|
||||
@@ -3006,6 +3371,12 @@
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"+catalog_name": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"+check_cols": {
|
||||
"anyOf": [
|
||||
{
|
||||
@@ -3365,11 +3736,10 @@
|
||||
},
|
||||
"+refresh_interval_minutes": {
|
||||
"type": [
|
||||
"integer",
|
||||
"number",
|
||||
"null"
|
||||
],
|
||||
"format": "uint64",
|
||||
"minimum": 0.0
|
||||
"format": "double"
|
||||
},
|
||||
"+refresh_mode": {
|
||||
"type": [
|
||||
@@ -3505,12 +3875,24 @@
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"+target_database": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"+target_lag": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"+target_schema": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"+tblproperties": {
|
||||
"type": [
|
||||
"object",
|
||||
@@ -3556,6 +3938,15 @@
|
||||
"ProjectSourceConfig": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"+adapter_properties": {
|
||||
"type": [
|
||||
"object",
|
||||
"null"
|
||||
],
|
||||
"additionalProperties": {
|
||||
"$ref": "#/definitions/AnyValue"
|
||||
}
|
||||
},
|
||||
"+as_columnstore": {
|
||||
"type": [
|
||||
"boolean",
|
||||
@@ -3617,6 +4008,12 @@
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"+catalog_name": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"+cluster_by": {
|
||||
"anyOf": [
|
||||
{
|
||||
@@ -3889,11 +4286,10 @@
|
||||
},
|
||||
"+refresh_interval_minutes": {
|
||||
"type": [
|
||||
"integer",
|
||||
"number",
|
||||
"null"
|
||||
],
|
||||
"format": "uint64",
|
||||
"minimum": 0.0
|
||||
"format": "double"
|
||||
},
|
||||
"+refresh_mode": {
|
||||
"type": [
|
||||
@@ -4042,6 +4438,15 @@
|
||||
"ProjectUnitTestConfig": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"+adapter_properties": {
|
||||
"type": [
|
||||
"object",
|
||||
"null"
|
||||
],
|
||||
"additionalProperties": {
|
||||
"$ref": "#/definitions/AnyValue"
|
||||
}
|
||||
},
|
||||
"+as_columnstore": {
|
||||
"type": [
|
||||
"boolean",
|
||||
@@ -4103,6 +4508,12 @@
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"+catalog_name": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"+cluster_by": {
|
||||
"anyOf": [
|
||||
{
|
||||
@@ -4337,11 +4748,10 @@
|
||||
},
|
||||
"+refresh_interval_minutes": {
|
||||
"type": [
|
||||
"integer",
|
||||
"number",
|
||||
"null"
|
||||
],
|
||||
"format": "uint64",
|
||||
"minimum": 0.0
|
||||
"format": "double"
|
||||
},
|
||||
"+refresh_mode": {
|
||||
"type": [
|
||||
@@ -4535,7 +4945,7 @@
|
||||
},
|
||||
"additionalProperties": false
|
||||
},
|
||||
"SavedQueriesConfigCache": {
|
||||
"SavedQueryCache": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"enabled": {
|
||||
@@ -4576,35 +4986,30 @@
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"dbt_is_deleted": {
|
||||
"default": "DBT_IS_DELETED",
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"dbt_scd_id": {
|
||||
"default": "DBT_SCD_ID",
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"dbt_updated_at": {
|
||||
"default": "DBT_UPDATED_AT",
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"dbt_valid_from": {
|
||||
"default": "DBT_VALID_FROM",
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"dbt_valid_to": {
|
||||
"default": "DBT_VALID_TO",
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
@@ -4694,6 +5099,31 @@
|
||||
"all"
|
||||
]
|
||||
},
|
||||
"Volatility": {
|
||||
"description": "Function volatility enum - defines the function's eligibility for certain optimizations Matches the Python Volatility enum from dbt-core",
|
||||
"oneOf": [
|
||||
{
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"stable"
|
||||
]
|
||||
},
|
||||
{
|
||||
"description": "Deterministic - An deterministic function will always return the same output when given the same input.",
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"deterministic"
|
||||
]
|
||||
},
|
||||
{
|
||||
"description": "NonDeterministic - A non-deterministic function may change the return value from evaluation to evaluation. Multiple invocations of a non-deterministic function may return different results when used in the same query.",
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"non-deterministic"
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"_Dispatch": {
|
||||
"type": "object",
|
||||
"required": [
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,4 +1,5 @@
|
||||
from dbt.artifacts.resources.types import NodeType
|
||||
from dbt.contracts.files import SourceFile
|
||||
from dbt.contracts.graph.nodes import FunctionNode, ManifestNode
|
||||
from dbt.parser.base import SimpleParser
|
||||
from dbt.parser.search import FileBlock
|
||||
@@ -21,10 +22,12 @@ class FunctionParser(SimpleParser[FileBlock, FunctionNode]):
|
||||
# overrides SimpleSQLParser.add_result_node
|
||||
def add_result_node(self, block: FileBlock, node: ManifestNode):
|
||||
assert isinstance(node, FunctionNode), "Got non FunctionNode in FunctionParser"
|
||||
file = block.file
|
||||
assert isinstance(file, SourceFile)
|
||||
if node.config.enabled:
|
||||
self.manifest.add_function(node)
|
||||
self.manifest.add_function(file, node)
|
||||
else:
|
||||
self.manifest.add_disabled(block.file, node)
|
||||
self.manifest.add_disabled(file, node)
|
||||
|
||||
def parse_file(self, file_block: FileBlock) -> None:
|
||||
self.parse_node(file_block)
|
||||
|
||||
@@ -77,6 +77,7 @@ from dbt.events.types import (
|
||||
InvalidDisabledTargetInTestNode,
|
||||
MicrobatchModelNoEventTimeInputs,
|
||||
NodeNotFoundOrDisabled,
|
||||
PackageNodeDependsOnRootProjectNode,
|
||||
ParsedFileLoadFailed,
|
||||
ParsePerfInfoPath,
|
||||
PartialParsingError,
|
||||
@@ -90,6 +91,7 @@ from dbt.events.types import (
|
||||
)
|
||||
from dbt.exceptions import (
|
||||
AmbiguousAliasError,
|
||||
DuplicateResourceNameError,
|
||||
InvalidAccessTypeError,
|
||||
TargetNotFoundError,
|
||||
scrub_secrets,
|
||||
@@ -525,6 +527,7 @@ class ManifestLoader:
|
||||
self.check_for_microbatch_deprecations()
|
||||
self.check_forcing_batch_concurrency()
|
||||
self.check_microbatch_model_has_a_filtered_input()
|
||||
self.check_function_default_arguments_ordering()
|
||||
|
||||
return self.manifest
|
||||
|
||||
@@ -536,6 +539,9 @@ class ManifestLoader:
|
||||
self.skip_parsing = self.partial_parser.skip_parsing()
|
||||
if self.skip_parsing:
|
||||
# nothing changed, so we don't need to generate project_parser_files
|
||||
fire_event(
|
||||
Note(msg="Nothing changed, skipping partial parsing."), level=EventLevel.DEBUG
|
||||
)
|
||||
self.manifest = self.saved_manifest # type: ignore[assignment]
|
||||
else:
|
||||
# create child_map and parent_map
|
||||
@@ -630,23 +636,24 @@ class ManifestLoader:
|
||||
def check_for_spaces_in_resource_names(self):
|
||||
"""Validates that resource names do not contain spaces
|
||||
|
||||
If `DEBUG` flag is `False`, logs only first bad model name
|
||||
If `DEBUG` flag is `False`, logs only first bad model name, unless `REQUIRE_RESOURCE_NAMES_WITHOUT_SPACES` is `True` as error will indicate all bad model names
|
||||
If `DEBUG` flag is `True`, logs every bad model name
|
||||
If `REQUIRE_RESOURCE_NAMES_WITHOUT_SPACES` is `True`, logs are `ERROR` level and an exception is raised if any names are bad
|
||||
If `REQUIRE_RESOURCE_NAMES_WITHOUT_SPACES` is `False`, logs are `WARN` level
|
||||
"""
|
||||
improper_resource_names = 0
|
||||
level = (
|
||||
EventLevel.ERROR
|
||||
if self.root_project.args.REQUIRE_RESOURCE_NAMES_WITHOUT_SPACES
|
||||
else EventLevel.WARN
|
||||
improper_resource_names_unique_ids = set()
|
||||
error_on_invalid_resource_name = (
|
||||
self.root_project.args.REQUIRE_RESOURCE_NAMES_WITHOUT_SPACES
|
||||
)
|
||||
level = EventLevel.ERROR if error_on_invalid_resource_name else EventLevel.WARN
|
||||
|
||||
flags = get_flags()
|
||||
|
||||
for node in self.manifest.nodes.values():
|
||||
if " " in node.name:
|
||||
if improper_resource_names == 0 or flags.DEBUG:
|
||||
if (
|
||||
not improper_resource_names_unique_ids and not error_on_invalid_resource_name
|
||||
) or flags.DEBUG:
|
||||
fire_event(
|
||||
SpacesInResourceNameDeprecation(
|
||||
unique_id=node.unique_id,
|
||||
@@ -654,17 +661,23 @@ class ManifestLoader:
|
||||
),
|
||||
level=level,
|
||||
)
|
||||
improper_resource_names += 1
|
||||
improper_resource_names_unique_ids.add(node.unique_id)
|
||||
|
||||
if improper_resource_names > 0:
|
||||
if improper_resource_names_unique_ids:
|
||||
if level == EventLevel.WARN:
|
||||
dbt.deprecations.warn(
|
||||
"resource-names-with-spaces",
|
||||
count_invalid_names=improper_resource_names,
|
||||
count_invalid_names=len(improper_resource_names_unique_ids),
|
||||
show_debug_hint=(not flags.DEBUG),
|
||||
)
|
||||
else: # ERROR level
|
||||
raise DbtValidationError("Resource names cannot contain spaces")
|
||||
formatted_resources_with_spaces = "\n".join(
|
||||
f" * '{unique_id}' ({self.manifest.nodes[unique_id].original_file_path})"
|
||||
for unique_id in improper_resource_names_unique_ids
|
||||
)
|
||||
raise DbtValidationError(
|
||||
f"Resource names cannot contain spaces:\n{formatted_resources_with_spaces}\nPlease rename the invalid model(s) so that their name(s) do not contain any spaces."
|
||||
)
|
||||
|
||||
def check_for_microbatch_deprecations(self) -> None:
|
||||
if not get_flags().require_batched_execution_for_custom_microbatch_strategy:
|
||||
@@ -1547,6 +1560,17 @@ class ManifestLoader:
|
||||
if not has_input_with_event_time_config:
|
||||
fire_event(MicrobatchModelNoEventTimeInputs(model_name=node.name))
|
||||
|
||||
def check_function_default_arguments_ordering(self):
|
||||
for function in self.manifest.functions.values():
|
||||
found_default_value = False
|
||||
for argument in function.arguments:
|
||||
if not found_default_value and argument.default_value is not None:
|
||||
found_default_value = True
|
||||
elif found_default_value and argument.default_value is None:
|
||||
raise dbt.exceptions.ParsingError(
|
||||
f"Non-defaulted argument '{argument.name}' of function '{function.name}' comes after a defaulted argument. Non-defaulted arguments cannot come after defaulted arguments. "
|
||||
)
|
||||
|
||||
def write_perf_info(self, target_path: str):
|
||||
path = os.path.join(target_path, PERF_INFO_FILE_NAME)
|
||||
write_file(path, json.dumps(self._perf_info, cls=dbt.utils.JSONEncoder, indent=4))
|
||||
@@ -1613,6 +1637,33 @@ def invalid_target_fail_unless_test(
|
||||
)
|
||||
|
||||
|
||||
def warn_if_package_node_depends_on_root_project_node(
|
||||
node: ManifestNode,
|
||||
target_model: ManifestNode,
|
||||
ref_package_name: Optional[str],
|
||||
current_project: str,
|
||||
) -> None:
|
||||
"""
|
||||
Args:
|
||||
node: The node that specifies the ref
|
||||
target_model: The node that is being ref'd to
|
||||
ref_package_name: The package name specified in the ref
|
||||
current_project: The root project
|
||||
"""
|
||||
if (
|
||||
node.package_name != current_project
|
||||
and target_model.package_name == current_project
|
||||
and ref_package_name != current_project
|
||||
):
|
||||
warn_or_error(
|
||||
PackageNodeDependsOnRootProjectNode(
|
||||
node_name=node.name,
|
||||
package_name=node.package_name,
|
||||
root_project_unique_id=target_model.unique_id,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _build_model_names_to_versions(manifest: Manifest) -> Dict[str, Dict]:
|
||||
model_names_to_versions: Dict[str, Dict] = {}
|
||||
for node in manifest.nodes.values():
|
||||
@@ -1635,12 +1686,26 @@ def _check_resource_uniqueness(
|
||||
alias_resources: Dict[str, ManifestNode] = {}
|
||||
name_resources: Dict[str, Dict] = {}
|
||||
|
||||
for resource, node in manifest.nodes.items():
|
||||
for _, node in manifest.nodes.items():
|
||||
if not node.is_relational:
|
||||
continue
|
||||
|
||||
if node.package_name not in name_resources:
|
||||
name_resources[node.package_name] = {"ver": {}, "unver": {}}
|
||||
|
||||
existing_unversioned_node = name_resources[node.package_name]["unver"].get(node.name)
|
||||
if existing_unversioned_node is not None and not node.is_versioned:
|
||||
if get_flags().require_unique_project_resource_names:
|
||||
raise DuplicateResourceNameError(existing_unversioned_node, node)
|
||||
else:
|
||||
dbt.deprecations.warn(
|
||||
"duplicate-name-distinct-node-types-deprecation",
|
||||
resource_name=node.name,
|
||||
unique_id1=existing_unversioned_node.unique_id,
|
||||
unique_id2=node.unique_id,
|
||||
package_name=node.package_name,
|
||||
)
|
||||
|
||||
if node.is_versioned:
|
||||
name_resources[node.package_name]["ver"][node.name] = node
|
||||
else:
|
||||
@@ -1856,6 +1921,11 @@ def _process_refs(
|
||||
scope=target_model.package_name,
|
||||
)
|
||||
|
||||
if not get_flags().require_ref_searches_node_package_before_root:
|
||||
warn_if_package_node_depends_on_root_project_node(
|
||||
node, target_model, ref.package, current_project
|
||||
)
|
||||
|
||||
target_model_id = target_model.unique_id
|
||||
node.depends_on.add_node(target_model_id)
|
||||
|
||||
|
||||
@@ -11,7 +11,13 @@ from dbt.contracts.files import (
|
||||
parse_file_type_to_parser,
|
||||
)
|
||||
from dbt.contracts.graph.manifest import Manifest
|
||||
from dbt.contracts.graph.nodes import AnalysisNode, ModelNode, SeedNode, SnapshotNode
|
||||
from dbt.contracts.graph.nodes import (
|
||||
AnalysisNode,
|
||||
GenericTestNode,
|
||||
ModelNode,
|
||||
SeedNode,
|
||||
SnapshotNode,
|
||||
)
|
||||
from dbt.events.types import PartialParsingEnabled, PartialParsingFile
|
||||
from dbt.node_types import NodeType
|
||||
from dbt_common.context import get_invocation_context
|
||||
@@ -58,6 +64,7 @@ special_override_macros = [
|
||||
"generate_schema_name",
|
||||
"generate_database_name",
|
||||
"generate_alias_name",
|
||||
"function",
|
||||
]
|
||||
|
||||
|
||||
@@ -295,6 +302,10 @@ class PartialParsing:
|
||||
if saved_source_file.parse_file_type == ParseFileType.Fixture:
|
||||
self.delete_fixture_node(saved_source_file)
|
||||
|
||||
# functions
|
||||
if saved_source_file.parse_file_type == ParseFileType.Function:
|
||||
self.delete_function_node(saved_source_file)
|
||||
|
||||
fire_event(PartialParsingFile(operation="deleted", file_id=file_id))
|
||||
|
||||
# Updates for non-schema files
|
||||
@@ -310,6 +321,8 @@ class PartialParsing:
|
||||
self.update_doc_in_saved(new_source_file, old_source_file)
|
||||
elif new_source_file.parse_file_type == ParseFileType.Fixture:
|
||||
self.update_fixture_in_saved(new_source_file, old_source_file)
|
||||
elif new_source_file.parse_file_type == ParseFileType.Function:
|
||||
self.update_function_in_saved(new_source_file, old_source_file)
|
||||
else:
|
||||
raise Exception(f"Invalid parse_file_type in source_file {file_id}")
|
||||
fire_event(PartialParsingFile(operation="updated", file_id=file_id))
|
||||
@@ -405,6 +418,15 @@ class PartialParsing:
|
||||
self.saved_files[new_source_file.file_id] = deepcopy(new_source_file)
|
||||
self.add_to_pp_files(new_source_file)
|
||||
|
||||
def update_function_in_saved(
|
||||
self, new_source_file: SourceFile, old_source_file: SourceFile
|
||||
) -> None:
|
||||
if self.already_scheduled_for_parsing(old_source_file):
|
||||
return
|
||||
self.delete_function_node(old_source_file)
|
||||
self.saved_files[new_source_file.file_id] = deepcopy(new_source_file)
|
||||
self.add_to_pp_files(new_source_file)
|
||||
|
||||
def remove_mssat_file(self, source_file: AnySourceFile):
|
||||
# nodes [unique_ids] -- SQL files
|
||||
# There should always be a node for a SQL file
|
||||
@@ -630,6 +652,42 @@ class PartialParsing:
|
||||
source_file.unit_tests.remove(unique_id)
|
||||
self.saved_manifest.files.pop(source_file.file_id)
|
||||
|
||||
def delete_function_node(self, source_file: SourceFile) -> None:
|
||||
# There should always be a node for a Function file
|
||||
if not isinstance(source_file, SourceFile) or not source_file.functions:
|
||||
return
|
||||
|
||||
# There can only be one node of a function
|
||||
function_unique_id = source_file.functions[0]
|
||||
|
||||
# Remove the function node from the saved manifest
|
||||
function_node = self.saved_manifest.functions.pop(function_unique_id)
|
||||
|
||||
# Remove the function node from the source file so that it's not viewed as a
|
||||
# duplicate when it's re-added
|
||||
source_file.functions.remove(function_unique_id)
|
||||
|
||||
# If this function had a schema patch, schedule that schema element to be reapplied.
|
||||
patch_path = function_node.patch_path
|
||||
if (
|
||||
patch_path is not None
|
||||
and patch_path in self.saved_files
|
||||
and patch_path not in self.file_diff["deleted_schema_files"]
|
||||
):
|
||||
schema_file = self.saved_files[patch_path]
|
||||
# Only proceed if this is a schema file
|
||||
if isinstance(schema_file, SchemaSourceFile):
|
||||
elements = schema_file.dict_from_yaml.get("functions", [])
|
||||
schema_element = self.get_schema_element(elements, function_node.name)
|
||||
if schema_element:
|
||||
# Remove any previous links and re-merge the patch to pp_dict so it gets reparsed
|
||||
self.delete_schema_function(schema_file, schema_element)
|
||||
self.merge_patch(schema_file, "functions", schema_element)
|
||||
|
||||
# Finally, remove the deleted function file from saved files
|
||||
if source_file.file_id in self.saved_manifest.files:
|
||||
self.saved_manifest.files.pop(source_file.file_id)
|
||||
|
||||
# Schema files -----------------------
|
||||
# Changed schema files
|
||||
def change_schema_file(self, file_id):
|
||||
@@ -744,6 +802,7 @@ class PartialParsing:
|
||||
handle_change("unit_tests", self.delete_schema_unit_test)
|
||||
handle_change("saved_queries", self.delete_schema_saved_query)
|
||||
handle_change("data_tests", self.delete_schema_data_test_patch)
|
||||
handle_change("functions", self.delete_schema_function)
|
||||
|
||||
def _handle_element_change(
|
||||
self, schema_file, saved_yaml_dict, new_yaml_dict, env_var_changes, dict_key: str, delete
|
||||
@@ -917,7 +976,7 @@ class PartialParsing:
|
||||
for child_id in self.saved_manifest.child_map[unique_id]:
|
||||
if child_id.startswith("test") and child_id in self.saved_manifest.nodes:
|
||||
child_test = self.saved_manifest.nodes[child_id]
|
||||
if child_test.attached_node:
|
||||
if isinstance(child_test, GenericTestNode) and child_test.attached_node:
|
||||
if child_test.attached_node in self.saved_manifest.nodes:
|
||||
attached_node = self.saved_manifest.nodes[child_test.attached_node]
|
||||
self.update_in_saved(attached_node.file_id)
|
||||
@@ -1080,6 +1139,24 @@ class PartialParsing:
|
||||
schema_file.unit_tests.remove(unique_id)
|
||||
# No disabled unit tests yet
|
||||
|
||||
def delete_schema_function(self, schema_file: SchemaSourceFile, function_dict: dict) -> None:
|
||||
function_name = function_dict["name"]
|
||||
functions = schema_file.node_patches.copy()
|
||||
for unique_id in functions:
|
||||
if unique_id in self.saved_manifest.functions:
|
||||
function = self.saved_manifest.functions[unique_id]
|
||||
if function.name == function_name:
|
||||
removed_function = self.saved_manifest.functions.pop(unique_id)
|
||||
# For schema patches, recorded unique_ids live in node_patches (ndp)
|
||||
if unique_id in schema_file.node_patches:
|
||||
schema_file.node_patches.remove(unique_id)
|
||||
# Schedule the function's SQL file for reparsing so the node is re-added
|
||||
file_id = removed_function.file_id
|
||||
if file_id and file_id in self.new_files:
|
||||
self.saved_files[file_id] = deepcopy(self.new_files[file_id])
|
||||
if file_id and file_id in self.saved_files:
|
||||
self.add_to_pp_files(self.saved_files[file_id])
|
||||
|
||||
def get_schema_element(self, elem_list, elem_name):
|
||||
for element in elem_list:
|
||||
if "name" in element and element["name"] == elem_name:
|
||||
|
||||
@@ -10,7 +10,7 @@ from dbt import utils
|
||||
from dbt.artifacts.resources import ModelConfig, UnitTestConfig, UnitTestFormat
|
||||
from dbt.config import RuntimeConfig
|
||||
from dbt.context.context_config import ContextConfig
|
||||
from dbt.context.providers import generate_parse_exposure, get_rendered
|
||||
from dbt.context.providers import generate_parser_unit_test_context, get_rendered
|
||||
from dbt.contracts.files import FileHash, SchemaSourceFile
|
||||
from dbt.contracts.graph.manifest import Manifest
|
||||
from dbt.contracts.graph.model_config import UnitTestNodeConfig
|
||||
@@ -100,12 +100,7 @@ class UnitTestManifestLoader:
|
||||
overrides=test_case.overrides,
|
||||
)
|
||||
|
||||
ctx = generate_parse_exposure(
|
||||
unit_test_node, # type: ignore
|
||||
self.root_project,
|
||||
self.manifest,
|
||||
test_case.package_name,
|
||||
)
|
||||
ctx = generate_parser_unit_test_context(unit_test_node, self.root_project, self.manifest)
|
||||
get_rendered(unit_test_node.raw_code, ctx, unit_test_node, capture_macros=True)
|
||||
# unit_test_node now has a populated refs/sources
|
||||
|
||||
@@ -173,6 +168,10 @@ class UnitTestManifestLoader:
|
||||
**common_fields,
|
||||
source_name=original_input_node.source_name, # needed for source lookup
|
||||
)
|
||||
# In the case of multiple sources with the same name, we add the source schema name to the unique id.
|
||||
# This additionally prevents duplicate CTE names during compilation.
|
||||
input_node.unique_id = f"model.{original_input_node.package_name}.{original_input_node.source_name}__{input_name}"
|
||||
|
||||
# Sources need to go in the sources dictionary in order to create the right lookup
|
||||
self.unit_test_manifest.sources[input_node.unique_id] = input_node # type: ignore
|
||||
|
||||
@@ -291,8 +290,11 @@ class UnitTestParser(YamlReader):
|
||||
)
|
||||
|
||||
if tested_model_node:
|
||||
unit_test_definition.depends_on.nodes.append(tested_model_node.unique_id)
|
||||
unit_test_definition.schema = tested_model_node.schema
|
||||
if tested_model_node.config.enabled:
|
||||
unit_test_definition.depends_on.nodes.append(tested_model_node.unique_id)
|
||||
unit_test_definition.schema = tested_model_node.schema
|
||||
else:
|
||||
unit_test_definition.config.enabled = False
|
||||
|
||||
# Check that format and type of rows matches for each given input,
|
||||
# convert rows to a list of dictionaries, and add the unique_id of
|
||||
@@ -303,7 +305,7 @@ class UnitTestParser(YamlReader):
|
||||
# for calculating state:modified
|
||||
unit_test_definition.build_unit_test_checksum()
|
||||
assert isinstance(self.yaml.file, SchemaSourceFile)
|
||||
if unit_test_config.enabled:
|
||||
if unit_test_definition.config.enabled:
|
||||
self.manifest.add_unit_test(self.yaml.file, unit_test_definition)
|
||||
else:
|
||||
self.manifest.add_disabled(self.yaml.file, unit_test_definition)
|
||||
@@ -493,6 +495,13 @@ def find_tested_model_node(
|
||||
model_version = model_name_split[1] if len(model_name_split) == 2 else None
|
||||
|
||||
tested_node = manifest.ref_lookup.find(model_name, current_project, model_version, manifest)
|
||||
if not tested_node:
|
||||
disabled_node = manifest.disabled_lookup.find(
|
||||
model_name, current_project, model_version, [NodeType.Model]
|
||||
)
|
||||
if disabled_node:
|
||||
tested_node = disabled_node[0]
|
||||
|
||||
return tested_node
|
||||
|
||||
|
||||
@@ -510,22 +519,36 @@ def process_models_for_unit_test(
|
||||
f"Unable to find model '{current_project}.{unit_test_def.model}' for "
|
||||
f"unit test '{unit_test_def.name}' in {unit_test_def.original_file_path}"
|
||||
)
|
||||
unit_test_def.depends_on.nodes.append(tested_node.unique_id)
|
||||
unit_test_def.schema = tested_node.schema
|
||||
if tested_node.config.enabled:
|
||||
unit_test_def.depends_on.nodes.append(tested_node.unique_id)
|
||||
unit_test_def.schema = tested_node.schema
|
||||
else:
|
||||
# If the model is disabled, the unit test should be disabled
|
||||
unit_test_def.config.enabled = False
|
||||
|
||||
# The UnitTestDefinition should only have one "depends_on" at this point,
|
||||
# the one that's found by the "model" field.
|
||||
target_model_id = unit_test_def.depends_on.nodes[0]
|
||||
if target_model_id not in manifest.nodes:
|
||||
if target_model_id in manifest.disabled:
|
||||
# The model is disabled, so we don't need to do anything (#10540)
|
||||
return
|
||||
# If the model is disabled, the unit test should be disabled
|
||||
unit_test_def.config.enabled = False
|
||||
else:
|
||||
# If we've reached here and the model is not disabled, throw an error
|
||||
raise ParsingError(
|
||||
f"Unit test '{unit_test_def.name}' references a model that does not exist: {target_model_id}"
|
||||
)
|
||||
|
||||
if not unit_test_def.config.enabled:
|
||||
# Ensure the unit test is disabled in the manifest
|
||||
if unit_test_def.unique_id in manifest.unit_tests:
|
||||
manifest.unit_tests.pop(unit_test_def.unique_id)
|
||||
if unit_test_def.unique_id not in manifest.disabled:
|
||||
manifest.add_disabled(manifest.files[unit_test_def.file_id], unit_test_def)
|
||||
|
||||
# The unit test is disabled, so we don't need to do any further processing (#10540)
|
||||
return
|
||||
|
||||
target_model = manifest.nodes[target_model_id]
|
||||
assert isinstance(target_model, ModelNode)
|
||||
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
from typing import Dict, List, Optional, Set, Type
|
||||
from typing import Dict, Iterable, List, Optional, Set, Type
|
||||
|
||||
from dbt.adapters.base import BaseRelation
|
||||
from dbt.artifacts.schemas.results import NodeStatus
|
||||
from dbt.artifacts.schemas.run import RunResult
|
||||
from dbt.cli.flags import Flags
|
||||
@@ -64,6 +65,22 @@ class BuildTask(RunTask):
|
||||
resource_types.remove(NodeType.Unit)
|
||||
return list(resource_types)
|
||||
|
||||
def get_model_schemas(self, adapter, selected_uids: Iterable[str]) -> Set[BaseRelation]:
|
||||
|
||||
# Get model schemas as usual
|
||||
model_schemas = super().get_model_schemas(adapter, selected_uids)
|
||||
|
||||
# Get function schemas
|
||||
function_schemas: Set[BaseRelation] = set()
|
||||
for function in (
|
||||
self.manifest.functions.values() if self.manifest else []
|
||||
): # functionally the manifest will never be None as we do an assert in super().get_model_schemas(...)
|
||||
if function.unique_id in selected_uids:
|
||||
relation = adapter.Relation.create_from(self.config, function)
|
||||
function_schemas.add(relation.without_identifier())
|
||||
|
||||
return model_schemas.union(function_schemas)
|
||||
|
||||
# overrides get_graph_queue in runnable.py
|
||||
def get_graph_queue(self) -> GraphQueue:
|
||||
# Following uses self.selection_arg and self.exclusion_arg
|
||||
@@ -72,12 +89,14 @@ class BuildTask(RunTask):
|
||||
# selector including unit tests
|
||||
full_selector = self.get_node_selector(no_unit_tests=False)
|
||||
# selected node unique_ids with unit_tests
|
||||
full_selected_nodes = full_selector.get_selected(spec)
|
||||
full_selected_nodes = full_selector.get_selected(spec=spec, warn_on_no_nodes=False)
|
||||
|
||||
# This selector removes the unit_tests from the selector
|
||||
selector_wo_unit_tests = self.get_node_selector(no_unit_tests=True)
|
||||
# selected node unique_ids without unit_tests
|
||||
selected_nodes_wo_unit_tests = selector_wo_unit_tests.get_selected(spec)
|
||||
selected_nodes_wo_unit_tests = selector_wo_unit_tests.get_selected(
|
||||
spec=spec, warn_on_no_nodes=False
|
||||
)
|
||||
|
||||
# Get the difference in the sets of nodes with and without unit tests and
|
||||
# save it
|
||||
|
||||
@@ -119,23 +119,97 @@ class DepsTask(BaseTask):
|
||||
)
|
||||
|
||||
def check_for_duplicate_packages(self, packages_yml):
|
||||
"""Loop through contents of `packages.yml` to ensure no duplicate package names + versions.
|
||||
"""Loop through contents of `packages.yml` to remove entries that match the package being added.
|
||||
|
||||
This duplicate check will take into consideration exact match of a package name, as well as
|
||||
a check to see if a package name exists within a name (i.e. a package name inside a git URL).
|
||||
This method is called only during `dbt deps --add-package` to check if the package
|
||||
being added already exists in packages.yml. It uses substring matching to identify
|
||||
duplicates, which means it will match across different package sources. For example,
|
||||
adding a hub package "dbt-labs/dbt_utils" will remove an existing git package
|
||||
"https://github.com/dbt-labs/dbt-utils.git" since both contain "dbt_utils" or "dbt-utils".
|
||||
|
||||
The matching is flexible to handle both underscore and hyphen variants of package names,
|
||||
as git repos often use hyphens (dbt-utils) while package names use underscores (dbt_utils).
|
||||
Word boundaries (/, .) are enforced to prevent false matches like "dbt-core" matching
|
||||
"dbt-core-utils".
|
||||
|
||||
Args:
|
||||
packages_yml (dict): In-memory read of `packages.yml` contents
|
||||
|
||||
Returns:
|
||||
dict: Updated or untouched packages_yml contents
|
||||
dict: Updated packages_yml contents with matching packages removed
|
||||
"""
|
||||
for i, pkg_entry in enumerate(packages_yml["packages"]):
|
||||
for val in pkg_entry.values():
|
||||
if self.args.add_package["name"] in val:
|
||||
del packages_yml["packages"][i]
|
||||
# Extract the package name for matching
|
||||
package_name = self.args.add_package["name"]
|
||||
|
||||
fire_event(DepsFoundDuplicatePackage(removed_package=pkg_entry))
|
||||
# Create variants for flexible matching (handle _ vs -)
|
||||
# Check multiple variants to handle naming inconsistencies between hub and git
|
||||
package_name_parts = [
|
||||
package_name, # Original: "dbt-labs/dbt_utils"
|
||||
package_name.replace("_", "-"), # Hyphens: "dbt-labs/dbt-utils"
|
||||
package_name.replace("-", "_"), # Underscores: "dbt_labs/dbt_utils"
|
||||
]
|
||||
# Extract just the package name without org (after last /)
|
||||
if "/" in package_name:
|
||||
short_name = package_name.split("/")[-1]
|
||||
package_name_parts.extend(
|
||||
[
|
||||
short_name, # "dbt_utils"
|
||||
short_name.replace("_", "-"), # "dbt-utils"
|
||||
short_name.replace("-", "_"), # "dbt_utils" (deduplicated)
|
||||
]
|
||||
)
|
||||
|
||||
# Remove duplicates from package_name_parts
|
||||
package_name_parts = list(set(package_name_parts))
|
||||
|
||||
# Iterate backwards to safely delete items without index shifting issues
|
||||
for i in range(len(packages_yml["packages"]) - 1, -1, -1):
|
||||
pkg_entry = packages_yml["packages"][i]
|
||||
|
||||
# Get the package identifier key (package type determines which key exists)
|
||||
# This avoids iterating over non-string values like warn-unpinned: false
|
||||
package_identifier = (
|
||||
pkg_entry.get("package") # hub/registry package
|
||||
or pkg_entry.get("git") # git package
|
||||
or pkg_entry.get("local") # local package
|
||||
or pkg_entry.get("tarball") # tarball package
|
||||
or pkg_entry.get("private") # private package
|
||||
)
|
||||
|
||||
# Check if any variant of the package name appears in the identifier
|
||||
# Use word boundaries to avoid false matches (e.g., "dbt-core" shouldn't match "dbt-core-utils")
|
||||
# Word boundaries are: start/end of string, /, or .
|
||||
# Note: - and _ are NOT boundaries since they're used within compound package names
|
||||
if package_identifier:
|
||||
is_duplicate = False
|
||||
for name_variant in package_name_parts:
|
||||
if name_variant in package_identifier:
|
||||
# Found a match, now verify it's not a substring of a larger word
|
||||
# Check characters before and after the match
|
||||
idx = package_identifier.find(name_variant)
|
||||
start_ok = idx == 0 or package_identifier[idx - 1] in "/."
|
||||
end_idx = idx + len(name_variant)
|
||||
end_ok = (
|
||||
end_idx == len(package_identifier)
|
||||
or package_identifier[end_idx] in "/."
|
||||
)
|
||||
|
||||
if start_ok and end_ok:
|
||||
is_duplicate = True
|
||||
break
|
||||
|
||||
if is_duplicate:
|
||||
del packages_yml["packages"][i]
|
||||
# Filter out non-string values (like warn-unpinned boolean) before logging
|
||||
# Note: Check for bool first since bool is a subclass of int in Python
|
||||
loggable_package = {
|
||||
k: v
|
||||
for k, v in pkg_entry.items()
|
||||
if not isinstance(v, bool)
|
||||
and isinstance(v, (str, int, float))
|
||||
and k != "unrendered"
|
||||
}
|
||||
fire_event(DepsFoundDuplicatePackage(removed_package=loggable_package))
|
||||
|
||||
return packages_yml
|
||||
|
||||
|
||||
@@ -126,6 +126,7 @@ class RetryTask(ConfiguredTask):
|
||||
result.unique_id
|
||||
for result in self.previous_results.results
|
||||
if result.status in RETRYABLE_STATUSES
|
||||
# Avoid retrying operation nodes unless we are retrying the run-operation command
|
||||
and not (
|
||||
self.previous_command_name != "run-operation"
|
||||
and result.unique_id.startswith("operation.")
|
||||
@@ -150,6 +151,11 @@ class RetryTask(ConfiguredTask):
|
||||
)
|
||||
}
|
||||
|
||||
# Tasks without get_graph_queue (e.g. run-operation) and no failed nodes to retry.
|
||||
if not unique_ids and not hasattr(self.task_class, "get_graph_queue"):
|
||||
# Return early with the previous results as the past invocation was successful
|
||||
return self.previous_results
|
||||
|
||||
class TaskWrapper(self.task_class):
|
||||
def get_graph_queue(self):
|
||||
new_graph = self.graph.get_subset_graph(unique_ids)
|
||||
|
||||
@@ -249,34 +249,17 @@ class GraphRunnableTask(ConfiguredTask):
|
||||
thread_exception = e
|
||||
finally:
|
||||
if result is not None:
|
||||
fire_event(
|
||||
NodeFinished(
|
||||
node_info=runner.node.node_info,
|
||||
run_result=result.to_msg_dict(),
|
||||
try:
|
||||
fire_event(
|
||||
NodeFinished(
|
||||
node_info=runner.node.node_info,
|
||||
run_result=result.to_msg_dict(),
|
||||
)
|
||||
)
|
||||
)
|
||||
except Exception as e:
|
||||
result = self._handle_thread_exception(runner, e)
|
||||
else:
|
||||
msg = f"Exception on worker thread. {thread_exception}"
|
||||
|
||||
fire_event(
|
||||
GenericExceptionOnRun(
|
||||
unique_id=runner.node.unique_id,
|
||||
exc=str(thread_exception),
|
||||
node_info=runner.node.node_info,
|
||||
)
|
||||
)
|
||||
|
||||
result = RunResult(
|
||||
status=RunStatus.Error, # type: ignore
|
||||
timing=[],
|
||||
thread_id="",
|
||||
execution_time=0.0,
|
||||
adapter_response={},
|
||||
message=msg,
|
||||
failures=None,
|
||||
batch_results=None,
|
||||
node=runner.node,
|
||||
)
|
||||
result = self._handle_thread_exception(runner, thread_exception)
|
||||
|
||||
# `_event_status` dict is only used for logging. Make sure
|
||||
# it gets deleted when we're done with it
|
||||
@@ -365,6 +348,32 @@ class GraphRunnableTask(ConfiguredTask):
|
||||
args = [runner]
|
||||
self._submit(pool, args, callback)
|
||||
|
||||
def _handle_thread_exception(
|
||||
self,
|
||||
runner: BaseRunner,
|
||||
thread_exception: Optional[Union[KeyboardInterrupt, SystemExit, Exception]],
|
||||
) -> RunResult:
|
||||
msg = f"Exception on worker thread. {thread_exception}"
|
||||
fire_event(
|
||||
GenericExceptionOnRun(
|
||||
unique_id=runner.node.unique_id,
|
||||
exc=str(thread_exception),
|
||||
node_info=runner.node.node_info,
|
||||
)
|
||||
)
|
||||
|
||||
return RunResult(
|
||||
status=RunStatus.Error, # type: ignore
|
||||
timing=[],
|
||||
thread_id="",
|
||||
execution_time=0.0,
|
||||
adapter_response={},
|
||||
message=msg,
|
||||
failures=None,
|
||||
batch_results=None,
|
||||
node=runner.node,
|
||||
)
|
||||
|
||||
def _handle_result(self, result: RunResult) -> None:
|
||||
"""Mark the result as completed, insert the `CompileResultNode` into
|
||||
the manifest, and mark any descendants (potentially with a 'cause' if
|
||||
|
||||
@@ -3,14 +3,13 @@ import importlib
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
from importlib import metadata as importlib_metadata
|
||||
from pathlib import Path
|
||||
from typing import Iterator, List, Optional, Tuple
|
||||
|
||||
import requests
|
||||
|
||||
import dbt_common.semver as semver
|
||||
from dbt.__version__ import version as __version_string
|
||||
from dbt_common.ui import green, yellow
|
||||
|
||||
PYPI_VERSION_URL = "https://pypi.org/pypi/dbt-core/json"
|
||||
@@ -233,16 +232,8 @@ def _resolve_version() -> str:
|
||||
try:
|
||||
return importlib_metadata.version("dbt-core")
|
||||
except importlib_metadata.PackageNotFoundError:
|
||||
pyproject_path = Path(__file__).resolve().parents[1] / "pyproject.toml"
|
||||
if not pyproject_path.exists():
|
||||
raise RuntimeError("Unable to locate pyproject.toml to determine dbt-core version")
|
||||
|
||||
text = pyproject_path.read_text(encoding="utf-8")
|
||||
match = re.search(r'^version\s*=\s*"(?P<version>[^"]+)"', text, re.MULTILINE)
|
||||
if match:
|
||||
return match.group("version")
|
||||
|
||||
raise RuntimeError("Unable to determine dbt-core version from pyproject.toml")
|
||||
# When running from source (not installed), use version from __version__.py
|
||||
return __version_string
|
||||
|
||||
|
||||
__version__ = _resolve_version()
|
||||
|
||||
206
core/hatch.toml
Normal file
206
core/hatch.toml
Normal file
@@ -0,0 +1,206 @@
|
||||
[version]
|
||||
path = "dbt/__version__.py"
|
||||
|
||||
[build.targets.wheel]
|
||||
packages = ["dbt"]
|
||||
only-packages = true
|
||||
exclude = [
|
||||
"**/*.md",
|
||||
]
|
||||
artifacts = [
|
||||
"dbt/include/**/*.py",
|
||||
"dbt/include/**/*.sql",
|
||||
"dbt/include/**/*.yml",
|
||||
"dbt/include/**/*.html",
|
||||
"dbt/include/**/*.md",
|
||||
"dbt/include/**/.gitkeep",
|
||||
"dbt/include/**/.gitignore",
|
||||
"dbt/task/docs/**/*.html",
|
||||
"dbt/jsonschemas/**/*.json",
|
||||
"dbt/py.typed",
|
||||
# Directories without __init__.py (namespace packages)
|
||||
"dbt/artifacts/resources/v1/**/*.py",
|
||||
"dbt/artifacts/utils/**/*.py",
|
||||
"dbt/event_time/**/*.py",
|
||||
"dbt/docs/source/**/*.py",
|
||||
"dbt/tests/util.py",
|
||||
]
|
||||
|
||||
[build.targets.sdist]
|
||||
include = [
|
||||
"/dbt",
|
||||
"/README.md",
|
||||
]
|
||||
|
||||
[build.targets.sdist.force-include]
|
||||
"dbt/task/docs/index.html" = "dbt/task/docs/index.html"
|
||||
|
||||
[envs.default]
|
||||
# Python 3.10-3.11 required locally due to flake8==4.0.1 compatibility
|
||||
# CI uses [envs.ci] which doesn't set python, allowing matrix testing
|
||||
python = "3.11"
|
||||
dependencies = [
|
||||
# Git dependencies for development against main branches
|
||||
"dbt-adapters @ git+https://github.com/dbt-labs/dbt-adapters.git@main#subdirectory=dbt-adapters",
|
||||
"dbt-tests-adapter @ git+https://github.com/dbt-labs/dbt-adapters.git@main#subdirectory=dbt-tests-adapter",
|
||||
"dbt-common @ git+https://github.com/dbt-labs/dbt-common.git@main",
|
||||
"dbt-postgres @ git+https://github.com/dbt-labs/dbt-adapters.git@main#subdirectory=dbt-postgres",
|
||||
# Code quality
|
||||
"pre-commit~=3.7.0",
|
||||
"black>=24.3,<25.0",
|
||||
"flake8==4.0.1", # requires python <3.12
|
||||
"mypy==1.4.1", # update requires code fixes
|
||||
"isort==5.13.2",
|
||||
# Testing
|
||||
"pytest>=7.0,<8.0",
|
||||
"pytest-xdist~=3.6",
|
||||
"pytest-csv~=3.0",
|
||||
"pytest-cov",
|
||||
"pytest-dotenv",
|
||||
"pytest-mock",
|
||||
"pytest-split",
|
||||
"pytest-logbook~=1.2",
|
||||
"logbook<1.9",
|
||||
"flaky",
|
||||
"freezegun>=1.5.1",
|
||||
"hypothesis",
|
||||
"mocker",
|
||||
# Debugging
|
||||
"ipdb",
|
||||
"ddtrace==2.21.3",
|
||||
# Documentation
|
||||
"docutils",
|
||||
"sphinx",
|
||||
# Type stubs
|
||||
"types-docutils",
|
||||
"types-PyYAML",
|
||||
"types-Jinja2",
|
||||
"types-jsonschema",
|
||||
"types-mock",
|
||||
"types-protobuf>=5.0,<6.0",
|
||||
"types-python-dateutil",
|
||||
"types-pytz",
|
||||
"types-requests",
|
||||
"types-setuptools",
|
||||
# Other
|
||||
"pip-tools",
|
||||
"protobuf>=6.0,<7.0",
|
||||
]
|
||||
|
||||
[envs.default.scripts]
|
||||
# Setup commands
|
||||
setup = [
|
||||
"pip install -e .",
|
||||
"pre-commit install",
|
||||
]
|
||||
|
||||
# Code quality commands
|
||||
code-quality = "pre-commit run --all-files --show-diff-on-failure"
|
||||
lint = [
|
||||
"pre-commit run flake8-check --hook-stage manual --all-files",
|
||||
"pre-commit run mypy-check --hook-stage manual --all-files",
|
||||
]
|
||||
flake8 = "pre-commit run flake8-check --hook-stage manual --all-files"
|
||||
mypy = "pre-commit run mypy-check --hook-stage manual --all-files"
|
||||
black = "pre-commit run black-check --hook-stage manual --all-files"
|
||||
|
||||
# Testing commands
|
||||
unit-tests = "python -m pytest {args} ../tests/unit"
|
||||
integration-tests = "python -m pytest -nauto {args} ../tests/functional"
|
||||
integration-tests-fail-fast = "python -m pytest -x -nauto {args} ../tests/functional"
|
||||
test = [
|
||||
"python -m pytest ../tests/unit",
|
||||
"pre-commit run black-check --hook-stage manual --all-files",
|
||||
"pre-commit run flake8-check --hook-stage manual --all-files",
|
||||
"pre-commit run mypy-check --hook-stage manual --all-files",
|
||||
]
|
||||
|
||||
# Database setup
|
||||
setup-db = [
|
||||
"docker compose up -d database",
|
||||
"bash ../scripts/setup_db.sh",
|
||||
]
|
||||
|
||||
# Utility commands
|
||||
clean = [
|
||||
"rm -f .coverage",
|
||||
"rm -f .coverage.*",
|
||||
"rm -rf .eggs/",
|
||||
"rm -rf build/",
|
||||
"rm -rf dbt.egg-info/",
|
||||
"rm -f dbt_project.yml",
|
||||
"rm -rf dist/",
|
||||
"find . -type f -name '*.pyc' -delete",
|
||||
"find . -type d -name __pycache__ -exec rm -rf {} +",
|
||||
]
|
||||
json-schema = "python ../scripts/collect-artifact-schema.py --path ../schemas"
|
||||
|
||||
[envs.build]
|
||||
python = "3.11"
|
||||
detached = true
|
||||
dependencies = [
|
||||
"wheel",
|
||||
"twine",
|
||||
"check-wheel-contents",
|
||||
]
|
||||
|
||||
[envs.build.scripts]
|
||||
check-all = [
|
||||
"- check-wheel",
|
||||
"- check-sdist",
|
||||
]
|
||||
check-wheel = [
|
||||
"twine check dist/*",
|
||||
"find ./dist/dbt_core-*.whl -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/",
|
||||
"pip freeze | grep dbt-core",
|
||||
"dbt --version",
|
||||
]
|
||||
check-sdist = [
|
||||
"check-wheel-contents dist/*.whl --ignore W007,W008",
|
||||
"find ./dist/dbt_core-*.gz -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/",
|
||||
"pip freeze | grep dbt-core",
|
||||
"dbt --version",
|
||||
]
|
||||
|
||||
# CI environment - isolated environment with test dependencies
|
||||
[envs.ci]
|
||||
dependencies = [
|
||||
# Git dependencies for development against main branches
|
||||
"dbt-adapters @ git+https://github.com/dbt-labs/dbt-adapters.git@main#subdirectory=dbt-adapters",
|
||||
"dbt-tests-adapter @ git+https://github.com/dbt-labs/dbt-adapters.git@main#subdirectory=dbt-tests-adapter",
|
||||
"dbt-common @ git+https://github.com/dbt-labs/dbt-common.git@main",
|
||||
"dbt-postgres @ git+https://github.com/dbt-labs/dbt-adapters.git@main#subdirectory=dbt-postgres",
|
||||
# Testing
|
||||
"pytest>=7.0,<8.0",
|
||||
"pytest-cov",
|
||||
"pytest-xdist~=3.6",
|
||||
"pytest-csv~=3.0",
|
||||
"pytest-dotenv",
|
||||
"pytest-mock",
|
||||
"pytest-split",
|
||||
"ddtrace==2.21.3",
|
||||
"flaky",
|
||||
"freezegun>=1.5.1",
|
||||
"hypothesis",
|
||||
]
|
||||
|
||||
pre-install-commands = [
|
||||
"pip install -e .",
|
||||
]
|
||||
|
||||
[envs.ci.env-vars]
|
||||
DBT_TEST_USER_1 = "dbt_test_user_1"
|
||||
DBT_TEST_USER_2 = "dbt_test_user_2"
|
||||
DBT_TEST_USER_3 = "dbt_test_user_3"
|
||||
|
||||
[envs.ci.scripts]
|
||||
unit-tests = "python -m pytest --cov=dbt --cov-report=xml {args} ../tests/unit"
|
||||
# Run as single command to avoid pre-install-commands running twice
|
||||
integration-tests = """
|
||||
python -m pytest --cov=dbt --cov-append --cov-report=xml {args} ../tests/functional -k "not tests/functional/graph_selection" && \
|
||||
python -m pytest --cov=dbt --cov-append --cov-report=xml {args} ../tests/functional/graph_selection
|
||||
"""
|
||||
|
||||
# Note: Python version matrix is handled by GitHub Actions CI, not hatch.
|
||||
# This avoids running tests 4x per job. The CI sets up the Python version
|
||||
# and hatch uses whatever Python is active.
|
||||
@@ -1,38 +1,12 @@
|
||||
[tool.setuptools]
|
||||
package-dir = {"" = "."}
|
||||
include-package-data = true
|
||||
zip-safe = false
|
||||
|
||||
[tool.setuptools.packages.find]
|
||||
where = ["."]
|
||||
include = [
|
||||
"dbt",
|
||||
"dbt.*",
|
||||
]
|
||||
|
||||
# this needs to match MANIFEST.in for the wheels
|
||||
[tool.setuptools.package-data]
|
||||
"dbt" = [
|
||||
"include/**/*.py",
|
||||
"include/**/*.sql",
|
||||
"include/**/*.yml",
|
||||
"include/**/*.html",
|
||||
"include/**/*.md",
|
||||
"include/**/.gitkeep",
|
||||
"include/**/.gitignore",
|
||||
"task/docs/**/*.html",
|
||||
"jsonschemas/**/*.json",
|
||||
"py.typed",
|
||||
]
|
||||
|
||||
[project]
|
||||
name = "dbt-core"
|
||||
version = "1.11.0b4"
|
||||
dynamic = ["version"]
|
||||
description = "With dbt, data analysts and engineers can build analytics the way engineers build applications."
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.10"
|
||||
license = "Apache-2.0"
|
||||
license-files = ["License.md"] # License.md copied to core/ by build script even though it lives at the root by convention
|
||||
license-files = { globs = ["LICENSE"] }
|
||||
keywords = []
|
||||
authors = [
|
||||
{ name = "dbt Labs", email = "info@dbtlabs.com" },
|
||||
@@ -80,9 +54,9 @@ dependencies = [
|
||||
"dbt-extractor>=0.5.0,<=0.6",
|
||||
"dbt-semantic-interfaces>=0.9.0,<0.10",
|
||||
# Minor versions for these are expected to be backwards-compatible
|
||||
"dbt-common>=1.27.0,<2.0",
|
||||
"dbt-common>=1.37.2,<2.0",
|
||||
"dbt-adapters>=1.15.5,<2.0",
|
||||
"dbt-protos>=1.0.375,<2.0",
|
||||
"dbt-protos>=1.0.405,<2.0",
|
||||
"pydantic<3",
|
||||
# ----
|
||||
# Expect compatibility with all new versions of these packages, so lower bounds only.
|
||||
@@ -102,9 +76,10 @@ Changelog = "https://github.com/dbt-labs/dbt-core/blob/main/CHANGELOG.md"
|
||||
[project.scripts]
|
||||
dbt = "dbt.cli.main:cli"
|
||||
|
||||
[tool.hatch.version]
|
||||
path = "dbt/__version__.py"
|
||||
|
||||
|
||||
[build-system]
|
||||
requires = [
|
||||
"setuptools>=61",
|
||||
"wheel",
|
||||
]
|
||||
build-backend = "setuptools.build_meta"
|
||||
requires = ["hatchling"]
|
||||
build-backend = "hatchling.build"
|
||||
|
||||
@@ -1,26 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
"""Legacy setuptools shim retained for compatibility with existing workflows. Will be removed in a future version."""
|
||||
|
||||
from setuptools import setup
|
||||
|
||||
# the user has a downlevel version of setuptools.
|
||||
# ----
|
||||
# dbt-core uses these packages deeply, throughout the codebase, and there have been breaking changes in past patch releases (even though these are major-version-one).
|
||||
# Pin to the patch or minor version, and bump in each new minor version of dbt-core.
|
||||
# ----
|
||||
# dbt-core uses these packages in standard ways. Pin to the major version, and check compatibility
|
||||
# with major versions in each new minor version of dbt-core.
|
||||
# ----
|
||||
# These packages are major-version-0. Keep upper bounds on upcoming minor versions (which could have breaking changes)
|
||||
# and check compatibility / bump in each new minor version of dbt-core.
|
||||
# ----
|
||||
# These are major-version-0 packages also maintained by dbt-labs.
|
||||
# Accept patches but avoid automatically updating past a set minor version range.
|
||||
# Minor versions for these are expected to be backwards-compatible
|
||||
# ----
|
||||
# Expect compatibility with all new versions of these packages, so lower bounds only.
|
||||
# ----
|
||||
|
||||
if __name__ == "__main__":
|
||||
setup()
|
||||
@@ -1,38 +0,0 @@
|
||||
git+https://github.com/dbt-labs/dbt-adapters.git@main#subdirectory=dbt-adapters
|
||||
git+https://github.com/dbt-labs/dbt-adapters.git@main#subdirectory=dbt-tests-adapter
|
||||
git+https://github.com/dbt-labs/dbt-common.git@main
|
||||
git+https://github.com/dbt-labs/dbt-adapters.git@main#subdirectory=dbt-postgres
|
||||
black==24.3.0
|
||||
bumpversion
|
||||
ddtrace==2.21.3
|
||||
docutils
|
||||
flake8==4.0.1
|
||||
flaky
|
||||
freezegun>=1.5.1
|
||||
hypothesis
|
||||
ipdb
|
||||
isort==5.13.2
|
||||
mypy==1.4.1
|
||||
pip-tools
|
||||
pre-commit
|
||||
protobuf>=6.0,<7.0
|
||||
pytest>=7.4,<8.0
|
||||
pytest-cov
|
||||
pytest-csv>=3.0,<4.0
|
||||
pytest-dotenv
|
||||
pytest-mock
|
||||
pytest-split
|
||||
pytest-xdist
|
||||
sphinx
|
||||
tox>=3.13
|
||||
types-docutils
|
||||
types-PyYAML
|
||||
types-Jinja2
|
||||
types-jsonschema
|
||||
types-mock
|
||||
types-protobuf>=5.0,<6.0
|
||||
types-python-dateutil
|
||||
types-pytz
|
||||
types-requests
|
||||
types-setuptools
|
||||
mocker
|
||||
@@ -23,7 +23,7 @@ services:
|
||||
# Run `make .env` to set $USER_ID and $GROUP_ID
|
||||
USER_ID: ${USER_ID:-}
|
||||
GROUP_ID: ${GROUP_ID:-}
|
||||
command: "/root/.virtualenvs/dbt/bin/pytest"
|
||||
command: "bash -c 'cd core && hatch run ci:unit-tests'"
|
||||
environment:
|
||||
POSTGRES_TEST_HOST: "database"
|
||||
volumes:
|
||||
|
||||
@@ -1 +0,0 @@
|
||||
-e ./core
|
||||
@@ -17,15 +17,11 @@ rm -rf "$DBT_PATH"/core/build
|
||||
|
||||
mkdir -p "$DBT_PATH"/dist
|
||||
|
||||
# Copy License.md to core/ for inclusion in distribution (required by Apache 2.0)
|
||||
# The license-files in pyproject.toml references it relative to core/
|
||||
cp "$DBT_PATH"/License.md "$DBT_PATH"/core/License.md
|
||||
|
||||
cd "$DBT_PATH"/core
|
||||
$PYTHON_BIN -m pip install --upgrade build
|
||||
$PYTHON_BIN -m build --outdir "$DBT_PATH/dist"
|
||||
$PYTHON_BIN -m pip install --upgrade hatch
|
||||
hatch build --clean
|
||||
|
||||
# Clean up License.md that was copied to core/ for build
|
||||
rm -f "$DBT_PATH/core/License.md"
|
||||
# Move built distributions to top-level dist/
|
||||
mv "$DBT_PATH"/core/dist/* "$DBT_PATH"/dist/
|
||||
|
||||
set +x
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user