mirror of
https://github.com/dbt-labs/dbt-core
synced 2025-12-23 04:51:28 +00:00
Compare commits
19 Commits
jerco/gran
...
test-docs-
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
734b6429c7 | ||
|
|
a75b2c0a90 | ||
|
|
6b6ae22434 | ||
|
|
287f443ec9 | ||
|
|
aea2c4a29b | ||
|
|
21ffe31270 | ||
|
|
70c9074625 | ||
|
|
2048a1af6f | ||
|
|
71223dc253 | ||
|
|
e03d35a9fc | ||
|
|
f988f76fcc | ||
|
|
0cacfd0f88 | ||
|
|
c25260e5dd | ||
|
|
c521fa6b74 | ||
|
|
f304b4b2da | ||
|
|
064d890172 | ||
|
|
9fca33cb29 | ||
|
|
6360247d39 | ||
|
|
f0fbb0e551 |
@@ -1,5 +1,5 @@
|
|||||||
[bumpversion]
|
[bumpversion]
|
||||||
current_version = 1.2.0b1
|
current_version = 1.3.0a1
|
||||||
parse = (?P<major>\d+)
|
parse = (?P<major>\d+)
|
||||||
\.(?P<minor>\d+)
|
\.(?P<minor>\d+)
|
||||||
\.(?P<patch>\d+)
|
\.(?P<patch>\d+)
|
||||||
|
|||||||
@@ -2,6 +2,8 @@
|
|||||||
|
|
||||||
For information on prior major and minor releases, see their changelogs:
|
For information on prior major and minor releases, see their changelogs:
|
||||||
|
|
||||||
|
|
||||||
|
* [1.2](https://github.com/dbt-labs/dbt-core/blob/1.2.latest/CHANGELOG.md)
|
||||||
* [1.1](https://github.com/dbt-labs/dbt-core/blob/1.1.latest/CHANGELOG.md)
|
* [1.1](https://github.com/dbt-labs/dbt-core/blob/1.1.latest/CHANGELOG.md)
|
||||||
* [1.0](https://github.com/dbt-labs/dbt-core/blob/1.0.latest/CHANGELOG.md)
|
* [1.0](https://github.com/dbt-labs/dbt-core/blob/1.0.latest/CHANGELOG.md)
|
||||||
* [0.21](https://github.com/dbt-labs/dbt-core/blob/0.21.latest/CHANGELOG.md)
|
* [0.21](https://github.com/dbt-labs/dbt-core/blob/0.21.latest/CHANGELOG.md)
|
||||||
|
|||||||
@@ -1,69 +0,0 @@
|
|||||||
## dbt-core 1.2.0-b1 - June 24, 2022
|
|
||||||
### Features
|
|
||||||
- Add selector method when reading selector definitions ([#4821](https://github.com/dbt-labs/dbt-core/issues/4821), [#4827](https://github.com/dbt-labs/dbt-core/pull/4827))
|
|
||||||
- Add set and zip function to contexts ([#2345](https://github.com/dbt-labs/dbt-core/issues/2345), [#5107](https://github.com/dbt-labs/dbt-core/pull/5107))
|
|
||||||
- Adds itertools to modules Jinja namespace ([#5130](https://github.com/dbt-labs/dbt-core/issues/5130), [#5140](https://github.com/dbt-labs/dbt-core/pull/5140))
|
|
||||||
- allow target as an option in profile_template.yml ([#5179](https://github.com/dbt-labs/dbt-core/issues/5179), [#5184](https://github.com/dbt-labs/dbt-core/pull/5184))
|
|
||||||
- seed: Add new macro get_csv_sql ([#5206](https://github.com/dbt-labs/dbt-core/issues/5206), [#5207](https://github.com/dbt-labs/dbt-core/pull/5207))
|
|
||||||
- Grants as Node Configs ([#5189](https://github.com/dbt-labs/dbt-core/issues/5189), [#5230](https://github.com/dbt-labs/dbt-core/pull/5230))
|
|
||||||
- Adds file selectors and support for file selectors in the default method selector ([#5240](https://github.com/dbt-labs/dbt-core/issues/5240), [#5241](https://github.com/dbt-labs/dbt-core/pull/5241))
|
|
||||||
- Move cross-db macros from dbt-utils into dbt-core global project ([#4813](https://github.com/dbt-labs/dbt-core/issues/4813), [#5265](https://github.com/dbt-labs/dbt-core/pull/5265))
|
|
||||||
- Prettify duration message at the end of execution ([#5253](https://github.com/dbt-labs/dbt-core/issues/5253), [#5364](https://github.com/dbt-labs/dbt-core/pull/5364))
|
|
||||||
- Early return from dbt init if no available adapters ([#5365](https://github.com/dbt-labs/dbt-core/issues/5365), [#5366](https://github.com/dbt-labs/dbt-core/pull/5366))
|
|
||||||
### Fixes
|
|
||||||
- Adding new cols to check_cols in snapshots ([#3146](https://github.com/dbt-labs/dbt-core/issues/3146), [#4893](https://github.com/dbt-labs/dbt-core/pull/4893))
|
|
||||||
- Truncate relation names when appending a suffix that will result in len > 63 characters using make_temp_relation and make_backup_relation macros ([#2869](https://github.com/dbt-labs/dbt-core/issues/2869), [#4921](https://github.com/dbt-labs/dbt-core/pull/4921))
|
|
||||||
- Restore ability to utilize `updated_at` for check_cols snapshots ([#5076](https://github.com/dbt-labs/dbt-core/issues/5076), [#5077](https://github.com/dbt-labs/dbt-core/pull/5077))
|
|
||||||
- Use yaml renderer (with target context) for rendering selectors ([#5131](https://github.com/dbt-labs/dbt-core/issues/5131), [#5136](https://github.com/dbt-labs/dbt-core/pull/5136))
|
|
||||||
- Fix retry logic to return values after initial try ([#5023](https://github.com/dbt-labs/dbt-core/issues/5023), [#5137](https://github.com/dbt-labs/dbt-core/pull/5137))
|
|
||||||
- Scrub secret env vars from CommandError in exception stacktrace ([#5151](https://github.com/dbt-labs/dbt-core/issues/5151), [#5152](https://github.com/dbt-labs/dbt-core/pull/5152))
|
|
||||||
- Ensure the metric name does not contain spaces ([#4572](https://github.com/dbt-labs/dbt-core/issues/4572), [#5173](https://github.com/dbt-labs/dbt-core/pull/5173))
|
|
||||||
- When parsing 'all_sources' should be a list of unique dirs ([#5120](https://github.com/dbt-labs/dbt-core/issues/5120), [#5176](https://github.com/dbt-labs/dbt-core/pull/5176))
|
|
||||||
- Add warning if yaml contains duplicate keys ([#5114](https://github.com/dbt-labs/dbt-core/issues/5114), [#5146](https://github.com/dbt-labs/dbt-core/pull/5146))
|
|
||||||
- Modifying the drop_test_schema to work better with Redshift issues around locked tables and current transactions ([#5200](https://github.com/dbt-labs/dbt-core/issues/5200), [#5198](https://github.com/dbt-labs/dbt-core/pull/5198))
|
|
||||||
- Fix column comparison in snapshot_check_all_get_existing_columns for check-strategy snapshots with explicit check_cols defined ([#5222](https://github.com/dbt-labs/dbt-core/issues/5222), [#5223](https://github.com/dbt-labs/dbt-core/pull/5223))
|
|
||||||
- Changed how `--select state:modified` detects changes for macros nodes depend on ([#5202](https://github.com/dbt-labs/dbt-core/issues/5202), [#5224](https://github.com/dbt-labs/dbt-core/pull/5224))
|
|
||||||
- Fix column comparison in snapshot_check_all_get_existing_columns to use adapter.get_columns_in_relation ([#5222](https://github.com/dbt-labs/dbt-core/issues/5222), [#5232](https://github.com/dbt-labs/dbt-core/pull/5232))
|
|
||||||
- Remove docs file from manifest when removing doc node ([#4146](https://github.com/dbt-labs/dbt-core/issues/4146), [#5270](https://github.com/dbt-labs/dbt-core/pull/5270))
|
|
||||||
- Remove duplicate dbt script entry ([#5314](https://github.com/dbt-labs/dbt-core/issues/5314), [#5304](https://github.com/dbt-labs/dbt-core/pull/5304))
|
|
||||||
- Change node ancestor/descendant algo, fixes issue where downstream models aren't run when using networkx >= 2.8.1 ([#5286](https://github.com/dbt-labs/dbt-core/issues/5286), [#5326](https://github.com/dbt-labs/dbt-core/pull/5326))
|
|
||||||
- Fixing Windows color regression ([#5191](https://github.com/dbt-labs/dbt-core/issues/5191), [#5327](https://github.com/dbt-labs/dbt-core/pull/5327))
|
|
||||||
- Define compatibility for older manifest versions when using state: selection methods ([#5213](https://github.com/dbt-labs/dbt-core/issues/5213), [#5346](https://github.com/dbt-labs/dbt-core/pull/5346))
|
|
||||||
- Remove duplicate key checking introduced in 1.2.0a1 ([#5331](https://github.com/dbt-labs/dbt-core/issues/5331), [#5403](https://github.com/dbt-labs/dbt-core/pull/5403))
|
|
||||||
### Under the Hood
|
|
||||||
- Migrating 005_simple_seed to the new test framework. ([#200](https://github.com/dbt-labs/dbt-core/issues/200), [#5013](https://github.com/dbt-labs/dbt-core/pull/5013))
|
|
||||||
- Convert 029_docs_generate tests to new framework ([#5035](https://github.com/dbt-labs/dbt-core/issues/5035), [#5058](https://github.com/dbt-labs/dbt-core/pull/5058))
|
|
||||||
- Move package deprecation check outside of package cache ([#5068](https://github.com/dbt-labs/dbt-core/issues/5068), [#5069](https://github.com/dbt-labs/dbt-core/pull/5069))
|
|
||||||
- removal of scaffold first attempt and create_adapter_plugin.py as they are deprecated new scaffold can be found https://github.com/dbt-labs/dbt-database-adapter-scaffold ([#4980](https://github.com/dbt-labs/dbt-core/issues/4980), [#5117](https://github.com/dbt-labs/dbt-core/pull/5117))
|
|
||||||
- Mypy -> 0.942 + fixed import logic to allow for full mypy coverage ([#4805](https://github.com/dbt-labs/dbt-core/issues/4805), [#5171](https://github.com/dbt-labs/dbt-core/pull/5171))
|
|
||||||
- Converted dbt list tests to pytest ([#5049](https://github.com/dbt-labs/dbt-core/issues/5049), [#5178](https://github.com/dbt-labs/dbt-core/pull/5178))
|
|
||||||
- Fix: Call str and repr for UnsetProfileConfig without a RuntimeException ([#5081](https://github.com/dbt-labs/dbt-core/issues/5081), [#5209](https://github.com/dbt-labs/dbt-core/pull/5209))
|
|
||||||
- Improve tracking error logging message ([#5197](https://github.com/dbt-labs/dbt-core/issues/5197), [#5211](https://github.com/dbt-labs/dbt-core/pull/5211))
|
|
||||||
- Clean up materialization logic: more consistent relation names, loading from cache ([#2869](https://github.com/dbt-labs/dbt-core/issues/2869), [#4921](https://github.com/dbt-labs/dbt-core/pull/4921))
|
|
||||||
- Use the default Python version for local dev and test instead of requiring Python 3.8 ([#5257](https://github.com/dbt-labs/dbt-core/issues/5257), [#5269](https://github.com/dbt-labs/dbt-core/pull/5269))
|
|
||||||
- Fix test for context set function ([#5266](https://github.com/dbt-labs/dbt-core/issues/5266), [#5272](https://github.com/dbt-labs/dbt-core/pull/5272))
|
|
||||||
- Fix pip upgrade step in CI for Windows ([#5321](https://github.com/dbt-labs/dbt-core/issues/5321), [#5320](https://github.com/dbt-labs/dbt-core/pull/5320))
|
|
||||||
- Fix unit test test_graph_selection ([#5323](https://github.com/dbt-labs/dbt-core/issues/5323), [#5324](https://github.com/dbt-labs/dbt-core/pull/5324))
|
|
||||||
- Update context readme + clean up context code" ([#4796](https://github.com/dbt-labs/dbt-core/issues/4796), [#5334](https://github.com/dbt-labs/dbt-core/pull/5334))
|
|
||||||
- removed script meant for snowflake to snowflake ([#5361](https://github.com/dbt-labs/dbt-core/issues/5361), [#5362](https://github.com/dbt-labs/dbt-core/pull/5362))
|
|
||||||
### Dependencies
|
|
||||||
- Bump ubuntu from 20.04 to 22.04 ([#4904](https://github.com/dbt-labs/dbt-core/issues/4904), [#5141](https://github.com/dbt-labs/dbt-core/pull/5141))
|
|
||||||
- Bumping hologram version ([#5219](https://github.com/dbt-labs/dbt-core/issues/5219), [#5218](https://github.com/dbt-labs/dbt-core/pull/5218))
|
|
||||||
- Bump python from 3.10.3-slim-bullseye to 3.10.5-slim-bullseye in /docker ([#4904](https://github.com/dbt-labs/dbt-core/issues/4904), [#5367](https://github.com/dbt-labs/dbt-core/pull/5367))
|
|
||||||
|
|
||||||
### Contributors
|
|
||||||
- [@GtheSheep](https://github.com/GtheSheep) ([#4893](https://github.com/dbt-labs/dbt-core/pull/4893))
|
|
||||||
- [@NicolasPA](https://github.com/NicolasPA) ([#5211](https://github.com/dbt-labs/dbt-core/pull/5211))
|
|
||||||
- [@adamantike](https://github.com/adamantike) ([#5207](https://github.com/dbt-labs/dbt-core/pull/5207))
|
|
||||||
- [@alexrosenfeld10](https://github.com/alexrosenfeld10) ([#5184](https://github.com/dbt-labs/dbt-core/pull/5184))
|
|
||||||
- [@bd3dowling](https://github.com/bd3dowling) ([#5140](https://github.com/dbt-labs/dbt-core/pull/5140))
|
|
||||||
- [@danieldiamond](https://github.com/danieldiamond) ([#4827](https://github.com/dbt-labs/dbt-core/pull/4827))
|
|
||||||
- [@dbeatty10](https://github.com/dbeatty10) ([#5265](https://github.com/dbt-labs/dbt-core/pull/5265), [#5077](https://github.com/dbt-labs/dbt-core/pull/5077))
|
|
||||||
- [@dependabot[bot]](https://github.com/dependabot[bot]) ([#5141](https://github.com/dbt-labs/dbt-core/pull/5141), [#5367](https://github.com/dbt-labs/dbt-core/pull/5367))
|
|
||||||
- [@epapineau](https://github.com/epapineau) ([#4921](https://github.com/dbt-labs/dbt-core/pull/4921))
|
|
||||||
- [@groodt](https://github.com/groodt) ([#5304](https://github.com/dbt-labs/dbt-core/pull/5304))
|
|
||||||
- [@jared-rimmer](https://github.com/jared-rimmer) ([#5364](https://github.com/dbt-labs/dbt-core/pull/5364))
|
|
||||||
- [@jeremyyeo](https://github.com/jeremyyeo) ([#5107](https://github.com/dbt-labs/dbt-core/pull/5107), [#5146](https://github.com/dbt-labs/dbt-core/pull/5146), [#5403](https://github.com/dbt-labs/dbt-core/pull/5403))
|
|
||||||
- [@jwills](https://github.com/jwills) ([#5241](https://github.com/dbt-labs/dbt-core/pull/5241), [#5269](https://github.com/dbt-labs/dbt-core/pull/5269))
|
|
||||||
- [@tomasfarias](https://github.com/tomasfarias) ([#5209](https://github.com/dbt-labs/dbt-core/pull/5209))
|
|
||||||
- [@ulisesojeda](https://github.com/ulisesojeda) ([#5366](https://github.com/dbt-labs/dbt-core/pull/5366))
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
kind: Dependencies
|
|
||||||
body: "Bump ubuntu from 20.04 to 22.04"
|
|
||||||
time: 2022-04-27T19:51:28.000000-05:00
|
|
||||||
custom:
|
|
||||||
Author: dependabot[bot]
|
|
||||||
Issue: "4904"
|
|
||||||
PR: "5141"
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
kind: Dependencies
|
|
||||||
body: "Bumping hologram version"
|
|
||||||
time: 2022-05-06T16:09:07.000000-05:00
|
|
||||||
custom:
|
|
||||||
Author: leahwicz
|
|
||||||
Issue: "5219"
|
|
||||||
PR: "5218"
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
kind: Dependencies
|
|
||||||
body: "Bump python from 3.10.3-slim-bullseye to 3.10.5-slim-bullseye in /docker"
|
|
||||||
time: 2022-06-13T00:14:56.000000-05:00
|
|
||||||
custom:
|
|
||||||
Author: dependabot[bot]
|
|
||||||
Issue: "4904"
|
|
||||||
PR: "5367"
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
kind: Features
|
|
||||||
body: Add selector method when reading selector definitions
|
|
||||||
time: 2022-04-08T11:26:10.713088+10:00
|
|
||||||
custom:
|
|
||||||
Author: danieldiamond
|
|
||||||
Issue: "4821"
|
|
||||||
PR: "4827"
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
kind: Features
|
|
||||||
body: Add set and zip function to contexts
|
|
||||||
time: 2022-04-23T23:17:56.851793+12:00
|
|
||||||
custom:
|
|
||||||
Author: jeremyyeo
|
|
||||||
Issue: "2345"
|
|
||||||
PR: "5107"
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
kind: Features
|
|
||||||
body: Adds itertools to modules Jinja namespace
|
|
||||||
time: 2022-04-24T13:26:55.008246+01:00
|
|
||||||
custom:
|
|
||||||
Author: bd3dowling
|
|
||||||
Issue: "5130"
|
|
||||||
PR: "5140"
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
kind: Features
|
|
||||||
body: allow target as an option in profile_template.yml
|
|
||||||
time: 2022-04-28T06:56:44.511519-04:00
|
|
||||||
custom:
|
|
||||||
Author: alexrosenfeld10
|
|
||||||
Issue: "5179"
|
|
||||||
PR: "5184"
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
kind: Features
|
|
||||||
body: 'seed: Add new macro get_csv_sql'
|
|
||||||
time: 2022-05-03T14:29:34.847959075Z
|
|
||||||
custom:
|
|
||||||
Author: adamantike
|
|
||||||
Issue: "5206"
|
|
||||||
PR: "5207"
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
kind: Features
|
|
||||||
body: Grants as Node Configs
|
|
||||||
time: 2022-05-10T20:49:49.197999-04:00
|
|
||||||
custom:
|
|
||||||
Author: gshank
|
|
||||||
Issue: "5189"
|
|
||||||
PR: "5230"
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
kind: Features
|
|
||||||
body: Adds file selectors and support for file selectors in the default method selector
|
|
||||||
time: 2022-05-12T21:57:48.289674-07:00
|
|
||||||
custom:
|
|
||||||
Author: jwills
|
|
||||||
Issue: "5240"
|
|
||||||
PR: "5241"
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
kind: Features
|
|
||||||
body: Move cross-db macros from dbt-utils into dbt-core global project
|
|
||||||
time: 2022-05-18T11:46:04.557104+02:00
|
|
||||||
custom:
|
|
||||||
Author: jtcohen6 dbeatty10
|
|
||||||
Issue: "4813"
|
|
||||||
PR: "5265"
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
kind: Features
|
|
||||||
body: Prettify duration message at the end of execution
|
|
||||||
time: 2022-06-11T16:39:31.725960083+01:00
|
|
||||||
custom:
|
|
||||||
Author: jared-rimmer
|
|
||||||
Issue: "5253"
|
|
||||||
PR: "5364"
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
kind: Features
|
|
||||||
body: Early return from dbt init if no available adapters
|
|
||||||
time: 2022-06-14T08:20:51.096872718+02:00
|
|
||||||
custom:
|
|
||||||
Author: ulisesojeda
|
|
||||||
Issue: "5365"
|
|
||||||
PR: "5366"
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
kind: Fixes
|
|
||||||
body: Adding new cols to check_cols in snapshots
|
|
||||||
time: 2022-03-17T21:09:16.977086+01:00
|
|
||||||
custom:
|
|
||||||
Author: GtheSheep
|
|
||||||
Issue: "3146"
|
|
||||||
PR: "4893"
|
|
||||||
@@ -1,8 +0,0 @@
|
|||||||
kind: Fixes
|
|
||||||
body: Truncate relation names when appending a suffix that will result in len > 63
|
|
||||||
characters using make_temp_relation and make_backup_relation macros
|
|
||||||
time: 2022-03-22T17:37:53.320082-07:00
|
|
||||||
custom:
|
|
||||||
Author: epapineau
|
|
||||||
Issue: "2869"
|
|
||||||
PR: "4921"
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
kind: Fixes
|
|
||||||
body: Restore ability to utilize `updated_at` for check_cols snapshots
|
|
||||||
time: 2022-04-15T11:29:27.063462-06:00
|
|
||||||
custom:
|
|
||||||
Author: dbeatty10
|
|
||||||
Issue: "5076"
|
|
||||||
PR: "5077"
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
kind: Fixes
|
|
||||||
body: Fix retry logic to return values after initial try
|
|
||||||
time: 2022-04-22T13:12:27.239055-05:00
|
|
||||||
custom:
|
|
||||||
Author: emmyoop
|
|
||||||
Issue: "5023"
|
|
||||||
PR: "5137"
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
kind: Fixes
|
|
||||||
body: Use yaml renderer (with target context) for rendering selectors
|
|
||||||
time: 2022-04-22T13:56:45.147893-04:00
|
|
||||||
custom:
|
|
||||||
Author: gshank
|
|
||||||
Issue: "5131"
|
|
||||||
PR: "5136"
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
kind: Fixes
|
|
||||||
body: Scrub secret env vars from CommandError in exception stacktrace
|
|
||||||
time: 2022-04-25T20:39:24.365495+02:00
|
|
||||||
custom:
|
|
||||||
Author: jtcohen6
|
|
||||||
Issue: "5151"
|
|
||||||
PR: "5152"
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
kind: Fixes
|
|
||||||
body: Ensure the metric name does not contain spaces
|
|
||||||
time: 2022-04-26T20:21:04.360693-04:00
|
|
||||||
custom:
|
|
||||||
Author: gshank
|
|
||||||
Issue: "4572"
|
|
||||||
PR: "5173"
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
kind: Fixes
|
|
||||||
body: When parsing 'all_sources' should be a list of unique dirs
|
|
||||||
time: 2022-04-27T10:26:48.648388-04:00
|
|
||||||
custom:
|
|
||||||
Author: gshank
|
|
||||||
Issue: "5120"
|
|
||||||
PR: "5176"
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
kind: Fixes
|
|
||||||
body: Add warning if yaml contains duplicate keys
|
|
||||||
time: 2022-04-28T10:01:57.893956+12:00
|
|
||||||
custom:
|
|
||||||
Author: jeremyyeo
|
|
||||||
Issue: "5114"
|
|
||||||
PR: "5146"
|
|
||||||
@@ -1,8 +0,0 @@
|
|||||||
kind: Fixes
|
|
||||||
body: Modifying the drop_test_schema to work better with Redshift issues around locked
|
|
||||||
tables and current transactions
|
|
||||||
time: 2022-04-29T16:07:42.750046-05:00
|
|
||||||
custom:
|
|
||||||
Author: Mcknight-42
|
|
||||||
Issue: "5200"
|
|
||||||
PR: "5198"
|
|
||||||
@@ -1,8 +0,0 @@
|
|||||||
kind: Fixes
|
|
||||||
body: Fix column comparison in snapshot_check_all_get_existing_columns for check-strategy
|
|
||||||
snapshots with explicit check_cols defined
|
|
||||||
time: 2022-05-09T13:00:21.649028+02:00
|
|
||||||
custom:
|
|
||||||
Author: jtcohen6
|
|
||||||
Issue: "5222"
|
|
||||||
PR: "5223"
|
|
||||||
@@ -1,8 +0,0 @@
|
|||||||
kind: Fixes
|
|
||||||
body: Changed how `--select state:modified` detects changes for macros nodes depend
|
|
||||||
on
|
|
||||||
time: 2022-05-09T13:13:12.889074-05:00
|
|
||||||
custom:
|
|
||||||
Author: stu-k
|
|
||||||
Issue: "5202"
|
|
||||||
PR: "5224"
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
kind: Fixes
|
|
||||||
body: Fix column comparison in snapshot_check_all_get_existing_columns to use adapter.get_columns_in_relation
|
|
||||||
time: 2022-05-11T12:32:38.313321+02:00
|
|
||||||
custom:
|
|
||||||
Author: jtcohen6
|
|
||||||
Issue: "5222"
|
|
||||||
PR: "5232"
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
kind: Fixes
|
|
||||||
body: Remove docs file from manifest when removing doc node
|
|
||||||
time: 2022-05-18T13:46:10.167143-04:00
|
|
||||||
custom:
|
|
||||||
Author: gshank
|
|
||||||
Issue: "4146"
|
|
||||||
PR: "5270"
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
kind: Fixes
|
|
||||||
body: Remove duplicate dbt script entry
|
|
||||||
time: 2022-06-01T08:13:14.067001+10:00
|
|
||||||
custom:
|
|
||||||
Author: groodt
|
|
||||||
Issue: "5314"
|
|
||||||
PR: "5304"
|
|
||||||
@@ -1,8 +0,0 @@
|
|||||||
kind: Fixes
|
|
||||||
body: Change node ancestor/descendant algo, fixes issue where downstream models aren't
|
|
||||||
run when using networkx >= 2.8.1
|
|
||||||
time: 2022-06-01T13:59:08.886215-05:00
|
|
||||||
custom:
|
|
||||||
Author: iknox-fa
|
|
||||||
Issue: "5286"
|
|
||||||
PR: "5326"
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
kind: Fixes
|
|
||||||
body: Fixing Windows color regression
|
|
||||||
time: 2022-06-01T19:42:34.263009-04:00
|
|
||||||
custom:
|
|
||||||
Author: leahwicz
|
|
||||||
Issue: "5191"
|
|
||||||
PR: "5327"
|
|
||||||
@@ -1,8 +0,0 @@
|
|||||||
kind: Fixes
|
|
||||||
body: 'Define compatibility for older manifest versions when using state: selection
|
|
||||||
methods'
|
|
||||||
time: 2022-06-08T08:09:14.321735+02:00
|
|
||||||
custom:
|
|
||||||
Author: jtcohen6
|
|
||||||
Issue: "5213"
|
|
||||||
PR: "5346"
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
kind: Fixes
|
|
||||||
body: Remove duplicate key checking introduced in 1.2.0a1
|
|
||||||
time: 2022-06-23T08:24:31.900647+12:00
|
|
||||||
custom:
|
|
||||||
Author: jeremyyeo
|
|
||||||
Issue: "5331"
|
|
||||||
PR: "5403"
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
kind: Under the Hood
|
|
||||||
body: Migrating 005_simple_seed to the new test framework.
|
|
||||||
time: 2022-04-09T04:05:39.20045-07:00
|
|
||||||
custom:
|
|
||||||
Author: versusfacit
|
|
||||||
Issue: "200"
|
|
||||||
PR: "5013"
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
kind: Under the Hood
|
|
||||||
body: Convert 029_docs_generate tests to new framework
|
|
||||||
time: 2022-04-13T18:30:14.706391-04:00
|
|
||||||
custom:
|
|
||||||
Author: gshank
|
|
||||||
Issue: "5035"
|
|
||||||
PR: "5058"
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
kind: Under the Hood
|
|
||||||
body: Move package deprecation check outside of package cache
|
|
||||||
time: 2022-04-14T13:22:06.157579-05:00
|
|
||||||
custom:
|
|
||||||
Author: emmyoop
|
|
||||||
Issue: "5068"
|
|
||||||
PR: "5069"
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
kind: Under the Hood
|
|
||||||
body: removal of scaffold first attempt and create_adapter_plugin.py as they are deprecated new scaffold can be found https://github.com/dbt-labs/dbt-database-adapter-scaffold
|
|
||||||
time: 2022-04-20T12:00:25.171923-05:00
|
|
||||||
custom:
|
|
||||||
Author: McKnight-42
|
|
||||||
Issue: "4980"
|
|
||||||
PR: "5117"
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
kind: Under the Hood
|
|
||||||
body: Mypy -> 0.942 + fixed import logic to allow for full mypy coverage
|
|
||||||
time: 2022-04-27T11:21:27.499359-05:00
|
|
||||||
custom:
|
|
||||||
Author: iknox-fa
|
|
||||||
Issue: "4805"
|
|
||||||
PR: "5171"
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
kind: Under the Hood
|
|
||||||
body: Converted dbt list tests to pytest
|
|
||||||
time: 2022-04-27T14:06:28.882908-05:00
|
|
||||||
custom:
|
|
||||||
Author: stu-k
|
|
||||||
Issue: "5049"
|
|
||||||
PR: "5178"
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
kind: Under the Hood
|
|
||||||
body: 'Fix: Call str and repr for UnsetProfileConfig without a RuntimeException'
|
|
||||||
time: 2022-05-03T19:52:12.793729384+02:00
|
|
||||||
custom:
|
|
||||||
Author: tomasfarias
|
|
||||||
Issue: "5081"
|
|
||||||
PR: "5209"
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
kind: Under the Hood
|
|
||||||
body: Improve tracking error logging message
|
|
||||||
time: 2022-05-04T01:00:31.60387036+02:00
|
|
||||||
custom:
|
|
||||||
Author: NicolasPA
|
|
||||||
Issue: "5197"
|
|
||||||
PR: "5211"
|
|
||||||
@@ -1,8 +0,0 @@
|
|||||||
kind: Under the Hood
|
|
||||||
body: 'Clean up materialization logic: more consistent relation names, loading from
|
|
||||||
cache'
|
|
||||||
time: 2022-05-09T09:26:28.551068+02:00
|
|
||||||
custom:
|
|
||||||
Author: jtcohen6
|
|
||||||
Issue: "2869"
|
|
||||||
PR: "4921"
|
|
||||||
@@ -1,8 +0,0 @@
|
|||||||
kind: Under the Hood
|
|
||||||
body: Use the default Python version for local dev and test instead of requiring Python
|
|
||||||
3.8
|
|
||||||
time: 2022-05-18T09:51:44.603193-07:00
|
|
||||||
custom:
|
|
||||||
Author: jwills
|
|
||||||
Issue: "5257"
|
|
||||||
PR: "5269"
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
kind: Under the Hood
|
|
||||||
body: Fix test for context set function
|
|
||||||
time: 2022-05-18T14:55:22.554316-04:00
|
|
||||||
custom:
|
|
||||||
Author: gshank
|
|
||||||
Issue: "5266"
|
|
||||||
PR: "5272"
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
kind: Under the Hood
|
|
||||||
body: Fix pip upgrade step in CI for Windows
|
|
||||||
time: 2022-06-01T10:52:45.872931-04:00
|
|
||||||
custom:
|
|
||||||
Author: gshank
|
|
||||||
Issue: "5321"
|
|
||||||
PR: "5320"
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
kind: Under the Hood
|
|
||||||
body: Fix unit test test_graph_selection
|
|
||||||
time: 2022-06-01T11:26:48.725831-04:00
|
|
||||||
custom:
|
|
||||||
Author: gshank
|
|
||||||
Issue: "5323"
|
|
||||||
PR: "5324"
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
kind: Under the Hood
|
|
||||||
body: Update context readme + clean up context code"
|
|
||||||
time: 2022-06-06T23:03:53.022568+02:00
|
|
||||||
custom:
|
|
||||||
Author: jtcohen6
|
|
||||||
Issue: "4796"
|
|
||||||
PR: "5334"
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
kind: Under the Hood
|
|
||||||
body: removed script meant for snowflake to snowflake
|
|
||||||
time: 2022-06-10T11:10:56.080236-05:00
|
|
||||||
custom:
|
|
||||||
Author: McKnight-42
|
|
||||||
Issue: "5361"
|
|
||||||
PR: "5362"
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
kind: Dependencies
|
|
||||||
body: "Bump mypy from 0.942 to 0.961"
|
|
||||||
time: 2022-06-07T00:09:25.000000-05:00
|
|
||||||
custom:
|
|
||||||
Author: dependabot[bot]
|
|
||||||
Issue: "4904"
|
|
||||||
PR: "5337"
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
kind: Dependencies
|
|
||||||
body: "Update colorama requirement from <0.4.5,>=0.3.9 to >=0.3.9,<0.4.6 in /core"
|
|
||||||
time: 2022-06-17T00:18:49.000000-05:00
|
|
||||||
custom:
|
|
||||||
Author: dependabot[bot]
|
|
||||||
Issue: "4904"
|
|
||||||
PR: "5388"
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
kind: Dependencies
|
|
||||||
body: "Bump black from 22.3.0 to 22.6.0"
|
|
||||||
time: 2022-06-29T00:08:31.000000-05:00
|
|
||||||
custom:
|
|
||||||
Author: dependabot[bot]
|
|
||||||
Issue: "4904"
|
|
||||||
PR: "5420"
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
kind: Features
|
|
||||||
body: Move type_* macros from dbt-utils into dbt-core, with tests
|
|
||||||
time: 2022-06-30T13:54:52.165139+02:00
|
|
||||||
custom:
|
|
||||||
Author: jtcohen6
|
|
||||||
Issue: "5317"
|
|
||||||
PR: "5428"
|
|
||||||
8
.changes/unreleased/Features-20220715-035555.yaml
Normal file
8
.changes/unreleased/Features-20220715-035555.yaml
Normal file
@@ -0,0 +1,8 @@
|
|||||||
|
kind: Features
|
||||||
|
body: Add reusable function for retrying adapter connections. Utilize said function
|
||||||
|
to add retries for Postgres (and Redshift).
|
||||||
|
time: 2022-07-15T03:55:55.270637265+02:00
|
||||||
|
custom:
|
||||||
|
Author: tomasfarias
|
||||||
|
Issue: "5022"
|
||||||
|
PR: "5432"
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
kind: Fixes
|
|
||||||
body: Add inheritance to materialization macro resolution
|
|
||||||
time: 2022-06-08T10:42:51.839945-04:00
|
|
||||||
custom:
|
|
||||||
Author: volkangurel
|
|
||||||
Issue: "4646"
|
|
||||||
PR: "5348"
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
kind: Fixes
|
|
||||||
body: Improve pluralizations for Documentation and SqlOperation NodeTypes
|
|
||||||
time: 2022-06-09T11:52:46.578469-05:00
|
|
||||||
custom:
|
|
||||||
Author: pdebelak
|
|
||||||
Issue: "5352"
|
|
||||||
PR: "5356"
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
kind: Fixes
|
|
||||||
body: Properly use quotes for Snowflake snapshots when checking all columns
|
|
||||||
time: 2022-06-17T11:44:43.978834+02:00
|
|
||||||
custom:
|
|
||||||
Author: pquadri
|
|
||||||
Issue: "2975"
|
|
||||||
PR: "5389"
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
kind: Fixes
|
|
||||||
body: fixes handling of RESET color code with USE_COLORS=False
|
|
||||||
time: 2022-06-17T16:00:27.038058-04:00
|
|
||||||
custom:
|
|
||||||
Author: darin-reify
|
|
||||||
Issue: "5288"
|
|
||||||
PR: "5394"
|
|
||||||
7
.changes/unreleased/Fixes-20220715-231148.yaml
Normal file
7
.changes/unreleased/Fixes-20220715-231148.yaml
Normal file
@@ -0,0 +1,7 @@
|
|||||||
|
kind: Fixes
|
||||||
|
body: Rename try to strict for more intuitiveness
|
||||||
|
time: 2022-07-15T23:11:48.327928+12:00
|
||||||
|
custom:
|
||||||
|
Author: jeremyyeo
|
||||||
|
Issue: "5475"
|
||||||
|
PR: "5477"
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
kind: Security
|
|
||||||
body: Move string interpolation of "secret" env vars outside of Jinja context. Update "contexts" README
|
|
||||||
time: 2022-06-06T23:03:53.022568+02:00
|
|
||||||
custom:
|
|
||||||
Author: jtcohen6
|
|
||||||
Issue: "4796"
|
|
||||||
PR: "5334"
|
|
||||||
@@ -1,8 +0,0 @@
|
|||||||
kind: Under the Hood
|
|
||||||
body: Added the suggested RegEx to check the SemVer string within a package dependency
|
|
||||||
and improved invalid version error handling.
|
|
||||||
time: 2022-06-13T17:07:03.773668-05:00
|
|
||||||
custom:
|
|
||||||
Author: fivetran-joemarkiewicz
|
|
||||||
Issue: "5201"
|
|
||||||
PR: "5370"
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
kind: Under the Hood
|
|
||||||
body: 'Add annotation to render_value method reimplemented in #5334'
|
|
||||||
time: 2022-06-16T12:05:16.474078+02:00
|
|
||||||
custom:
|
|
||||||
Author: jtcohen6
|
|
||||||
Issue: "4796"
|
|
||||||
PR: "5382"
|
|
||||||
6
.github/workflows/main.yml
vendored
6
.github/workflows/main.yml
vendored
@@ -38,6 +38,7 @@ jobs:
|
|||||||
name: code-quality
|
name: code-quality
|
||||||
|
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
|
timeout-minutes: 10
|
||||||
|
|
||||||
steps:
|
steps:
|
||||||
- name: Check out the repository
|
- name: Check out the repository
|
||||||
@@ -65,6 +66,7 @@ jobs:
|
|||||||
name: unit test / python ${{ matrix.python-version }}
|
name: unit test / python ${{ matrix.python-version }}
|
||||||
|
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
|
timeout-minutes: 10
|
||||||
|
|
||||||
strategy:
|
strategy:
|
||||||
fail-fast: false
|
fail-fast: false
|
||||||
@@ -109,6 +111,7 @@ jobs:
|
|||||||
name: integration test / python ${{ matrix.python-version }} / ${{ matrix.os }}
|
name: integration test / python ${{ matrix.python-version }} / ${{ matrix.os }}
|
||||||
|
|
||||||
runs-on: ${{ matrix.os }}
|
runs-on: ${{ matrix.os }}
|
||||||
|
timeout-minutes: 45
|
||||||
|
|
||||||
strategy:
|
strategy:
|
||||||
fail-fast: false
|
fail-fast: false
|
||||||
@@ -125,6 +128,9 @@ jobs:
|
|||||||
TOXENV: integration
|
TOXENV: integration
|
||||||
PYTEST_ADDOPTS: "-v --color=yes -n4 --csv integration_results.csv"
|
PYTEST_ADDOPTS: "-v --color=yes -n4 --csv integration_results.csv"
|
||||||
DBT_INVOCATION_ENV: github-actions
|
DBT_INVOCATION_ENV: github-actions
|
||||||
|
DBT_TEST_USER_1: dbt_test_user_1
|
||||||
|
DBT_TEST_USER_2: dbt_test_user_2
|
||||||
|
DBT_TEST_USER_3: dbt_test_user_3
|
||||||
|
|
||||||
steps:
|
steps:
|
||||||
- name: Check out the repository
|
- name: Check out the repository
|
||||||
|
|||||||
@@ -30,6 +30,11 @@ jobs:
|
|||||||
LOG_DIR: "/home/runner/work/dbt-core/dbt-core/logs"
|
LOG_DIR: "/home/runner/work/dbt-core/dbt-core/logs"
|
||||||
# tells integration tests to output into json format
|
# tells integration tests to output into json format
|
||||||
DBT_LOG_FORMAT: "json"
|
DBT_LOG_FORMAT: "json"
|
||||||
|
# Additional test users
|
||||||
|
DBT_TEST_USER_1: dbt_test_user_1
|
||||||
|
DBT_TEST_USER_2: dbt_test_user_2
|
||||||
|
DBT_TEST_USER_3: dbt_test_user_3
|
||||||
|
|
||||||
steps:
|
steps:
|
||||||
- name: checkout dev
|
- name: checkout dev
|
||||||
uses: actions/checkout@v2
|
uses: actions/checkout@v2
|
||||||
|
|||||||
73
CHANGELOG.md
73
CHANGELOG.md
@@ -6,81 +6,12 @@
|
|||||||
- Do not edit this file directly. This file is auto-generated using [changie](https://github.com/miniscruff/changie). For details on how to document a change, see [the contributing guide](https://github.com/dbt-labs/dbt-core/blob/main/CONTRIBUTING.md#adding-changelog-entry)
|
- Do not edit this file directly. This file is auto-generated using [changie](https://github.com/miniscruff/changie). For details on how to document a change, see [the contributing guide](https://github.com/dbt-labs/dbt-core/blob/main/CONTRIBUTING.md#adding-changelog-entry)
|
||||||
|
|
||||||
|
|
||||||
## dbt-core 1.2.0-b1 - June 24, 2022
|
|
||||||
### Features
|
|
||||||
- Add selector method when reading selector definitions ([#4821](https://github.com/dbt-labs/dbt-core/issues/4821), [#4827](https://github.com/dbt-labs/dbt-core/pull/4827))
|
|
||||||
- Add set and zip function to contexts ([#2345](https://github.com/dbt-labs/dbt-core/issues/2345), [#5107](https://github.com/dbt-labs/dbt-core/pull/5107))
|
|
||||||
- Adds itertools to modules Jinja namespace ([#5130](https://github.com/dbt-labs/dbt-core/issues/5130), [#5140](https://github.com/dbt-labs/dbt-core/pull/5140))
|
|
||||||
- allow target as an option in profile_template.yml ([#5179](https://github.com/dbt-labs/dbt-core/issues/5179), [#5184](https://github.com/dbt-labs/dbt-core/pull/5184))
|
|
||||||
- seed: Add new macro get_csv_sql ([#5206](https://github.com/dbt-labs/dbt-core/issues/5206), [#5207](https://github.com/dbt-labs/dbt-core/pull/5207))
|
|
||||||
- Grants as Node Configs ([#5189](https://github.com/dbt-labs/dbt-core/issues/5189), [#5230](https://github.com/dbt-labs/dbt-core/pull/5230))
|
|
||||||
- Adds file selectors and support for file selectors in the default method selector ([#5240](https://github.com/dbt-labs/dbt-core/issues/5240), [#5241](https://github.com/dbt-labs/dbt-core/pull/5241))
|
|
||||||
- Move cross-db macros from dbt-utils into dbt-core global project ([#4813](https://github.com/dbt-labs/dbt-core/issues/4813), [#5265](https://github.com/dbt-labs/dbt-core/pull/5265))
|
|
||||||
- Prettify duration message at the end of execution ([#5253](https://github.com/dbt-labs/dbt-core/issues/5253), [#5364](https://github.com/dbt-labs/dbt-core/pull/5364))
|
|
||||||
- Early return from dbt init if no available adapters ([#5365](https://github.com/dbt-labs/dbt-core/issues/5365), [#5366](https://github.com/dbt-labs/dbt-core/pull/5366))
|
|
||||||
### Fixes
|
|
||||||
- Adding new cols to check_cols in snapshots ([#3146](https://github.com/dbt-labs/dbt-core/issues/3146), [#4893](https://github.com/dbt-labs/dbt-core/pull/4893))
|
|
||||||
- Truncate relation names when appending a suffix that will result in len > 63 characters using make_temp_relation and make_backup_relation macros ([#2869](https://github.com/dbt-labs/dbt-core/issues/2869), [#4921](https://github.com/dbt-labs/dbt-core/pull/4921))
|
|
||||||
- Restore ability to utilize `updated_at` for check_cols snapshots ([#5076](https://github.com/dbt-labs/dbt-core/issues/5076), [#5077](https://github.com/dbt-labs/dbt-core/pull/5077))
|
|
||||||
- Use yaml renderer (with target context) for rendering selectors ([#5131](https://github.com/dbt-labs/dbt-core/issues/5131), [#5136](https://github.com/dbt-labs/dbt-core/pull/5136))
|
|
||||||
- Fix retry logic to return values after initial try ([#5023](https://github.com/dbt-labs/dbt-core/issues/5023), [#5137](https://github.com/dbt-labs/dbt-core/pull/5137))
|
|
||||||
- Scrub secret env vars from CommandError in exception stacktrace ([#5151](https://github.com/dbt-labs/dbt-core/issues/5151), [#5152](https://github.com/dbt-labs/dbt-core/pull/5152))
|
|
||||||
- Ensure the metric name does not contain spaces ([#4572](https://github.com/dbt-labs/dbt-core/issues/4572), [#5173](https://github.com/dbt-labs/dbt-core/pull/5173))
|
|
||||||
- When parsing 'all_sources' should be a list of unique dirs ([#5120](https://github.com/dbt-labs/dbt-core/issues/5120), [#5176](https://github.com/dbt-labs/dbt-core/pull/5176))
|
|
||||||
- Add warning if yaml contains duplicate keys ([#5114](https://github.com/dbt-labs/dbt-core/issues/5114), [#5146](https://github.com/dbt-labs/dbt-core/pull/5146))
|
|
||||||
- Modifying the drop_test_schema to work better with Redshift issues around locked tables and current transactions ([#5200](https://github.com/dbt-labs/dbt-core/issues/5200), [#5198](https://github.com/dbt-labs/dbt-core/pull/5198))
|
|
||||||
- Fix column comparison in snapshot_check_all_get_existing_columns for check-strategy snapshots with explicit check_cols defined ([#5222](https://github.com/dbt-labs/dbt-core/issues/5222), [#5223](https://github.com/dbt-labs/dbt-core/pull/5223))
|
|
||||||
- Changed how `--select state:modified` detects changes for macros nodes depend on ([#5202](https://github.com/dbt-labs/dbt-core/issues/5202), [#5224](https://github.com/dbt-labs/dbt-core/pull/5224))
|
|
||||||
- Fix column comparison in snapshot_check_all_get_existing_columns to use adapter.get_columns_in_relation ([#5222](https://github.com/dbt-labs/dbt-core/issues/5222), [#5232](https://github.com/dbt-labs/dbt-core/pull/5232))
|
|
||||||
- Remove docs file from manifest when removing doc node ([#4146](https://github.com/dbt-labs/dbt-core/issues/4146), [#5270](https://github.com/dbt-labs/dbt-core/pull/5270))
|
|
||||||
- Remove duplicate dbt script entry ([#5314](https://github.com/dbt-labs/dbt-core/issues/5314), [#5304](https://github.com/dbt-labs/dbt-core/pull/5304))
|
|
||||||
- Change node ancestor/descendant algo, fixes issue where downstream models aren't run when using networkx >= 2.8.1 ([#5286](https://github.com/dbt-labs/dbt-core/issues/5286), [#5326](https://github.com/dbt-labs/dbt-core/pull/5326))
|
|
||||||
- Fixing Windows color regression ([#5191](https://github.com/dbt-labs/dbt-core/issues/5191), [#5327](https://github.com/dbt-labs/dbt-core/pull/5327))
|
|
||||||
- Define compatibility for older manifest versions when using state: selection methods ([#5213](https://github.com/dbt-labs/dbt-core/issues/5213), [#5346](https://github.com/dbt-labs/dbt-core/pull/5346))
|
|
||||||
- Remove duplicate key checking introduced in 1.2.0a1 ([#5331](https://github.com/dbt-labs/dbt-core/issues/5331), [#5403](https://github.com/dbt-labs/dbt-core/pull/5403))
|
|
||||||
### Under the Hood
|
|
||||||
- Migrating 005_simple_seed to the new test framework. ([#200](https://github.com/dbt-labs/dbt-core/issues/200), [#5013](https://github.com/dbt-labs/dbt-core/pull/5013))
|
|
||||||
- Convert 029_docs_generate tests to new framework ([#5035](https://github.com/dbt-labs/dbt-core/issues/5035), [#5058](https://github.com/dbt-labs/dbt-core/pull/5058))
|
|
||||||
- Move package deprecation check outside of package cache ([#5068](https://github.com/dbt-labs/dbt-core/issues/5068), [#5069](https://github.com/dbt-labs/dbt-core/pull/5069))
|
|
||||||
- removal of scaffold first attempt and create_adapter_plugin.py as they are deprecated new scaffold can be found https://github.com/dbt-labs/dbt-database-adapter-scaffold ([#4980](https://github.com/dbt-labs/dbt-core/issues/4980), [#5117](https://github.com/dbt-labs/dbt-core/pull/5117))
|
|
||||||
- Mypy -> 0.942 + fixed import logic to allow for full mypy coverage ([#4805](https://github.com/dbt-labs/dbt-core/issues/4805), [#5171](https://github.com/dbt-labs/dbt-core/pull/5171))
|
|
||||||
- Converted dbt list tests to pytest ([#5049](https://github.com/dbt-labs/dbt-core/issues/5049), [#5178](https://github.com/dbt-labs/dbt-core/pull/5178))
|
|
||||||
- Fix: Call str and repr for UnsetProfileConfig without a RuntimeException ([#5081](https://github.com/dbt-labs/dbt-core/issues/5081), [#5209](https://github.com/dbt-labs/dbt-core/pull/5209))
|
|
||||||
- Improve tracking error logging message ([#5197](https://github.com/dbt-labs/dbt-core/issues/5197), [#5211](https://github.com/dbt-labs/dbt-core/pull/5211))
|
|
||||||
- Clean up materialization logic: more consistent relation names, loading from cache ([#2869](https://github.com/dbt-labs/dbt-core/issues/2869), [#4921](https://github.com/dbt-labs/dbt-core/pull/4921))
|
|
||||||
- Use the default Python version for local dev and test instead of requiring Python 3.8 ([#5257](https://github.com/dbt-labs/dbt-core/issues/5257), [#5269](https://github.com/dbt-labs/dbt-core/pull/5269))
|
|
||||||
- Fix test for context set function ([#5266](https://github.com/dbt-labs/dbt-core/issues/5266), [#5272](https://github.com/dbt-labs/dbt-core/pull/5272))
|
|
||||||
- Fix pip upgrade step in CI for Windows ([#5321](https://github.com/dbt-labs/dbt-core/issues/5321), [#5320](https://github.com/dbt-labs/dbt-core/pull/5320))
|
|
||||||
- Fix unit test test_graph_selection ([#5323](https://github.com/dbt-labs/dbt-core/issues/5323), [#5324](https://github.com/dbt-labs/dbt-core/pull/5324))
|
|
||||||
- Update context readme + clean up context code" ([#4796](https://github.com/dbt-labs/dbt-core/issues/4796), [#5334](https://github.com/dbt-labs/dbt-core/pull/5334))
|
|
||||||
- removed script meant for snowflake to snowflake ([#5361](https://github.com/dbt-labs/dbt-core/issues/5361), [#5362](https://github.com/dbt-labs/dbt-core/pull/5362))
|
|
||||||
### Dependencies
|
|
||||||
- Bump ubuntu from 20.04 to 22.04 ([#4904](https://github.com/dbt-labs/dbt-core/issues/4904), [#5141](https://github.com/dbt-labs/dbt-core/pull/5141))
|
|
||||||
- Bumping hologram version ([#5219](https://github.com/dbt-labs/dbt-core/issues/5219), [#5218](https://github.com/dbt-labs/dbt-core/pull/5218))
|
|
||||||
- Bump python from 3.10.3-slim-bullseye to 3.10.5-slim-bullseye in /docker ([#4904](https://github.com/dbt-labs/dbt-core/issues/4904), [#5367](https://github.com/dbt-labs/dbt-core/pull/5367))
|
|
||||||
|
|
||||||
### Contributors
|
|
||||||
- [@GtheSheep](https://github.com/GtheSheep) ([#4893](https://github.com/dbt-labs/dbt-core/pull/4893))
|
|
||||||
- [@NicolasPA](https://github.com/NicolasPA) ([#5211](https://github.com/dbt-labs/dbt-core/pull/5211))
|
|
||||||
- [@adamantike](https://github.com/adamantike) ([#5207](https://github.com/dbt-labs/dbt-core/pull/5207))
|
|
||||||
- [@alexrosenfeld10](https://github.com/alexrosenfeld10) ([#5184](https://github.com/dbt-labs/dbt-core/pull/5184))
|
|
||||||
- [@bd3dowling](https://github.com/bd3dowling) ([#5140](https://github.com/dbt-labs/dbt-core/pull/5140))
|
|
||||||
- [@danieldiamond](https://github.com/danieldiamond) ([#4827](https://github.com/dbt-labs/dbt-core/pull/4827))
|
|
||||||
- [@dbeatty10](https://github.com/dbeatty10) ([#5265](https://github.com/dbt-labs/dbt-core/pull/5265), [#5077](https://github.com/dbt-labs/dbt-core/pull/5077))
|
|
||||||
- [@dependabot[bot]](https://github.com/dependabot[bot]) ([#5141](https://github.com/dbt-labs/dbt-core/pull/5141), [#5367](https://github.com/dbt-labs/dbt-core/pull/5367))
|
|
||||||
- [@epapineau](https://github.com/epapineau) ([#4921](https://github.com/dbt-labs/dbt-core/pull/4921))
|
|
||||||
- [@groodt](https://github.com/groodt) ([#5304](https://github.com/dbt-labs/dbt-core/pull/5304))
|
|
||||||
- [@jared-rimmer](https://github.com/jared-rimmer) ([#5364](https://github.com/dbt-labs/dbt-core/pull/5364))
|
|
||||||
- [@jeremyyeo](https://github.com/jeremyyeo) ([#5107](https://github.com/dbt-labs/dbt-core/pull/5107), [#5146](https://github.com/dbt-labs/dbt-core/pull/5146), [#5403](https://github.com/dbt-labs/dbt-core/pull/5403))
|
|
||||||
- [@jwills](https://github.com/jwills) ([#5241](https://github.com/dbt-labs/dbt-core/pull/5241), [#5269](https://github.com/dbt-labs/dbt-core/pull/5269))
|
|
||||||
- [@tomasfarias](https://github.com/tomasfarias) ([#5209](https://github.com/dbt-labs/dbt-core/pull/5209))
|
|
||||||
- [@ulisesojeda](https://github.com/ulisesojeda) ([#5366](https://github.com/dbt-labs/dbt-core/pull/5366))
|
|
||||||
|
|
||||||
|
|
||||||
## Previous Releases
|
## Previous Releases
|
||||||
|
|
||||||
For information on prior major and minor releases, see their changelogs:
|
For information on prior major and minor releases, see their changelogs:
|
||||||
|
|
||||||
|
|
||||||
|
* [1.2](https://github.com/dbt-labs/dbt-core/blob/1.2.latest/CHANGELOG.md)
|
||||||
* [1.1](https://github.com/dbt-labs/dbt-core/blob/1.1.latest/CHANGELOG.md)
|
* [1.1](https://github.com/dbt-labs/dbt-core/blob/1.1.latest/CHANGELOG.md)
|
||||||
* [1.0](https://github.com/dbt-labs/dbt-core/blob/1.0.latest/CHANGELOG.md)
|
* [1.0](https://github.com/dbt-labs/dbt-core/blob/1.0.latest/CHANGELOG.md)
|
||||||
* [0.21](https://github.com/dbt-labs/dbt-core/blob/0.21.latest/CHANGELOG.md)
|
* [0.21](https://github.com/dbt-labs/dbt-core/blob/0.21.latest/CHANGELOG.md)
|
||||||
|
|||||||
@@ -1,10 +1,24 @@
|
|||||||
import abc
|
import abc
|
||||||
import os
|
import os
|
||||||
|
from time import sleep
|
||||||
|
import sys
|
||||||
|
|
||||||
# multiprocessing.RLock is a function returning this type
|
# multiprocessing.RLock is a function returning this type
|
||||||
from multiprocessing.synchronize import RLock
|
from multiprocessing.synchronize import RLock
|
||||||
from threading import get_ident
|
from threading import get_ident
|
||||||
from typing import Dict, Tuple, Hashable, Optional, ContextManager, List
|
from typing import (
|
||||||
|
Any,
|
||||||
|
Dict,
|
||||||
|
Tuple,
|
||||||
|
Hashable,
|
||||||
|
Optional,
|
||||||
|
ContextManager,
|
||||||
|
List,
|
||||||
|
Type,
|
||||||
|
Union,
|
||||||
|
Iterable,
|
||||||
|
Callable,
|
||||||
|
)
|
||||||
|
|
||||||
import agate
|
import agate
|
||||||
|
|
||||||
@@ -21,6 +35,7 @@ from dbt.contracts.graph.manifest import Manifest
|
|||||||
from dbt.adapters.base.query_headers import (
|
from dbt.adapters.base.query_headers import (
|
||||||
MacroQueryStringSetter,
|
MacroQueryStringSetter,
|
||||||
)
|
)
|
||||||
|
from dbt.events import AdapterLogger
|
||||||
from dbt.events.functions import fire_event
|
from dbt.events.functions import fire_event
|
||||||
from dbt.events.types import (
|
from dbt.events.types import (
|
||||||
NewConnection,
|
NewConnection,
|
||||||
@@ -34,6 +49,9 @@ from dbt.events.types import (
|
|||||||
)
|
)
|
||||||
from dbt import flags
|
from dbt import flags
|
||||||
|
|
||||||
|
SleepTime = Union[int, float] # As taken by time.sleep.
|
||||||
|
AdapterHandle = Any # Adapter connection handle objects can be any class.
|
||||||
|
|
||||||
|
|
||||||
class BaseConnectionManager(metaclass=abc.ABCMeta):
|
class BaseConnectionManager(metaclass=abc.ABCMeta):
|
||||||
"""Methods to implement:
|
"""Methods to implement:
|
||||||
@@ -159,6 +177,94 @@ class BaseConnectionManager(metaclass=abc.ABCMeta):
|
|||||||
conn.name = conn_name
|
conn.name = conn_name
|
||||||
return conn
|
return conn
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def retry_connection(
|
||||||
|
cls,
|
||||||
|
connection: Connection,
|
||||||
|
connect: Callable[[], AdapterHandle],
|
||||||
|
logger: AdapterLogger,
|
||||||
|
retryable_exceptions: Iterable[Type[Exception]],
|
||||||
|
retry_limit: int = 1,
|
||||||
|
retry_timeout: Union[Callable[[int], SleepTime], SleepTime] = 1,
|
||||||
|
_attempts: int = 0,
|
||||||
|
) -> Connection:
|
||||||
|
"""Given a Connection, set its handle by calling connect.
|
||||||
|
|
||||||
|
The calls to connect will be retried up to retry_limit times to deal with transient
|
||||||
|
connection errors. By default, one retry will be attempted if retryable_exceptions is set.
|
||||||
|
|
||||||
|
:param Connection connection: An instance of a Connection that needs a handle to be set,
|
||||||
|
usually when attempting to open it.
|
||||||
|
:param connect: A callable that returns the appropiate connection handle for a
|
||||||
|
given adapter. This callable will be retried retry_limit times if a subclass of any
|
||||||
|
Exception in retryable_exceptions is raised by connect.
|
||||||
|
:type connect: Callable[[], AdapterHandle]
|
||||||
|
:param AdapterLogger logger: A logger to emit messages on retry attempts or errors. When
|
||||||
|
handling expected errors, we call debug, and call warning on unexpected errors or when
|
||||||
|
all retry attempts have been exhausted.
|
||||||
|
:param retryable_exceptions: An iterable of exception classes that if raised by
|
||||||
|
connect should trigger a retry.
|
||||||
|
:type retryable_exceptions: Iterable[Type[Exception]]
|
||||||
|
:param int retry_limit: How many times to retry the call to connect. If this limit
|
||||||
|
is exceeded before a successful call, a FailedToConnectException will be raised.
|
||||||
|
Must be non-negative.
|
||||||
|
:param retry_timeout: Time to wait between attempts to connect. Can also take a
|
||||||
|
Callable that takes the number of attempts so far, beginning at 0, and returns an int
|
||||||
|
or float to be passed to time.sleep.
|
||||||
|
:type retry_timeout: Union[Callable[[int], SleepTime], SleepTime] = 1
|
||||||
|
:param int _attempts: Parameter used to keep track of the number of attempts in calling the
|
||||||
|
connect function across recursive calls. Passed as an argument to retry_timeout if it
|
||||||
|
is a Callable. This parameter should not be set by the initial caller.
|
||||||
|
:raises dbt.exceptions.FailedToConnectException: Upon exhausting all retry attempts without
|
||||||
|
successfully acquiring a handle.
|
||||||
|
:return: The given connection with its appropriate state and handle attributes set
|
||||||
|
depending on whether we successfully acquired a handle or not.
|
||||||
|
"""
|
||||||
|
timeout = retry_timeout(_attempts) if callable(retry_timeout) else retry_timeout
|
||||||
|
if timeout < 0:
|
||||||
|
raise dbt.exceptions.FailedToConnectException(
|
||||||
|
"retry_timeout cannot be negative or return a negative time."
|
||||||
|
)
|
||||||
|
|
||||||
|
if retry_limit < 0 or retry_limit > sys.getrecursionlimit():
|
||||||
|
# This guard is not perfect others may add to the recursion limit (e.g. built-ins).
|
||||||
|
connection.handle = None
|
||||||
|
connection.state = ConnectionState.FAIL
|
||||||
|
raise dbt.exceptions.FailedToConnectException("retry_limit cannot be negative")
|
||||||
|
|
||||||
|
try:
|
||||||
|
connection.handle = connect()
|
||||||
|
connection.state = ConnectionState.OPEN
|
||||||
|
return connection
|
||||||
|
|
||||||
|
except tuple(retryable_exceptions) as e:
|
||||||
|
if retry_limit <= 0:
|
||||||
|
connection.handle = None
|
||||||
|
connection.state = ConnectionState.FAIL
|
||||||
|
raise dbt.exceptions.FailedToConnectException(str(e))
|
||||||
|
|
||||||
|
logger.debug(
|
||||||
|
f"Got a retryable error when attempting to open a {cls.TYPE} connection.\n"
|
||||||
|
f"{retry_limit} attempts remaining. Retrying in {timeout} seconds.\n"
|
||||||
|
f"Error:\n{e}"
|
||||||
|
)
|
||||||
|
|
||||||
|
sleep(timeout)
|
||||||
|
return cls.retry_connection(
|
||||||
|
connection=connection,
|
||||||
|
connect=connect,
|
||||||
|
logger=logger,
|
||||||
|
retry_limit=retry_limit - 1,
|
||||||
|
retry_timeout=retry_timeout,
|
||||||
|
retryable_exceptions=retryable_exceptions,
|
||||||
|
_attempts=_attempts + 1,
|
||||||
|
)
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
connection.handle = None
|
||||||
|
connection.state = ConnectionState.FAIL
|
||||||
|
raise dbt.exceptions.FailedToConnectException(str(e))
|
||||||
|
|
||||||
@abc.abstractmethod
|
@abc.abstractmethod
|
||||||
def cancel_open(self) -> Optional[List[str]]:
|
def cancel_open(self) -> Optional[List[str]]:
|
||||||
"""Cancel all open connections on the adapter. (passable)"""
|
"""Cancel all open connections on the adapter. (passable)"""
|
||||||
|
|||||||
@@ -159,6 +159,7 @@ class BaseAdapter(metaclass=AdapterMeta):
|
|||||||
- convert_datetime_type
|
- convert_datetime_type
|
||||||
- convert_date_type
|
- convert_date_type
|
||||||
- convert_time_type
|
- convert_time_type
|
||||||
|
- standardize_grants_dict
|
||||||
|
|
||||||
Macros:
|
Macros:
|
||||||
- get_catalog
|
- get_catalog
|
||||||
@@ -538,6 +539,33 @@ class BaseAdapter(metaclass=AdapterMeta):
|
|||||||
"`list_relations_without_caching` is not implemented for this " "adapter!"
|
"`list_relations_without_caching` is not implemented for this " "adapter!"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
###
|
||||||
|
# Methods about grants
|
||||||
|
###
|
||||||
|
@available
|
||||||
|
def standardize_grants_dict(self, grants_table: agate.Table) -> dict:
|
||||||
|
"""Translate the result of `show grants` (or equivalent) to match the
|
||||||
|
grants which a user would configure in their project.
|
||||||
|
|
||||||
|
Ideally, the SQL to show grants should also be filtering:
|
||||||
|
filter OUT any grants TO the current user/role (e.g. OWNERSHIP).
|
||||||
|
If that's not possible in SQL, it can be done in this method instead.
|
||||||
|
|
||||||
|
:param grants_table: An agate table containing the query result of
|
||||||
|
the SQL returned by get_show_grant_sql
|
||||||
|
:return: A standardized dictionary matching the `grants` config
|
||||||
|
:rtype: dict
|
||||||
|
"""
|
||||||
|
grants_dict: Dict[str, List[str]] = {}
|
||||||
|
for row in grants_table:
|
||||||
|
grantee = row["grantee"]
|
||||||
|
privilege = row["privilege_type"]
|
||||||
|
if privilege in grants_dict.keys():
|
||||||
|
grants_dict[privilege].append(grantee)
|
||||||
|
else:
|
||||||
|
grants_dict.update({privilege: [grantee]})
|
||||||
|
return grants_dict
|
||||||
|
|
||||||
###
|
###
|
||||||
# Provided methods about relations
|
# Provided methods about relations
|
||||||
###
|
###
|
||||||
|
|||||||
@@ -397,6 +397,8 @@ class Compiler:
|
|||||||
linker.dependency(node.unique_id, (manifest.nodes[dependency].unique_id))
|
linker.dependency(node.unique_id, (manifest.nodes[dependency].unique_id))
|
||||||
elif dependency in manifest.sources:
|
elif dependency in manifest.sources:
|
||||||
linker.dependency(node.unique_id, (manifest.sources[dependency].unique_id))
|
linker.dependency(node.unique_id, (manifest.sources[dependency].unique_id))
|
||||||
|
elif dependency in manifest.metrics:
|
||||||
|
linker.dependency(node.unique_id, (manifest.metrics[dependency].unique_id))
|
||||||
else:
|
else:
|
||||||
dependency_not_found(node, dependency)
|
dependency_not_found(node, dependency)
|
||||||
|
|
||||||
|
|||||||
@@ -15,7 +15,7 @@ from typing_extensions import Protocol, runtime_checkable
|
|||||||
import hashlib
|
import hashlib
|
||||||
import os
|
import os
|
||||||
|
|
||||||
from dbt import deprecations
|
from dbt import flags, deprecations
|
||||||
from dbt.clients.system import resolve_path_from_base
|
from dbt.clients.system import resolve_path_from_base
|
||||||
from dbt.clients.system import path_exists
|
from dbt.clients.system import path_exists
|
||||||
from dbt.clients.system import load_file_contents
|
from dbt.clients.system import load_file_contents
|
||||||
@@ -142,6 +142,13 @@ def _all_source_paths(
|
|||||||
T = TypeVar("T")
|
T = TypeVar("T")
|
||||||
|
|
||||||
|
|
||||||
|
def flag_or(flag: Optional[T], value: Optional[T], default: T) -> T:
|
||||||
|
if flag is None:
|
||||||
|
return value_or(value, default)
|
||||||
|
else:
|
||||||
|
return flag
|
||||||
|
|
||||||
|
|
||||||
def value_or(value: Optional[T], default: T) -> T:
|
def value_or(value: Optional[T], default: T) -> T:
|
||||||
if value is None:
|
if value is None:
|
||||||
return default
|
return default
|
||||||
@@ -356,9 +363,9 @@ class PartialProject(RenderComponents):
|
|||||||
|
|
||||||
docs_paths: List[str] = value_or(cfg.docs_paths, all_source_paths)
|
docs_paths: List[str] = value_or(cfg.docs_paths, all_source_paths)
|
||||||
asset_paths: List[str] = value_or(cfg.asset_paths, [])
|
asset_paths: List[str] = value_or(cfg.asset_paths, [])
|
||||||
target_path: str = value_or(cfg.target_path, "target")
|
target_path: str = flag_or(flags.TARGET_PATH, cfg.target_path, "target")
|
||||||
clean_targets: List[str] = value_or(cfg.clean_targets, [target_path])
|
clean_targets: List[str] = value_or(cfg.clean_targets, [target_path])
|
||||||
log_path: str = value_or(cfg.log_path, "logs")
|
log_path: str = flag_or(flags.LOG_PATH, cfg.log_path, "logs")
|
||||||
packages_install_path: str = value_or(cfg.packages_install_path, "dbt_packages")
|
packages_install_path: str = value_or(cfg.packages_install_path, "dbt_packages")
|
||||||
# in the default case we'll populate this once we know the adapter type
|
# in the default case we'll populate this once we know the adapter type
|
||||||
# It would be nice to just pass along a Quoting here, but that would
|
# It would be nice to just pass along a Quoting here, but that would
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
from typing import Any, Dict, NoReturn, Optional, Mapping, Iterable, Set
|
from typing import Any, Dict, NoReturn, Optional, Mapping, Iterable, Set, List
|
||||||
|
|
||||||
from dbt import flags
|
from dbt import flags
|
||||||
from dbt import tracking
|
from dbt import tracking
|
||||||
@@ -474,19 +474,17 @@ class BaseContext(metaclass=ContextMeta):
|
|||||||
|
|
||||||
@contextmember
|
@contextmember
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def try_set(value: Iterable[Any]) -> Set[Any]:
|
def set_strict(value: Iterable[Any]) -> Set[Any]:
|
||||||
"""The `try_set` context method can be used to convert any iterable
|
"""The `set_strict` context method can be used to convert any iterable
|
||||||
to a sequence of iterable elements that are unique (a set). The
|
to a sequence of iterable elements that are unique (a set). The
|
||||||
difference to the `set` context method is that the `try_set` method
|
difference to the `set` context method is that the `set_strict` method
|
||||||
will raise an exception on a TypeError.
|
will raise an exception on a TypeError.
|
||||||
|
|
||||||
:param value: The iterable
|
:param value: The iterable
|
||||||
:param default: A default value to return if the `value` argument
|
|
||||||
is not an iterable
|
|
||||||
|
|
||||||
Usage:
|
Usage:
|
||||||
{% set my_list = [1, 2, 2, 3] %}
|
{% set my_list = [1, 2, 2, 3] %}
|
||||||
{% set my_set = try_set(my_list) %}
|
{% set my_set = set_strict(my_list) %}
|
||||||
{% do log(my_set) %} {# {1, 2, 3} #}
|
{% do log(my_set) %} {# {1, 2, 3} #}
|
||||||
"""
|
"""
|
||||||
try:
|
try:
|
||||||
@@ -497,7 +495,7 @@ class BaseContext(metaclass=ContextMeta):
|
|||||||
@contextmember("zip")
|
@contextmember("zip")
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _zip(*args: Iterable[Any], default: Any = None) -> Optional[Iterable[Any]]:
|
def _zip(*args: Iterable[Any], default: Any = None) -> Optional[Iterable[Any]]:
|
||||||
"""The `try_zip` context method can be used to used to return
|
"""The `zip` context method can be used to used to return
|
||||||
an iterator of tuples, where the i-th tuple contains the i-th
|
an iterator of tuples, where the i-th tuple contains the i-th
|
||||||
element from each of the argument iterables.
|
element from each of the argument iterables.
|
||||||
|
|
||||||
@@ -518,21 +516,19 @@ class BaseContext(metaclass=ContextMeta):
|
|||||||
|
|
||||||
@contextmember
|
@contextmember
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def try_zip(*args: Iterable[Any]) -> Iterable[Any]:
|
def zip_strict(*args: Iterable[Any]) -> Iterable[Any]:
|
||||||
"""The `try_zip` context method can be used to used to return
|
"""The `zip_strict` context method can be used to used to return
|
||||||
an iterator of tuples, where the i-th tuple contains the i-th
|
an iterator of tuples, where the i-th tuple contains the i-th
|
||||||
element from each of the argument iterables. The difference to the
|
element from each of the argument iterables. The difference to the
|
||||||
`zip` context method is that the `try_zip` method will raise an
|
`zip` context method is that the `zip_strict` method will raise an
|
||||||
exception on a TypeError.
|
exception on a TypeError.
|
||||||
|
|
||||||
:param *args: Any number of iterables
|
:param *args: Any number of iterables
|
||||||
:param default: A default value to return if `*args` is not
|
|
||||||
iterable
|
|
||||||
|
|
||||||
Usage:
|
Usage:
|
||||||
{% set my_list_a = [1, 2] %}
|
{% set my_list_a = [1, 2] %}
|
||||||
{% set my_list_b = ['alice', 'bob'] %}
|
{% set my_list_b = ['alice', 'bob'] %}
|
||||||
{% set my_zip = try_zip(my_list_a, my_list_b) | list %}
|
{% set my_zip = zip_strict(my_list_a, my_list_b) | list %}
|
||||||
{% do log(my_set) %} {# [(1, 'alice'), (2, 'bob')] #}
|
{% do log(my_set) %} {# [(1, 'alice'), (2, 'bob')] #}
|
||||||
"""
|
"""
|
||||||
try:
|
try:
|
||||||
@@ -657,6 +653,35 @@ class BaseContext(metaclass=ContextMeta):
|
|||||||
print(msg)
|
print(msg)
|
||||||
return ""
|
return ""
|
||||||
|
|
||||||
|
@contextmember
|
||||||
|
@staticmethod
|
||||||
|
def diff_of_two_dicts(
|
||||||
|
dict_a: Dict[str, List[str]], dict_b: Dict[str, List[str]]
|
||||||
|
) -> Dict[str, List[str]]:
|
||||||
|
"""
|
||||||
|
Given two dictionaries of type Dict[str, List[str]]:
|
||||||
|
dict_a = {'key_x': ['value_1', 'VALUE_2'], 'KEY_Y': ['value_3']}
|
||||||
|
dict_b = {'key_x': ['value_1'], 'key_z': ['value_4']}
|
||||||
|
Return the same dictionary representation of dict_a MINUS dict_b,
|
||||||
|
performing a case-insensitive comparison between the strings in each.
|
||||||
|
All keys returned will be in the original case of dict_a.
|
||||||
|
returns {'key_x': ['VALUE_2'], 'KEY_Y': ['value_3']}
|
||||||
|
"""
|
||||||
|
|
||||||
|
dict_diff = {}
|
||||||
|
dict_b_lowered = {k.casefold(): [x.casefold() for x in v] for k, v in dict_b.items()}
|
||||||
|
for k in dict_a:
|
||||||
|
if k.casefold() in dict_b_lowered.keys():
|
||||||
|
diff = []
|
||||||
|
for v in dict_a[k]:
|
||||||
|
if v.casefold() not in dict_b_lowered[k.casefold()]:
|
||||||
|
diff.append(v)
|
||||||
|
if diff:
|
||||||
|
dict_diff.update({k: diff})
|
||||||
|
else:
|
||||||
|
dict_diff.update({k: dict_a[k]})
|
||||||
|
return dict_diff
|
||||||
|
|
||||||
|
|
||||||
def generate_base_context(cli_vars: Dict[str, Any]) -> Dict[str, Any]:
|
def generate_base_context(cli_vars: Dict[str, Any]) -> Dict[str, Any]:
|
||||||
ctx = BaseContext(cli_vars)
|
ctx = BaseContext(cli_vars)
|
||||||
|
|||||||
@@ -40,6 +40,7 @@ from dbt.contracts.graph.parsed import (
|
|||||||
ParsedSeedNode,
|
ParsedSeedNode,
|
||||||
ParsedSourceDefinition,
|
ParsedSourceDefinition,
|
||||||
)
|
)
|
||||||
|
from dbt.contracts.graph.metrics import MetricReference, ResolvedMetricReference
|
||||||
from dbt.exceptions import (
|
from dbt.exceptions import (
|
||||||
CompilationException,
|
CompilationException,
|
||||||
ParsingException,
|
ParsingException,
|
||||||
@@ -50,7 +51,9 @@ from dbt.exceptions import (
|
|||||||
missing_config,
|
missing_config,
|
||||||
raise_compiler_error,
|
raise_compiler_error,
|
||||||
ref_invalid_args,
|
ref_invalid_args,
|
||||||
|
metric_invalid_args,
|
||||||
ref_target_not_found,
|
ref_target_not_found,
|
||||||
|
metric_target_not_found,
|
||||||
ref_bad_context,
|
ref_bad_context,
|
||||||
source_target_not_found,
|
source_target_not_found,
|
||||||
wrapped_exports,
|
wrapped_exports,
|
||||||
@@ -199,7 +202,7 @@ class BaseResolver(metaclass=abc.ABCMeta):
|
|||||||
return self.db_wrapper.Relation
|
return self.db_wrapper.Relation
|
||||||
|
|
||||||
@abc.abstractmethod
|
@abc.abstractmethod
|
||||||
def __call__(self, *args: str) -> Union[str, RelationProxy]:
|
def __call__(self, *args: str) -> Union[str, RelationProxy, MetricReference]:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
||||||
@@ -265,6 +268,41 @@ class BaseSourceResolver(BaseResolver):
|
|||||||
return self.resolve(args[0], args[1])
|
return self.resolve(args[0], args[1])
|
||||||
|
|
||||||
|
|
||||||
|
class BaseMetricResolver(BaseResolver):
|
||||||
|
def resolve(self, name: str, package: Optional[str] = None) -> MetricReference:
|
||||||
|
...
|
||||||
|
|
||||||
|
def _repack_args(self, name: str, package: Optional[str]) -> List[str]:
|
||||||
|
if package is None:
|
||||||
|
return [name]
|
||||||
|
else:
|
||||||
|
return [package, name]
|
||||||
|
|
||||||
|
def validate_args(self, name: str, package: Optional[str]):
|
||||||
|
if not isinstance(name, str):
|
||||||
|
raise CompilationException(
|
||||||
|
f"The name argument to metric() must be a string, got {type(name)}"
|
||||||
|
)
|
||||||
|
|
||||||
|
if package is not None and not isinstance(package, str):
|
||||||
|
raise CompilationException(
|
||||||
|
f"The package argument to metric() must be a string or None, got {type(package)}"
|
||||||
|
)
|
||||||
|
|
||||||
|
def __call__(self, *args: str) -> MetricReference:
|
||||||
|
name: str
|
||||||
|
package: Optional[str] = None
|
||||||
|
|
||||||
|
if len(args) == 1:
|
||||||
|
name = args[0]
|
||||||
|
elif len(args) == 2:
|
||||||
|
package, name = args
|
||||||
|
else:
|
||||||
|
metric_invalid_args(self.model, args)
|
||||||
|
self.validate_args(name, package)
|
||||||
|
return self.resolve(name, package)
|
||||||
|
|
||||||
|
|
||||||
class Config(Protocol):
|
class Config(Protocol):
|
||||||
def __init__(self, model, context_config: Optional[ContextConfig]):
|
def __init__(self, model, context_config: Optional[ContextConfig]):
|
||||||
...
|
...
|
||||||
@@ -511,6 +549,34 @@ class RuntimeSourceResolver(BaseSourceResolver):
|
|||||||
return self.Relation.create_from_source(target_source)
|
return self.Relation.create_from_source(target_source)
|
||||||
|
|
||||||
|
|
||||||
|
# metric` implementations
|
||||||
|
class ParseMetricResolver(BaseMetricResolver):
|
||||||
|
def resolve(self, name: str, package: Optional[str] = None) -> MetricReference:
|
||||||
|
self.model.metrics.append(self._repack_args(name, package))
|
||||||
|
|
||||||
|
return MetricReference(name, package)
|
||||||
|
|
||||||
|
|
||||||
|
class RuntimeMetricResolver(BaseMetricResolver):
|
||||||
|
def resolve(self, target_name: str, target_package: Optional[str] = None) -> MetricReference:
|
||||||
|
target_metric = self.manifest.resolve_metric(
|
||||||
|
target_name,
|
||||||
|
target_package,
|
||||||
|
self.current_project,
|
||||||
|
self.model.package_name,
|
||||||
|
)
|
||||||
|
|
||||||
|
if target_metric is None or isinstance(target_metric, Disabled):
|
||||||
|
# TODO : Use a different exception!!
|
||||||
|
metric_target_not_found(
|
||||||
|
self.model,
|
||||||
|
target_name,
|
||||||
|
target_package,
|
||||||
|
)
|
||||||
|
|
||||||
|
return ResolvedMetricReference(target_metric, self.manifest, self.Relation)
|
||||||
|
|
||||||
|
|
||||||
# `var` implementations.
|
# `var` implementations.
|
||||||
class ModelConfiguredVar(Var):
|
class ModelConfiguredVar(Var):
|
||||||
def __init__(
|
def __init__(
|
||||||
@@ -568,6 +634,7 @@ class Provider(Protocol):
|
|||||||
Var: Type[ModelConfiguredVar]
|
Var: Type[ModelConfiguredVar]
|
||||||
ref: Type[BaseRefResolver]
|
ref: Type[BaseRefResolver]
|
||||||
source: Type[BaseSourceResolver]
|
source: Type[BaseSourceResolver]
|
||||||
|
metric: Type[BaseMetricResolver]
|
||||||
|
|
||||||
|
|
||||||
class ParseProvider(Provider):
|
class ParseProvider(Provider):
|
||||||
@@ -577,6 +644,7 @@ class ParseProvider(Provider):
|
|||||||
Var = ParseVar
|
Var = ParseVar
|
||||||
ref = ParseRefResolver
|
ref = ParseRefResolver
|
||||||
source = ParseSourceResolver
|
source = ParseSourceResolver
|
||||||
|
metric = ParseMetricResolver
|
||||||
|
|
||||||
|
|
||||||
class GenerateNameProvider(Provider):
|
class GenerateNameProvider(Provider):
|
||||||
@@ -586,6 +654,7 @@ class GenerateNameProvider(Provider):
|
|||||||
Var = RuntimeVar
|
Var = RuntimeVar
|
||||||
ref = ParseRefResolver
|
ref = ParseRefResolver
|
||||||
source = ParseSourceResolver
|
source = ParseSourceResolver
|
||||||
|
metric = ParseMetricResolver
|
||||||
|
|
||||||
|
|
||||||
class RuntimeProvider(Provider):
|
class RuntimeProvider(Provider):
|
||||||
@@ -595,6 +664,7 @@ class RuntimeProvider(Provider):
|
|||||||
Var = RuntimeVar
|
Var = RuntimeVar
|
||||||
ref = RuntimeRefResolver
|
ref = RuntimeRefResolver
|
||||||
source = RuntimeSourceResolver
|
source = RuntimeSourceResolver
|
||||||
|
metric = RuntimeMetricResolver
|
||||||
|
|
||||||
|
|
||||||
class OperationProvider(RuntimeProvider):
|
class OperationProvider(RuntimeProvider):
|
||||||
@@ -778,6 +848,10 @@ class ProviderContext(ManifestContext):
|
|||||||
def source(self) -> Callable:
|
def source(self) -> Callable:
|
||||||
return self.provider.source(self.db_wrapper, self.model, self.config, self.manifest)
|
return self.provider.source(self.db_wrapper, self.model, self.config, self.manifest)
|
||||||
|
|
||||||
|
@contextproperty
|
||||||
|
def metric(self) -> Callable:
|
||||||
|
return self.provider.metric(self.db_wrapper, self.model, self.config, self.manifest)
|
||||||
|
|
||||||
@contextproperty("config")
|
@contextproperty("config")
|
||||||
def ctx_config(self) -> Config:
|
def ctx_config(self) -> Config:
|
||||||
"""The `config` variable exists to handle end-user configuration for
|
"""The `config` variable exists to handle end-user configuration for
|
||||||
@@ -1355,7 +1429,7 @@ class MetricRefResolver(BaseResolver):
|
|||||||
if not isinstance(name, str):
|
if not isinstance(name, str):
|
||||||
raise ParsingException(
|
raise ParsingException(
|
||||||
f"In a metrics section in {self.model.original_file_path} "
|
f"In a metrics section in {self.model.original_file_path} "
|
||||||
f"the name argument to ref() must be a string"
|
"the name argument to ref() must be a string"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@@ -1373,6 +1447,12 @@ def generate_parse_metrics(
|
|||||||
project,
|
project,
|
||||||
manifest,
|
manifest,
|
||||||
),
|
),
|
||||||
|
"metric": ParseMetricResolver(
|
||||||
|
None,
|
||||||
|
metric,
|
||||||
|
project,
|
||||||
|
manifest,
|
||||||
|
),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -183,6 +183,39 @@ class RefableLookup(dbtClassMixin):
|
|||||||
return manifest.nodes[unique_id]
|
return manifest.nodes[unique_id]
|
||||||
|
|
||||||
|
|
||||||
|
class MetricLookup(dbtClassMixin):
|
||||||
|
def __init__(self, manifest: "Manifest"):
|
||||||
|
self.storage: Dict[str, Dict[PackageName, UniqueID]] = {}
|
||||||
|
self.populate(manifest)
|
||||||
|
|
||||||
|
def get_unique_id(self, search_name, package: Optional[PackageName]):
|
||||||
|
return find_unique_id_for_package(self.storage, search_name, package)
|
||||||
|
|
||||||
|
def find(self, search_name, package: Optional[PackageName], manifest: "Manifest"):
|
||||||
|
unique_id = self.get_unique_id(search_name, package)
|
||||||
|
if unique_id is not None:
|
||||||
|
return self.perform_lookup(unique_id, manifest)
|
||||||
|
return None
|
||||||
|
|
||||||
|
def add_metric(self, metric: ParsedMetric):
|
||||||
|
if metric.search_name not in self.storage:
|
||||||
|
self.storage[metric.search_name] = {}
|
||||||
|
|
||||||
|
self.storage[metric.search_name][metric.package_name] = metric.unique_id
|
||||||
|
|
||||||
|
def populate(self, manifest):
|
||||||
|
for metric in manifest.metrics.values():
|
||||||
|
if hasattr(metric, "name"):
|
||||||
|
self.add_metric(metric)
|
||||||
|
|
||||||
|
def perform_lookup(self, unique_id: UniqueID, manifest: "Manifest") -> ParsedMetric:
|
||||||
|
if unique_id not in manifest.metrics:
|
||||||
|
raise dbt.exceptions.InternalException(
|
||||||
|
f"Metric {unique_id} found in cache but not found in manifest"
|
||||||
|
)
|
||||||
|
return manifest.metrics[unique_id]
|
||||||
|
|
||||||
|
|
||||||
# This handles both models/seeds/snapshots and sources
|
# This handles both models/seeds/snapshots and sources
|
||||||
class DisabledLookup(dbtClassMixin):
|
class DisabledLookup(dbtClassMixin):
|
||||||
def __init__(self, manifest: "Manifest"):
|
def __init__(self, manifest: "Manifest"):
|
||||||
@@ -431,6 +464,9 @@ class Disabled(Generic[D]):
|
|||||||
target: D
|
target: D
|
||||||
|
|
||||||
|
|
||||||
|
MaybeMetricNode = Optional[ParsedMetric]
|
||||||
|
|
||||||
|
|
||||||
MaybeDocumentation = Optional[ParsedDocumentation]
|
MaybeDocumentation = Optional[ParsedDocumentation]
|
||||||
|
|
||||||
|
|
||||||
@@ -592,6 +628,9 @@ class Manifest(MacroMethods, DataClassMessagePackMixin, dbtClassMixin):
|
|||||||
_ref_lookup: Optional[RefableLookup] = field(
|
_ref_lookup: Optional[RefableLookup] = field(
|
||||||
default=None, metadata={"serialize": lambda x: None, "deserialize": lambda x: None}
|
default=None, metadata={"serialize": lambda x: None, "deserialize": lambda x: None}
|
||||||
)
|
)
|
||||||
|
_metric_lookup: Optional[MetricLookup] = field(
|
||||||
|
default=None, metadata={"serialize": lambda x: None, "deserialize": lambda x: None}
|
||||||
|
)
|
||||||
_disabled_lookup: Optional[DisabledLookup] = field(
|
_disabled_lookup: Optional[DisabledLookup] = field(
|
||||||
default=None, metadata={"serialize": lambda x: None, "deserialize": lambda x: None}
|
default=None, metadata={"serialize": lambda x: None, "deserialize": lambda x: None}
|
||||||
)
|
)
|
||||||
@@ -837,6 +876,12 @@ class Manifest(MacroMethods, DataClassMessagePackMixin, dbtClassMixin):
|
|||||||
self._ref_lookup = RefableLookup(self)
|
self._ref_lookup = RefableLookup(self)
|
||||||
return self._ref_lookup
|
return self._ref_lookup
|
||||||
|
|
||||||
|
@property
|
||||||
|
def metric_lookup(self) -> MetricLookup:
|
||||||
|
if self._metric_lookup is None:
|
||||||
|
self._metric_lookup = MetricLookup(self)
|
||||||
|
return self._metric_lookup
|
||||||
|
|
||||||
def rebuild_ref_lookup(self):
|
def rebuild_ref_lookup(self):
|
||||||
self._ref_lookup = RefableLookup(self)
|
self._ref_lookup = RefableLookup(self)
|
||||||
|
|
||||||
@@ -912,6 +957,22 @@ class Manifest(MacroMethods, DataClassMessagePackMixin, dbtClassMixin):
|
|||||||
return Disabled(disabled[0])
|
return Disabled(disabled[0])
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
def resolve_metric(
|
||||||
|
self,
|
||||||
|
target_metric_name: str,
|
||||||
|
target_metric_package: Optional[str],
|
||||||
|
current_project: str,
|
||||||
|
node_package: str,
|
||||||
|
) -> MaybeMetricNode:
|
||||||
|
metric: Optional[ParsedMetric] = None
|
||||||
|
|
||||||
|
candidates = _search_packages(current_project, node_package, target_metric_package)
|
||||||
|
for pkg in candidates:
|
||||||
|
metric = self.metric_lookup.find(target_metric_name, pkg, self)
|
||||||
|
if metric is not None:
|
||||||
|
return metric
|
||||||
|
return None
|
||||||
|
|
||||||
# Called by DocsRuntimeContext.doc
|
# Called by DocsRuntimeContext.doc
|
||||||
def resolve_doc(
|
def resolve_doc(
|
||||||
self,
|
self,
|
||||||
@@ -1076,6 +1137,7 @@ class Manifest(MacroMethods, DataClassMessagePackMixin, dbtClassMixin):
|
|||||||
self._doc_lookup,
|
self._doc_lookup,
|
||||||
self._source_lookup,
|
self._source_lookup,
|
||||||
self._ref_lookup,
|
self._ref_lookup,
|
||||||
|
self._metric_lookup,
|
||||||
self._disabled_lookup,
|
self._disabled_lookup,
|
||||||
self._analysis_lookup,
|
self._analysis_lookup,
|
||||||
)
|
)
|
||||||
@@ -1095,7 +1157,7 @@ AnyManifest = Union[Manifest, MacroManifest]
|
|||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
@schema_version("manifest", 5)
|
@schema_version("manifest", 6)
|
||||||
class WritableManifest(ArtifactMixin):
|
class WritableManifest(ArtifactMixin):
|
||||||
nodes: Mapping[UniqueID, ManifestNode] = field(
|
nodes: Mapping[UniqueID, ManifestNode] = field(
|
||||||
metadata=dict(description=("The nodes defined in the dbt project and its dependencies"))
|
metadata=dict(description=("The nodes defined in the dbt project and its dependencies"))
|
||||||
@@ -1141,7 +1203,7 @@ class WritableManifest(ArtifactMixin):
|
|||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def compatible_previous_versions(self):
|
def compatible_previous_versions(self):
|
||||||
return [("manifest", 4)]
|
return [("manifest", 4), ("manifest", 5)]
|
||||||
|
|
||||||
def __post_serialize__(self, dct):
|
def __post_serialize__(self, dct):
|
||||||
for unique_id, node in dct["nodes"].items():
|
for unique_id, node in dct["nodes"].items():
|
||||||
|
|||||||
70
core/dbt/contracts/graph/metrics.py
Normal file
70
core/dbt/contracts/graph/metrics.py
Normal file
@@ -0,0 +1,70 @@
|
|||||||
|
from dbt.node_types import NodeType
|
||||||
|
|
||||||
|
|
||||||
|
class MetricReference(object):
|
||||||
|
def __init__(self, metric_name, package_name=None):
|
||||||
|
self.metric_name = metric_name
|
||||||
|
self.package_name = package_name
|
||||||
|
|
||||||
|
def __str__(self):
|
||||||
|
return f"{self.metric_name}"
|
||||||
|
|
||||||
|
|
||||||
|
class ResolvedMetricReference(MetricReference):
|
||||||
|
"""
|
||||||
|
Simple proxy over a ParsedMetric which delegates property
|
||||||
|
lookups to the underlying node. Also adds helper functions
|
||||||
|
for working with metrics (ie. __str__ and templating functions)
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, node, manifest, Relation):
|
||||||
|
super().__init__(node.name, node.package_name)
|
||||||
|
self.node = node
|
||||||
|
self.manifest = manifest
|
||||||
|
self.Relation = Relation
|
||||||
|
|
||||||
|
def __getattr__(self, key):
|
||||||
|
return getattr(self.node, key)
|
||||||
|
|
||||||
|
def __str__(self):
|
||||||
|
return f"{self.node.name}"
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def parent_metrics(cls, metric_node, manifest):
|
||||||
|
yield metric_node
|
||||||
|
|
||||||
|
for parent_unique_id in metric_node.depends_on.nodes:
|
||||||
|
node = manifest.metrics.get(parent_unique_id)
|
||||||
|
if node and node.resource_type == NodeType.Metric:
|
||||||
|
yield from cls.parent_metrics(node, manifest)
|
||||||
|
|
||||||
|
def parent_models(self):
|
||||||
|
in_scope_metrics = list(self.parent_metrics(self.node, self.manifest))
|
||||||
|
|
||||||
|
to_return = {
|
||||||
|
"base": [],
|
||||||
|
"derived": [],
|
||||||
|
}
|
||||||
|
for metric in in_scope_metrics:
|
||||||
|
if metric.type == "expression":
|
||||||
|
to_return["derived"].append(
|
||||||
|
{"metric_source": None, "metric": metric, "is_derived": True}
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
for node_unique_id in metric.depends_on.nodes:
|
||||||
|
node = self.manifest.nodes.get(node_unique_id)
|
||||||
|
if node and node.resource_type in NodeType.refable():
|
||||||
|
to_return["base"].append(
|
||||||
|
{
|
||||||
|
"metric_relation_node": node,
|
||||||
|
"metric_relation": self.Relation.create(
|
||||||
|
database=node.database,
|
||||||
|
schema=node.schema,
|
||||||
|
identifier=node.alias,
|
||||||
|
),
|
||||||
|
"metric": metric,
|
||||||
|
"is_derived": False,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
return to_return
|
||||||
@@ -7,7 +7,8 @@ from dbt.dataclass_schema import (
|
|||||||
ValidationError,
|
ValidationError,
|
||||||
register_pattern,
|
register_pattern,
|
||||||
)
|
)
|
||||||
from dbt.contracts.graph.unparsed import AdditionalPropertiesAllowed
|
from dbt.contracts.graph.unparsed import AdditionalPropertiesAllowed, Docs
|
||||||
|
from dbt.contracts.graph.utils import validate_color
|
||||||
from dbt.exceptions import InternalException, CompilationException
|
from dbt.exceptions import InternalException, CompilationException
|
||||||
from dbt.contracts.util import Replaceable, list_str
|
from dbt.contracts.util import Replaceable, list_str
|
||||||
from dbt import hooks
|
from dbt import hooks
|
||||||
@@ -285,7 +286,7 @@ class BaseConfig(AdditionalPropertiesAllowed, Replaceable):
|
|||||||
# 'meta' moved here from node
|
# 'meta' moved here from node
|
||||||
mergebehavior = {
|
mergebehavior = {
|
||||||
"append": ["pre-hook", "pre_hook", "post-hook", "post_hook", "tags"],
|
"append": ["pre-hook", "pre_hook", "post-hook", "post_hook", "tags"],
|
||||||
"update": ["quoting", "column_types", "meta"],
|
"update": ["quoting", "column_types", "meta", "docs"],
|
||||||
"dict_key_append": ["grants"],
|
"dict_key_append": ["grants"],
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -460,6 +461,20 @@ class NodeConfig(NodeAndTestConfig):
|
|||||||
grants: Dict[str, Any] = field(
|
grants: Dict[str, Any] = field(
|
||||||
default_factory=dict, metadata=MergeBehavior.DictKeyAppend.meta()
|
default_factory=dict, metadata=MergeBehavior.DictKeyAppend.meta()
|
||||||
)
|
)
|
||||||
|
docs: Docs = field(
|
||||||
|
default_factory=lambda: Docs(show=True),
|
||||||
|
metadata=MergeBehavior.Update.meta(),
|
||||||
|
)
|
||||||
|
|
||||||
|
# we validate that node_color has a suitable value to prevent dbt-docs from crashing
|
||||||
|
def __post_init__(self):
|
||||||
|
if self.docs.node_color:
|
||||||
|
node_color = self.docs.node_color
|
||||||
|
if not validate_color(node_color):
|
||||||
|
raise ValidationError(
|
||||||
|
f"Invalid color name for docs.node_color: {node_color}. "
|
||||||
|
"It is neither a valid HTML color name nor a valid HEX code."
|
||||||
|
)
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def __pre_deserialize__(cls, data):
|
def __pre_deserialize__(cls, data):
|
||||||
|
|||||||
@@ -157,7 +157,6 @@ class ParsedNodeMixins(dbtClassMixin):
|
|||||||
self.created_at = time.time()
|
self.created_at = time.time()
|
||||||
self.description = patch.description
|
self.description = patch.description
|
||||||
self.columns = patch.columns
|
self.columns = patch.columns
|
||||||
self.docs = patch.docs
|
|
||||||
|
|
||||||
def get_materialization(self):
|
def get_materialization(self):
|
||||||
return self.config.materialized
|
return self.config.materialized
|
||||||
@@ -198,11 +197,12 @@ class ParsedNodeDefaults(NodeInfoMixin, ParsedNodeMandatory):
|
|||||||
tags: List[str] = field(default_factory=list)
|
tags: List[str] = field(default_factory=list)
|
||||||
refs: List[List[str]] = field(default_factory=list)
|
refs: List[List[str]] = field(default_factory=list)
|
||||||
sources: List[List[str]] = field(default_factory=list)
|
sources: List[List[str]] = field(default_factory=list)
|
||||||
|
metrics: List[List[str]] = field(default_factory=list)
|
||||||
depends_on: DependsOn = field(default_factory=DependsOn)
|
depends_on: DependsOn = field(default_factory=DependsOn)
|
||||||
description: str = field(default="")
|
description: str = field(default="")
|
||||||
columns: Dict[str, ColumnInfo] = field(default_factory=dict)
|
columns: Dict[str, ColumnInfo] = field(default_factory=dict)
|
||||||
meta: Dict[str, Any] = field(default_factory=dict)
|
meta: Dict[str, Any] = field(default_factory=dict)
|
||||||
docs: Docs = field(default_factory=Docs)
|
docs: Docs = field(default_factory=lambda: Docs(show=True))
|
||||||
patch_path: Optional[str] = None
|
patch_path: Optional[str] = None
|
||||||
compiled_path: Optional[str] = None
|
compiled_path: Optional[str] = None
|
||||||
build_path: Optional[str] = None
|
build_path: Optional[str] = None
|
||||||
@@ -793,24 +793,32 @@ class ParsedExposure(UnparsedBaseNode, HasUniqueID, HasFqn):
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class MetricReference(dbtClassMixin, Replaceable):
|
||||||
|
sql: Optional[Union[str, int]]
|
||||||
|
unique_id: Optional[str]
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
class ParsedMetric(UnparsedBaseNode, HasUniqueID, HasFqn):
|
class ParsedMetric(UnparsedBaseNode, HasUniqueID, HasFqn):
|
||||||
model: str
|
|
||||||
name: str
|
name: str
|
||||||
description: str
|
description: str
|
||||||
label: str
|
label: str
|
||||||
type: str
|
type: str
|
||||||
sql: Optional[str]
|
sql: str
|
||||||
timestamp: Optional[str]
|
timestamp: Optional[str]
|
||||||
filters: List[MetricFilter]
|
filters: List[MetricFilter]
|
||||||
time_grains: List[str]
|
time_grains: List[str]
|
||||||
dimensions: List[str]
|
dimensions: List[str]
|
||||||
|
model: Optional[str] = None
|
||||||
|
model_unique_id: Optional[str] = None
|
||||||
resource_type: NodeType = NodeType.Metric
|
resource_type: NodeType = NodeType.Metric
|
||||||
meta: Dict[str, Any] = field(default_factory=dict)
|
meta: Dict[str, Any] = field(default_factory=dict)
|
||||||
tags: List[str] = field(default_factory=list)
|
tags: List[str] = field(default_factory=list)
|
||||||
sources: List[List[str]] = field(default_factory=list)
|
sources: List[List[str]] = field(default_factory=list)
|
||||||
depends_on: DependsOn = field(default_factory=DependsOn)
|
depends_on: DependsOn = field(default_factory=DependsOn)
|
||||||
refs: List[List[str]] = field(default_factory=list)
|
refs: List[List[str]] = field(default_factory=list)
|
||||||
|
metrics: List[List[str]] = field(default_factory=list)
|
||||||
created_at: float = field(default_factory=lambda: time.time())
|
created_at: float = field(default_factory=lambda: time.time())
|
||||||
|
|
||||||
@property
|
@property
|
||||||
|
|||||||
@@ -1,15 +1,11 @@
|
|||||||
from dbt.node_types import NodeType
|
from dbt.node_types import NodeType
|
||||||
from dbt.contracts.util import (
|
from dbt.contracts.util import AdditionalPropertiesMixin, Mergeable, Replaceable
|
||||||
AdditionalPropertiesMixin,
|
|
||||||
Mergeable,
|
|
||||||
Replaceable,
|
|
||||||
)
|
|
||||||
|
|
||||||
# trigger the PathEncoder
|
# trigger the PathEncoder
|
||||||
import dbt.helper_types # noqa:F401
|
import dbt.helper_types # noqa:F401
|
||||||
from dbt.exceptions import CompilationException, ParsingException
|
from dbt.exceptions import CompilationException, ParsingException
|
||||||
|
|
||||||
from dbt.dataclass_schema import dbtClassMixin, StrEnum, ExtensibleDbtClassMixin
|
from dbt.dataclass_schema import dbtClassMixin, StrEnum, ExtensibleDbtClassMixin, ValidationError
|
||||||
|
|
||||||
from dataclasses import dataclass, field
|
from dataclasses import dataclass, field
|
||||||
from datetime import timedelta
|
from datetime import timedelta
|
||||||
@@ -80,6 +76,7 @@ class UnparsedRunHook(UnparsedNode):
|
|||||||
@dataclass
|
@dataclass
|
||||||
class Docs(dbtClassMixin, Replaceable):
|
class Docs(dbtClassMixin, Replaceable):
|
||||||
show: bool = True
|
show: bool = True
|
||||||
|
node_color: Optional[str] = None
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
@@ -448,12 +445,15 @@ class MetricFilter(dbtClassMixin, Replaceable):
|
|||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
class UnparsedMetric(dbtClassMixin, Replaceable):
|
class UnparsedMetric(dbtClassMixin, Replaceable):
|
||||||
model: str
|
# TODO : verify that this disallows metric names with spaces
|
||||||
|
# TODO: fix validation that you broke :p
|
||||||
|
# name: Identifier
|
||||||
name: str
|
name: str
|
||||||
label: str
|
label: str
|
||||||
type: str
|
type: str
|
||||||
|
model: Optional[str] = None
|
||||||
description: str = ""
|
description: str = ""
|
||||||
sql: Optional[str] = None
|
sql: Union[str, int] = ""
|
||||||
timestamp: Optional[str] = None
|
timestamp: Optional[str] = None
|
||||||
time_grains: List[str] = field(default_factory=list)
|
time_grains: List[str] = field(default_factory=list)
|
||||||
dimensions: List[str] = field(default_factory=list)
|
dimensions: List[str] = field(default_factory=list)
|
||||||
@@ -463,6 +463,15 @@ class UnparsedMetric(dbtClassMixin, Replaceable):
|
|||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def validate(cls, data):
|
def validate(cls, data):
|
||||||
|
# super().validate(data)
|
||||||
|
# TODO: putting this back for now to get tests passing. Do we want to implement name: Identifier?
|
||||||
super(UnparsedMetric, cls).validate(data)
|
super(UnparsedMetric, cls).validate(data)
|
||||||
if "name" in data and " " in data["name"]:
|
if "name" in data and " " in data["name"]:
|
||||||
raise ParsingException(f"Metrics name '{data['name']}' cannot contain spaces")
|
raise ParsingException(f"Metrics name '{data['name']}' cannot contain spaces")
|
||||||
|
|
||||||
|
# TODO: Expressions _cannot_ have `model` properties
|
||||||
|
if data.get("model") is None and data.get("type") != "expression":
|
||||||
|
raise ValidationError("Non-expression metrics require a 'model' property")
|
||||||
|
|
||||||
|
if data.get("model") is not None and data.get("type") == "expression":
|
||||||
|
raise ValidationError("Expression metrics cannot have a 'model' property")
|
||||||
|
|||||||
153
core/dbt/contracts/graph/utils.py
Normal file
153
core/dbt/contracts/graph/utils.py
Normal file
@@ -0,0 +1,153 @@
|
|||||||
|
import re
|
||||||
|
|
||||||
|
HTML_COLORS = [
|
||||||
|
"aliceblue",
|
||||||
|
"antiquewhite",
|
||||||
|
"aqua",
|
||||||
|
"aquamarine",
|
||||||
|
"azure",
|
||||||
|
"beige",
|
||||||
|
"bisque",
|
||||||
|
"black",
|
||||||
|
"blanchedalmond",
|
||||||
|
"blue",
|
||||||
|
"blueviolet",
|
||||||
|
"brown",
|
||||||
|
"burlywood",
|
||||||
|
"cadetblue",
|
||||||
|
"chartreuse",
|
||||||
|
"chocolate",
|
||||||
|
"coral",
|
||||||
|
"cornflowerblue",
|
||||||
|
"cornsilk",
|
||||||
|
"crimson",
|
||||||
|
"cyan",
|
||||||
|
"darkblue",
|
||||||
|
"darkcyan",
|
||||||
|
"darkgoldenrod",
|
||||||
|
"darkgray",
|
||||||
|
"darkgreen",
|
||||||
|
"darkkhaki",
|
||||||
|
"darkmagenta",
|
||||||
|
"darkolivegreen",
|
||||||
|
"darkorange",
|
||||||
|
"darkorchid",
|
||||||
|
"darkred",
|
||||||
|
"darksalmon",
|
||||||
|
"darkseagreen",
|
||||||
|
"darkslateblue",
|
||||||
|
"darkslategray",
|
||||||
|
"darkturquoise",
|
||||||
|
"darkviolet",
|
||||||
|
"deeppink",
|
||||||
|
"deepskyblue",
|
||||||
|
"dimgray",
|
||||||
|
"dodgerblue",
|
||||||
|
"firebrick",
|
||||||
|
"floralwhite",
|
||||||
|
"forestgreen",
|
||||||
|
"fuchsia",
|
||||||
|
"gainsboro",
|
||||||
|
"ghostwhite",
|
||||||
|
"gold",
|
||||||
|
"goldenrod",
|
||||||
|
"gray",
|
||||||
|
"green",
|
||||||
|
"greenyellow",
|
||||||
|
"honeydew",
|
||||||
|
"hotpink",
|
||||||
|
"indianred",
|
||||||
|
"indigo",
|
||||||
|
"ivory",
|
||||||
|
"khaki",
|
||||||
|
"lavender",
|
||||||
|
"lavenderblush",
|
||||||
|
"lawngreen",
|
||||||
|
"lemonchiffon",
|
||||||
|
"lightblue",
|
||||||
|
"lightcoral",
|
||||||
|
"lightcyan",
|
||||||
|
"lightgoldenrodyellow",
|
||||||
|
"lightgray",
|
||||||
|
"lightgreen",
|
||||||
|
"lightpink",
|
||||||
|
"lightsalmon",
|
||||||
|
"lightsalmon",
|
||||||
|
"lightseagreen",
|
||||||
|
"lightskyblue",
|
||||||
|
"lightslategray",
|
||||||
|
"lightsteelblue",
|
||||||
|
"lightyellow",
|
||||||
|
"lime",
|
||||||
|
"limegreen",
|
||||||
|
"linen",
|
||||||
|
"magenta",
|
||||||
|
"maroon",
|
||||||
|
"mediumaquamarine",
|
||||||
|
"mediumblue",
|
||||||
|
"mediumorchid",
|
||||||
|
"mediumpurple",
|
||||||
|
"mediumseagreen",
|
||||||
|
"mediumslateblue",
|
||||||
|
"mediumslateblue",
|
||||||
|
"mediumspringgreen",
|
||||||
|
"mediumturquoise",
|
||||||
|
"mediumvioletred",
|
||||||
|
"midnightblue",
|
||||||
|
"mintcream",
|
||||||
|
"mistyrose",
|
||||||
|
"moccasin",
|
||||||
|
"navajowhite",
|
||||||
|
"navy",
|
||||||
|
"oldlace",
|
||||||
|
"olive",
|
||||||
|
"olivedrab",
|
||||||
|
"orange",
|
||||||
|
"orangered",
|
||||||
|
"orchid",
|
||||||
|
"palegoldenrod",
|
||||||
|
"palegreen",
|
||||||
|
"paleturquoise",
|
||||||
|
"palevioletred",
|
||||||
|
"papayawhip",
|
||||||
|
"peachpuff",
|
||||||
|
"peru",
|
||||||
|
"pink",
|
||||||
|
"plum",
|
||||||
|
"powderblue",
|
||||||
|
"purple",
|
||||||
|
"rebeccapurple",
|
||||||
|
"red",
|
||||||
|
"rosybrown",
|
||||||
|
"royalblue",
|
||||||
|
"saddlebrown",
|
||||||
|
"salmon",
|
||||||
|
"sandybrown",
|
||||||
|
"seagreen",
|
||||||
|
"seashell",
|
||||||
|
"sienna",
|
||||||
|
"silver",
|
||||||
|
"skyblue",
|
||||||
|
"slateblue",
|
||||||
|
"slategray",
|
||||||
|
"snow",
|
||||||
|
"springgreen",
|
||||||
|
"steelblue",
|
||||||
|
"tan",
|
||||||
|
"teal",
|
||||||
|
"thistle",
|
||||||
|
"tomato",
|
||||||
|
"turquoise",
|
||||||
|
"violet",
|
||||||
|
"wheat",
|
||||||
|
"white",
|
||||||
|
"whitesmoke",
|
||||||
|
"yellow",
|
||||||
|
"yellowgreen",
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def validate_color(color: str) -> bool:
|
||||||
|
match_hex = re.search(r"^#(?:[0-9a-f]{3}){1,2}$", color.lower())
|
||||||
|
match_html_color_name = color.lower() in HTML_COLORS
|
||||||
|
return bool(match_hex or match_html_color_name)
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
from dbt.contracts.util import Replaceable, Mergeable, list_str
|
from dbt.contracts.util import Replaceable, Mergeable, list_str, Identifier
|
||||||
from dbt.contracts.connection import QueryComment, UserConfigContract
|
from dbt.contracts.connection import QueryComment, UserConfigContract
|
||||||
from dbt.helper_types import NoValue
|
from dbt.helper_types import NoValue
|
||||||
from dbt.dataclass_schema import (
|
from dbt.dataclass_schema import (
|
||||||
@@ -7,7 +7,6 @@ from dbt.dataclass_schema import (
|
|||||||
HyphenatedDbtClassMixin,
|
HyphenatedDbtClassMixin,
|
||||||
ExtensibleDbtClassMixin,
|
ExtensibleDbtClassMixin,
|
||||||
register_pattern,
|
register_pattern,
|
||||||
ValidatedStringMixin,
|
|
||||||
)
|
)
|
||||||
from dataclasses import dataclass, field
|
from dataclasses import dataclass, field
|
||||||
from typing import Optional, List, Dict, Union, Any
|
from typing import Optional, List, Dict, Union, Any
|
||||||
@@ -19,25 +18,6 @@ PIN_PACKAGE_URL = (
|
|||||||
DEFAULT_SEND_ANONYMOUS_USAGE_STATS = True
|
DEFAULT_SEND_ANONYMOUS_USAGE_STATS = True
|
||||||
|
|
||||||
|
|
||||||
class Name(ValidatedStringMixin):
|
|
||||||
ValidationRegex = r"^[^\d\W]\w*$"
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def is_valid(cls, value: Any) -> bool:
|
|
||||||
if not isinstance(value, str):
|
|
||||||
return False
|
|
||||||
|
|
||||||
try:
|
|
||||||
cls.validate(value)
|
|
||||||
except ValidationError:
|
|
||||||
return False
|
|
||||||
|
|
||||||
return True
|
|
||||||
|
|
||||||
|
|
||||||
register_pattern(Name, r"^[^\d\W]\w*$")
|
|
||||||
|
|
||||||
|
|
||||||
class SemverString(str, SerializableType):
|
class SemverString(str, SerializableType):
|
||||||
def _serialize(self) -> str:
|
def _serialize(self) -> str:
|
||||||
return self
|
return self
|
||||||
@@ -182,7 +162,7 @@ BANNED_PROJECT_NAMES = {
|
|||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
class Project(HyphenatedDbtClassMixin, Replaceable):
|
class Project(HyphenatedDbtClassMixin, Replaceable):
|
||||||
name: Name
|
name: Identifier
|
||||||
version: Union[SemverString, float]
|
version: Union[SemverString, float]
|
||||||
config_version: int
|
config_version: int
|
||||||
project_root: Optional[str] = None
|
project_root: Optional[str] = None
|
||||||
|
|||||||
@@ -9,6 +9,13 @@ from dbt.version import __version__
|
|||||||
from dbt.events.functions import get_invocation_id
|
from dbt.events.functions import get_invocation_id
|
||||||
from dbt.dataclass_schema import dbtClassMixin
|
from dbt.dataclass_schema import dbtClassMixin
|
||||||
|
|
||||||
|
from dbt.dataclass_schema import (
|
||||||
|
ValidatedStringMixin,
|
||||||
|
ValidationError,
|
||||||
|
register_pattern,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
SourceKey = Tuple[str, str]
|
SourceKey = Tuple[str, str]
|
||||||
|
|
||||||
|
|
||||||
@@ -250,3 +257,22 @@ class ArtifactMixin(VersionedSchema, Writable, Readable):
|
|||||||
super().validate(data)
|
super().validate(data)
|
||||||
if cls.dbt_schema_version is None:
|
if cls.dbt_schema_version is None:
|
||||||
raise InternalException("Cannot call from_dict with no schema version!")
|
raise InternalException("Cannot call from_dict with no schema version!")
|
||||||
|
|
||||||
|
|
||||||
|
class Identifier(ValidatedStringMixin):
|
||||||
|
ValidationRegex = r"^[^\d\W]\w*$"
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def is_valid(cls, value: Any) -> bool:
|
||||||
|
if not isinstance(value, str):
|
||||||
|
return False
|
||||||
|
|
||||||
|
try:
|
||||||
|
cls.validate(value)
|
||||||
|
except ValidationError:
|
||||||
|
return False
|
||||||
|
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
register_pattern(Identifier, r"^[^\d\W]\w*$")
|
||||||
|
|||||||
@@ -520,6 +520,12 @@ def ref_invalid_args(model, args) -> NoReturn:
|
|||||||
raise_compiler_error("ref() takes at most two arguments ({} given)".format(len(args)), model)
|
raise_compiler_error("ref() takes at most two arguments ({} given)".format(len(args)), model)
|
||||||
|
|
||||||
|
|
||||||
|
def metric_invalid_args(model, args) -> NoReturn:
|
||||||
|
raise_compiler_error(
|
||||||
|
"metric() takes at most two arguments ({} given)".format(len(args)), model
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def ref_bad_context(model, args) -> NoReturn:
|
def ref_bad_context(model, args) -> NoReturn:
|
||||||
ref_args = ", ".join("'{}'".format(a) for a in args)
|
ref_args = ", ".join("'{}'".format(a) for a in args)
|
||||||
ref_string = "{{{{ ref({}) }}}}".format(ref_args)
|
ref_string = "{{{{ ref({}) }}}}".format(ref_args)
|
||||||
@@ -650,6 +656,23 @@ def source_target_not_found(
|
|||||||
raise_compiler_error(msg, model)
|
raise_compiler_error(msg, model)
|
||||||
|
|
||||||
|
|
||||||
|
def get_metric_not_found_msg(
|
||||||
|
model,
|
||||||
|
target_name: str,
|
||||||
|
target_package: Optional[str],
|
||||||
|
) -> str:
|
||||||
|
reason = "was not found"
|
||||||
|
return _get_target_failure_msg(
|
||||||
|
model, target_name, target_package, include_path=True, reason=reason, target_kind="metric"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def metric_target_not_found(metric, target_name: str, target_package: Optional[str]) -> NoReturn:
|
||||||
|
msg = get_metric_not_found_msg(metric, target_name, target_package)
|
||||||
|
|
||||||
|
raise_compiler_error(msg, metric)
|
||||||
|
|
||||||
|
|
||||||
def dependency_not_found(model, target_model_name):
|
def dependency_not_found(model, target_model_name):
|
||||||
raise_compiler_error(
|
raise_compiler_error(
|
||||||
"'{}' depends on '{}' which is not in the graph!".format(
|
"'{}' depends on '{}' which is not in the graph!".format(
|
||||||
|
|||||||
@@ -37,6 +37,8 @@ EVENT_BUFFER_SIZE = 100000
|
|||||||
QUIET = None
|
QUIET = None
|
||||||
NO_PRINT = None
|
NO_PRINT = None
|
||||||
CACHE_SELECTED_ONLY = None
|
CACHE_SELECTED_ONLY = None
|
||||||
|
TARGET_PATH = None
|
||||||
|
LOG_PATH = None
|
||||||
|
|
||||||
_NON_BOOLEAN_FLAGS = [
|
_NON_BOOLEAN_FLAGS = [
|
||||||
"LOG_FORMAT",
|
"LOG_FORMAT",
|
||||||
@@ -44,6 +46,8 @@ _NON_BOOLEAN_FLAGS = [
|
|||||||
"PROFILES_DIR",
|
"PROFILES_DIR",
|
||||||
"INDIRECT_SELECTION",
|
"INDIRECT_SELECTION",
|
||||||
"EVENT_BUFFER_SIZE",
|
"EVENT_BUFFER_SIZE",
|
||||||
|
"TARGET_PATH",
|
||||||
|
"LOG_PATH",
|
||||||
]
|
]
|
||||||
|
|
||||||
_NON_DBT_ENV_FLAGS = ["DO_NOT_TRACK"]
|
_NON_DBT_ENV_FLAGS = ["DO_NOT_TRACK"]
|
||||||
@@ -71,6 +75,8 @@ flag_defaults = {
|
|||||||
"QUIET": False,
|
"QUIET": False,
|
||||||
"NO_PRINT": False,
|
"NO_PRINT": False,
|
||||||
"CACHE_SELECTED_ONLY": False,
|
"CACHE_SELECTED_ONLY": False,
|
||||||
|
"TARGET_PATH": None,
|
||||||
|
"LOG_PATH": None,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@@ -121,6 +127,7 @@ def set_from_args(args, user_config):
|
|||||||
global WRITE_JSON, PARTIAL_PARSE, USE_COLORS, STORE_FAILURES, PROFILES_DIR, DEBUG, LOG_FORMAT
|
global WRITE_JSON, PARTIAL_PARSE, USE_COLORS, STORE_FAILURES, PROFILES_DIR, DEBUG, LOG_FORMAT
|
||||||
global INDIRECT_SELECTION, VERSION_CHECK, FAIL_FAST, SEND_ANONYMOUS_USAGE_STATS
|
global INDIRECT_SELECTION, VERSION_CHECK, FAIL_FAST, SEND_ANONYMOUS_USAGE_STATS
|
||||||
global PRINTER_WIDTH, WHICH, LOG_CACHE_EVENTS, EVENT_BUFFER_SIZE, QUIET, NO_PRINT, CACHE_SELECTED_ONLY
|
global PRINTER_WIDTH, WHICH, LOG_CACHE_EVENTS, EVENT_BUFFER_SIZE, QUIET, NO_PRINT, CACHE_SELECTED_ONLY
|
||||||
|
global TARGET_PATH, LOG_PATH
|
||||||
|
|
||||||
STRICT_MODE = False # backwards compatibility
|
STRICT_MODE = False # backwards compatibility
|
||||||
# cli args without user_config or env var option
|
# cli args without user_config or env var option
|
||||||
@@ -148,6 +155,8 @@ def set_from_args(args, user_config):
|
|||||||
QUIET = get_flag_value("QUIET", args, user_config)
|
QUIET = get_flag_value("QUIET", args, user_config)
|
||||||
NO_PRINT = get_flag_value("NO_PRINT", args, user_config)
|
NO_PRINT = get_flag_value("NO_PRINT", args, user_config)
|
||||||
CACHE_SELECTED_ONLY = get_flag_value("CACHE_SELECTED_ONLY", args, user_config)
|
CACHE_SELECTED_ONLY = get_flag_value("CACHE_SELECTED_ONLY", args, user_config)
|
||||||
|
TARGET_PATH = get_flag_value("TARGET_PATH", args, user_config)
|
||||||
|
LOG_PATH = get_flag_value("LOG_PATH", args, user_config)
|
||||||
|
|
||||||
_set_overrides_from_env()
|
_set_overrides_from_env()
|
||||||
|
|
||||||
|
|||||||
167
core/dbt/include/global_project/macros/adapters/apply_grants.sql
Normal file
167
core/dbt/include/global_project/macros/adapters/apply_grants.sql
Normal file
@@ -0,0 +1,167 @@
|
|||||||
|
{# ------- BOOLEAN MACROS --------- #}
|
||||||
|
|
||||||
|
{#
|
||||||
|
-- COPY GRANTS
|
||||||
|
-- When a relational object (view or table) is replaced in this database,
|
||||||
|
-- do previous grants carry over to the new object? This may depend on:
|
||||||
|
-- whether we use alter-rename-swap versus CREATE OR REPLACE
|
||||||
|
-- user-supplied configuration (e.g. copy_grants on Snowflake)
|
||||||
|
-- By default, play it safe, assume TRUE: that grants ARE copied over.
|
||||||
|
-- This means dbt will first "show" current grants and then calculate diffs.
|
||||||
|
-- It may require an additional query than is strictly necessary,
|
||||||
|
-- but better safe than sorry.
|
||||||
|
#}
|
||||||
|
|
||||||
|
{% macro copy_grants() %}
|
||||||
|
{{ return(adapter.dispatch('copy_grants', 'dbt')()) }}
|
||||||
|
{% endmacro %}
|
||||||
|
|
||||||
|
{% macro default__copy_grants() %}
|
||||||
|
{{ return(True) }}
|
||||||
|
{% endmacro %}
|
||||||
|
|
||||||
|
|
||||||
|
{#
|
||||||
|
-- SUPPORT MULTIPLE GRANTEES PER DCL STATEMENT
|
||||||
|
-- Does this database support 'grant {privilege} to {grantee_1}, {grantee_2}, ...'
|
||||||
|
-- Or must these be separate statements:
|
||||||
|
-- `grant {privilege} to {grantee_1}`;
|
||||||
|
-- `grant {privilege} to {grantee_2}`;
|
||||||
|
-- By default, pick the former, because it's what we prefer when available.
|
||||||
|
#}
|
||||||
|
|
||||||
|
{% macro support_multiple_grantees_per_dcl_statement() %}
|
||||||
|
{{ return(adapter.dispatch('support_multiple_grantees_per_dcl_statement', 'dbt')()) }}
|
||||||
|
{% endmacro %}
|
||||||
|
|
||||||
|
{%- macro default__support_multiple_grantees_per_dcl_statement() -%}
|
||||||
|
{{ return(True) }}
|
||||||
|
{%- endmacro -%}
|
||||||
|
|
||||||
|
|
||||||
|
{% macro should_revoke(existing_relation, full_refresh_mode=True) %}
|
||||||
|
|
||||||
|
{% if not existing_relation %}
|
||||||
|
{#-- The table doesn't already exist, so no grants to copy over --#}
|
||||||
|
{{ return(False) }}
|
||||||
|
{% elif full_refresh_mode %}
|
||||||
|
{#-- The object is being REPLACED -- whether grants are copied over depends on the value of user config --#}
|
||||||
|
{{ return(copy_grants()) }}
|
||||||
|
{% else %}
|
||||||
|
{#-- The table is being merged/upserted/inserted -- grants will be carried over --#}
|
||||||
|
{{ return(True) }}
|
||||||
|
{% endif %}
|
||||||
|
|
||||||
|
{% endmacro %}
|
||||||
|
|
||||||
|
{# ------- DCL STATEMENT TEMPLATES --------- #}
|
||||||
|
|
||||||
|
{% macro get_show_grant_sql(relation) %}
|
||||||
|
{{ return(adapter.dispatch("get_show_grant_sql", "dbt")(relation)) }}
|
||||||
|
{% endmacro %}
|
||||||
|
|
||||||
|
{% macro default__get_show_grant_sql(relation) %}
|
||||||
|
show grants on {{ relation }}
|
||||||
|
{% endmacro %}
|
||||||
|
|
||||||
|
|
||||||
|
{% macro get_grant_sql(relation, privilege, grantees) %}
|
||||||
|
{{ return(adapter.dispatch('get_grant_sql', 'dbt')(relation, privilege, grantees)) }}
|
||||||
|
{% endmacro %}
|
||||||
|
|
||||||
|
{%- macro default__get_grant_sql(relation, privilege, grantees) -%}
|
||||||
|
grant {{ privilege }} on {{ relation }} to {{ grantees | join(', ') }}
|
||||||
|
{%- endmacro -%}
|
||||||
|
|
||||||
|
|
||||||
|
{% macro get_revoke_sql(relation, privilege, grantees) %}
|
||||||
|
{{ return(adapter.dispatch('get_revoke_sql', 'dbt')(relation, privilege, grantees)) }}
|
||||||
|
{% endmacro %}
|
||||||
|
|
||||||
|
{%- macro default__get_revoke_sql(relation, privilege, grantees) -%}
|
||||||
|
revoke {{ privilege }} on {{ relation }} from {{ grantees | join(', ') }}
|
||||||
|
{%- endmacro -%}
|
||||||
|
|
||||||
|
|
||||||
|
{# ------- RUNTIME APPLICATION --------- #}
|
||||||
|
|
||||||
|
{% macro get_dcl_statement_list(relation, grant_config, get_dcl_macro) %}
|
||||||
|
{{ return(adapter.dispatch('get_dcl_statement_list', 'dbt')(relation, grant_config, get_dcl_macro)) }}
|
||||||
|
{% endmacro %}
|
||||||
|
|
||||||
|
{%- macro default__get_dcl_statement_list(relation, grant_config, get_dcl_macro) -%}
|
||||||
|
{#
|
||||||
|
-- Unpack grant_config into specific privileges and the set of users who need them granted/revoked.
|
||||||
|
-- Depending on whether this database supports multiple grantees per statement, pass in the list of
|
||||||
|
-- all grantees per privilege, or (if not) template one statement per privilege-grantee pair.
|
||||||
|
-- `get_dcl_macro` will be either `get_grant_sql` or `get_revoke_sql`
|
||||||
|
#}
|
||||||
|
{%- set dcl_statements = [] -%}
|
||||||
|
{%- for privilege, grantees in grant_config.items() %}
|
||||||
|
{%- if support_multiple_grantees_per_dcl_statement() and grantees -%}
|
||||||
|
{%- set dcl = get_dcl_macro(relation, privilege, grantees) -%}
|
||||||
|
{%- do dcl_statements.append(dcl) -%}
|
||||||
|
{%- else -%}
|
||||||
|
{%- for grantee in grantees -%}
|
||||||
|
{% set dcl = get_dcl_macro(relation, privilege, [grantee]) %}
|
||||||
|
{%- do dcl_statements.append(dcl) -%}
|
||||||
|
{% endfor -%}
|
||||||
|
{%- endif -%}
|
||||||
|
{%- endfor -%}
|
||||||
|
{{ return(dcl_statements) }}
|
||||||
|
{%- endmacro %}
|
||||||
|
|
||||||
|
|
||||||
|
{% macro call_dcl_statements(dcl_statement_list) %}
|
||||||
|
{{ return(adapter.dispatch("call_dcl_statements", "dbt")(dcl_statement_list)) }}
|
||||||
|
{% endmacro %}
|
||||||
|
|
||||||
|
{% macro default__call_dcl_statements(dcl_statement_list) %}
|
||||||
|
{#
|
||||||
|
-- By default, supply all grant + revoke statements in a single semicolon-separated block,
|
||||||
|
-- so that they're all processed together.
|
||||||
|
|
||||||
|
-- Some databases do not support this. Those adapters will need to override this macro
|
||||||
|
-- to run each statement individually.
|
||||||
|
#}
|
||||||
|
{% call statement('grants') %}
|
||||||
|
{% for dcl_statement in dcl_statement_list %}
|
||||||
|
{{ dcl_statement }};
|
||||||
|
{% endfor %}
|
||||||
|
{% endcall %}
|
||||||
|
{% endmacro %}
|
||||||
|
|
||||||
|
|
||||||
|
{% macro apply_grants(relation, grant_config, should_revoke) %}
|
||||||
|
{{ return(adapter.dispatch("apply_grants", "dbt")(relation, grant_config, should_revoke)) }}
|
||||||
|
{% endmacro %}
|
||||||
|
|
||||||
|
{% macro default__apply_grants(relation, grant_config, should_revoke=True) %}
|
||||||
|
{#-- If grant_config is {} or None, this is a no-op --#}
|
||||||
|
{% if grant_config %}
|
||||||
|
{% if should_revoke %}
|
||||||
|
{#-- We think previous grants may have carried over --#}
|
||||||
|
{#-- Show current grants and calculate diffs --#}
|
||||||
|
{% set current_grants_table = run_query(get_show_grant_sql(relation)) %}
|
||||||
|
{% set current_grants_dict = adapter.standardize_grants_dict(current_grants_table) %}
|
||||||
|
{% set needs_granting = diff_of_two_dicts(grant_config, current_grants_dict) %}
|
||||||
|
{% set needs_revoking = diff_of_two_dicts(current_grants_dict, grant_config) %}
|
||||||
|
{% if not (needs_granting or needs_revoking) %}
|
||||||
|
{{ log('On ' ~ relation ~': All grants are in place, no revocation or granting needed.')}}
|
||||||
|
{% endif %}
|
||||||
|
{% else %}
|
||||||
|
{#-- We don't think there's any chance of previous grants having carried over. --#}
|
||||||
|
{#-- Jump straight to granting what the user has configured. --#}
|
||||||
|
{% set needs_revoking = {} %}
|
||||||
|
{% set needs_granting = grant_config %}
|
||||||
|
{% endif %}
|
||||||
|
{% if needs_granting or needs_revoking %}
|
||||||
|
{% set revoke_statement_list = get_dcl_statement_list(relation, needs_revoking, get_revoke_sql) %}
|
||||||
|
{% set grant_statement_list = get_dcl_statement_list(relation, needs_granting, get_grant_sql) %}
|
||||||
|
{% set dcl_statement_list = revoke_statement_list + grant_statement_list %}
|
||||||
|
{% if dcl_statement_list %}
|
||||||
|
{{ call_dcl_statements(dcl_statement_list) }}
|
||||||
|
{% endif %}
|
||||||
|
{% endif %}
|
||||||
|
{% endif %}
|
||||||
|
{% endmacro %}
|
||||||
@@ -20,6 +20,8 @@
|
|||||||
-- BEGIN, in a separate transaction
|
-- BEGIN, in a separate transaction
|
||||||
{%- set preexisting_intermediate_relation = load_cached_relation(intermediate_relation)-%}
|
{%- set preexisting_intermediate_relation = load_cached_relation(intermediate_relation)-%}
|
||||||
{%- set preexisting_backup_relation = load_cached_relation(backup_relation) -%}
|
{%- set preexisting_backup_relation = load_cached_relation(backup_relation) -%}
|
||||||
|
-- grab current tables grants config for comparision later on
|
||||||
|
{% set grant_config = config.get('grants') %}
|
||||||
{{ drop_relation_if_exists(preexisting_intermediate_relation) }}
|
{{ drop_relation_if_exists(preexisting_intermediate_relation) }}
|
||||||
{{ drop_relation_if_exists(preexisting_backup_relation) }}
|
{{ drop_relation_if_exists(preexisting_backup_relation) }}
|
||||||
|
|
||||||
@@ -59,6 +61,9 @@
|
|||||||
{% do to_drop.append(backup_relation) %}
|
{% do to_drop.append(backup_relation) %}
|
||||||
{% endif %}
|
{% endif %}
|
||||||
|
|
||||||
|
{% set should_revoke = should_revoke(existing_relation, full_refresh_mode) %}
|
||||||
|
{% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}
|
||||||
|
|
||||||
{% do persist_docs(target_relation, model) %}
|
{% do persist_docs(target_relation, model) %}
|
||||||
|
|
||||||
{% if existing_relation is none or existing_relation.is_view or should_full_refresh() %}
|
{% if existing_relation is none or existing_relation.is_view or should_full_refresh() %}
|
||||||
|
|||||||
@@ -14,6 +14,8 @@
|
|||||||
{%- set backup_relation = make_backup_relation(target_relation, backup_relation_type) -%}
|
{%- set backup_relation = make_backup_relation(target_relation, backup_relation_type) -%}
|
||||||
-- as above, the backup_relation should not already exist
|
-- as above, the backup_relation should not already exist
|
||||||
{%- set preexisting_backup_relation = load_cached_relation(backup_relation) -%}
|
{%- set preexisting_backup_relation = load_cached_relation(backup_relation) -%}
|
||||||
|
-- grab current tables grants config for comparision later on
|
||||||
|
{% set grant_config = config.get('grants') %}
|
||||||
|
|
||||||
-- drop the temp relations if they exist already in the database
|
-- drop the temp relations if they exist already in the database
|
||||||
{{ drop_relation_if_exists(preexisting_intermediate_relation) }}
|
{{ drop_relation_if_exists(preexisting_intermediate_relation) }}
|
||||||
@@ -40,6 +42,9 @@
|
|||||||
|
|
||||||
{{ run_hooks(post_hooks, inside_transaction=True) }}
|
{{ run_hooks(post_hooks, inside_transaction=True) }}
|
||||||
|
|
||||||
|
{% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}
|
||||||
|
{% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}
|
||||||
|
|
||||||
{% do persist_docs(target_relation, model) %}
|
{% do persist_docs(target_relation, model) %}
|
||||||
|
|
||||||
-- `COMMIT` happens here
|
-- `COMMIT` happens here
|
||||||
|
|||||||
@@ -13,12 +13,12 @@
|
|||||||
{%- set identifier = model['alias'] -%}
|
{%- set identifier = model['alias'] -%}
|
||||||
|
|
||||||
{%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}
|
{%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}
|
||||||
|
|
||||||
{%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%}
|
{%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%}
|
||||||
|
|
||||||
{%- set target_relation = api.Relation.create(
|
{%- set target_relation = api.Relation.create(
|
||||||
identifier=identifier, schema=schema, database=database,
|
identifier=identifier, schema=schema, database=database,
|
||||||
type='view') -%}
|
type='view') -%}
|
||||||
|
{% set grant_config = config.get('grants') %}
|
||||||
|
|
||||||
{{ run_hooks(pre_hooks) }}
|
{{ run_hooks(pre_hooks) }}
|
||||||
|
|
||||||
@@ -34,6 +34,9 @@
|
|||||||
{{ get_create_view_as_sql(target_relation, sql) }}
|
{{ get_create_view_as_sql(target_relation, sql) }}
|
||||||
{%- endcall %}
|
{%- endcall %}
|
||||||
|
|
||||||
|
{% set should_revoke = should_revoke(exists_as_view, full_refresh_mode=True) %}
|
||||||
|
{% do apply_grants(target_relation, grant_config, should_revoke=True) %}
|
||||||
|
|
||||||
{{ run_hooks(post_hooks) }}
|
{{ run_hooks(post_hooks) }}
|
||||||
|
|
||||||
{{ return({'relations': [target_relation]}) }}
|
{{ return({'relations': [target_relation]}) }}
|
||||||
|
|||||||
@@ -25,6 +25,8 @@
|
|||||||
{%- set backup_relation = make_backup_relation(target_relation, backup_relation_type) -%}
|
{%- set backup_relation = make_backup_relation(target_relation, backup_relation_type) -%}
|
||||||
-- as above, the backup_relation should not already exist
|
-- as above, the backup_relation should not already exist
|
||||||
{%- set preexisting_backup_relation = load_cached_relation(backup_relation) -%}
|
{%- set preexisting_backup_relation = load_cached_relation(backup_relation) -%}
|
||||||
|
-- grab current tables grants config for comparision later on
|
||||||
|
{% set grant_config = config.get('grants') %}
|
||||||
|
|
||||||
{{ run_hooks(pre_hooks, inside_transaction=False) }}
|
{{ run_hooks(pre_hooks, inside_transaction=False) }}
|
||||||
|
|
||||||
@@ -47,6 +49,9 @@
|
|||||||
{% endif %}
|
{% endif %}
|
||||||
{{ adapter.rename_relation(intermediate_relation, target_relation) }}
|
{{ adapter.rename_relation(intermediate_relation, target_relation) }}
|
||||||
|
|
||||||
|
{% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}
|
||||||
|
{% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}
|
||||||
|
|
||||||
{% do persist_docs(target_relation, model) %}
|
{% do persist_docs(target_relation, model) %}
|
||||||
|
|
||||||
{{ run_hooks(post_hooks, inside_transaction=True) }}
|
{{ run_hooks(post_hooks, inside_transaction=True) }}
|
||||||
|
|||||||
@@ -8,7 +8,10 @@
|
|||||||
{%- set exists_as_table = (old_relation is not none and old_relation.is_table) -%}
|
{%- set exists_as_table = (old_relation is not none and old_relation.is_table) -%}
|
||||||
{%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%}
|
{%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%}
|
||||||
|
|
||||||
|
{%- set grant_config = config.get('grants') -%}
|
||||||
{%- set agate_table = load_agate_table() -%}
|
{%- set agate_table = load_agate_table() -%}
|
||||||
|
-- grab current tables grants config for comparision later on
|
||||||
|
|
||||||
{%- do store_result('agate_table', response='OK', agate_table=agate_table) -%}
|
{%- do store_result('agate_table', response='OK', agate_table=agate_table) -%}
|
||||||
|
|
||||||
{{ run_hooks(pre_hooks, inside_transaction=False) }}
|
{{ run_hooks(pre_hooks, inside_transaction=False) }}
|
||||||
@@ -35,6 +38,10 @@
|
|||||||
{% endcall %}
|
{% endcall %}
|
||||||
|
|
||||||
{% set target_relation = this.incorporate(type='table') %}
|
{% set target_relation = this.incorporate(type='table') %}
|
||||||
|
|
||||||
|
{% set should_revoke = should_revoke(old_relation, full_refresh_mode) %}
|
||||||
|
{% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}
|
||||||
|
|
||||||
{% do persist_docs(target_relation, model) %}
|
{% do persist_docs(target_relation, model) %}
|
||||||
|
|
||||||
{% if full_refresh_mode or not exists_as_table %}
|
{% if full_refresh_mode or not exists_as_table %}
|
||||||
|
|||||||
@@ -5,6 +5,8 @@
|
|||||||
|
|
||||||
{%- set strategy_name = config.get('strategy') -%}
|
{%- set strategy_name = config.get('strategy') -%}
|
||||||
{%- set unique_key = config.get('unique_key') %}
|
{%- set unique_key = config.get('unique_key') %}
|
||||||
|
-- grab current tables grants config for comparision later on
|
||||||
|
{%- set grant_config = config.get('grants') -%}
|
||||||
|
|
||||||
{% set target_relation_exists, target_relation = get_or_create_relation(
|
{% set target_relation_exists, target_relation = get_or_create_relation(
|
||||||
database=model.database,
|
database=model.database,
|
||||||
@@ -73,6 +75,9 @@
|
|||||||
{{ final_sql }}
|
{{ final_sql }}
|
||||||
{% endcall %}
|
{% endcall %}
|
||||||
|
|
||||||
|
{% set should_revoke = should_revoke(target_relation_exists, full_refresh_mode=False) %}
|
||||||
|
{% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}
|
||||||
|
|
||||||
{% do persist_docs(target_relation, model) %}
|
{% do persist_docs(target_relation, model) %}
|
||||||
|
|
||||||
{% if not target_relation_exists %}
|
{% if not target_relation_exists %}
|
||||||
|
|||||||
File diff suppressed because one or more lines are too long
@@ -142,6 +142,7 @@ def main(args=None):
|
|||||||
exit_code = e.code
|
exit_code = e.code
|
||||||
|
|
||||||
except BaseException as e:
|
except BaseException as e:
|
||||||
|
traceback.print_exc()
|
||||||
fire_event(MainEncounteredError(e=str(e)))
|
fire_event(MainEncounteredError(e=str(e)))
|
||||||
fire_event(MainStackTrace(stack_trace=traceback.format_exc()))
|
fire_event(MainStackTrace(stack_trace=traceback.format_exc()))
|
||||||
exit_code = ExitCodes.UnhandledError.value
|
exit_code = ExitCodes.UnhandledError.value
|
||||||
@@ -651,6 +652,22 @@ def _add_common_arguments(*subparsers):
|
|||||||
settings in profiles.yml.
|
settings in profiles.yml.
|
||||||
""",
|
""",
|
||||||
)
|
)
|
||||||
|
sub.add_argument(
|
||||||
|
"--target-path",
|
||||||
|
required=False,
|
||||||
|
help="""
|
||||||
|
Configure the 'target-path'. Only applies this setting for the
|
||||||
|
current run. Overrides the 'DBT_TARGET_PATH' if it is set.
|
||||||
|
""",
|
||||||
|
)
|
||||||
|
sub.add_argument(
|
||||||
|
"--log-path",
|
||||||
|
required=False,
|
||||||
|
help="""
|
||||||
|
Configure the 'log-path'. Only applies this setting for the
|
||||||
|
current run. Overrides the 'DBT_LOG_PATH' if it is set.
|
||||||
|
""",
|
||||||
|
)
|
||||||
_add_version_check(sub)
|
_add_version_check(sub)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -17,7 +17,7 @@ from dbt.config import Project, RuntimeConfig
|
|||||||
from dbt.context.context_config import ContextConfig
|
from dbt.context.context_config import ContextConfig
|
||||||
from dbt.contracts.graph.manifest import Manifest
|
from dbt.contracts.graph.manifest import Manifest
|
||||||
from dbt.contracts.graph.parsed import HasUniqueID, ManifestNodes
|
from dbt.contracts.graph.parsed import HasUniqueID, ManifestNodes
|
||||||
from dbt.contracts.graph.unparsed import UnparsedNode
|
from dbt.contracts.graph.unparsed import UnparsedNode, Docs
|
||||||
from dbt.exceptions import ParsingException, validator_error_message, InternalException
|
from dbt.exceptions import ParsingException, validator_error_message, InternalException
|
||||||
from dbt import hooks
|
from dbt import hooks
|
||||||
from dbt.node_types import NodeType
|
from dbt.node_types import NodeType
|
||||||
@@ -287,6 +287,11 @@ class ConfiguredParser(
|
|||||||
if "meta" in config_dict and config_dict["meta"]:
|
if "meta" in config_dict and config_dict["meta"]:
|
||||||
parsed_node.meta = config_dict["meta"]
|
parsed_node.meta = config_dict["meta"]
|
||||||
|
|
||||||
|
# If we have docs in the config, copy to node level, for backwards
|
||||||
|
# compatibility with earlier node-only config.
|
||||||
|
if "docs" in config_dict and config_dict["docs"]:
|
||||||
|
parsed_node.docs = Docs(config_dict["docs"])
|
||||||
|
|
||||||
# unrendered_config is used to compare the original database/schema/alias
|
# unrendered_config is used to compare the original database/schema/alias
|
||||||
# values and to handle 'same_config' and 'same_contents' calls
|
# values and to handle 'same_config' and 'same_contents' calls
|
||||||
parsed_node.unrendered_config = config.build_config_dict(rendered=False)
|
parsed_node.unrendered_config = config.build_config_dict(rendered=False)
|
||||||
|
|||||||
@@ -73,6 +73,7 @@ from dbt.exceptions import (
|
|||||||
ref_target_not_found,
|
ref_target_not_found,
|
||||||
get_target_not_found_or_disabled_msg,
|
get_target_not_found_or_disabled_msg,
|
||||||
source_target_not_found,
|
source_target_not_found,
|
||||||
|
metric_target_not_found,
|
||||||
get_source_not_found_or_disabled_msg,
|
get_source_not_found_or_disabled_msg,
|
||||||
warn_or_error,
|
warn_or_error,
|
||||||
)
|
)
|
||||||
@@ -389,6 +390,7 @@ class ManifestLoader:
|
|||||||
self.process_sources(self.root_project.project_name)
|
self.process_sources(self.root_project.project_name)
|
||||||
self.process_refs(self.root_project.project_name)
|
self.process_refs(self.root_project.project_name)
|
||||||
self.process_docs(self.root_project)
|
self.process_docs(self.root_project)
|
||||||
|
self.process_metrics(self.root_project)
|
||||||
|
|
||||||
# update tracking data
|
# update tracking data
|
||||||
self._perf_info.process_manifest_elapsed = time.perf_counter() - start_process
|
self._perf_info.process_manifest_elapsed = time.perf_counter() - start_process
|
||||||
@@ -634,7 +636,9 @@ class ManifestLoader:
|
|||||||
if not flags.PARTIAL_PARSE:
|
if not flags.PARTIAL_PARSE:
|
||||||
fire_event(PartialParsingNotEnabled())
|
fire_event(PartialParsingNotEnabled())
|
||||||
return None
|
return None
|
||||||
path = os.path.join(self.root_project.target_path, PARTIAL_PARSE_FILE_NAME)
|
path = os.path.join(
|
||||||
|
self.root_project.project_root, self.root_project.target_path, PARTIAL_PARSE_FILE_NAME
|
||||||
|
)
|
||||||
|
|
||||||
reparse_reason = None
|
reparse_reason = None
|
||||||
|
|
||||||
@@ -833,6 +837,21 @@ class ManifestLoader:
|
|||||||
continue
|
continue
|
||||||
_process_refs_for_metric(self.manifest, current_project, metric)
|
_process_refs_for_metric(self.manifest, current_project, metric)
|
||||||
|
|
||||||
|
# Takes references in 'metrics' array of nodes and exposures, finds the target
|
||||||
|
# node, and updates 'depends_on.nodes' with the unique id
|
||||||
|
def process_metrics(self, config: RuntimeConfig):
|
||||||
|
current_project = config.project_name
|
||||||
|
for node in self.manifest.nodes.values():
|
||||||
|
if node.created_at < self.started_at:
|
||||||
|
continue
|
||||||
|
_process_metrics_for_node(self.manifest, current_project, node)
|
||||||
|
for metric in self.manifest.metrics.values():
|
||||||
|
# TODO: Can we do this if the metric is derived & depends on
|
||||||
|
# some other metric for its definition? Maybe....
|
||||||
|
if metric.created_at < self.started_at:
|
||||||
|
continue
|
||||||
|
_process_metrics_for_node(self.manifest, current_project, metric)
|
||||||
|
|
||||||
# nodes: node and column descriptions
|
# nodes: node and column descriptions
|
||||||
# sources: source and table descriptions, column descriptions
|
# sources: source and table descriptions, column descriptions
|
||||||
# macros: macro argument descriptions
|
# macros: macro argument descriptions
|
||||||
@@ -936,6 +955,19 @@ def invalid_source_fail_unless_test(node, target_name, target_table_name, disabl
|
|||||||
source_target_not_found(node, target_name, target_table_name, disabled=disabled)
|
source_target_not_found(node, target_name, target_table_name, disabled=disabled)
|
||||||
|
|
||||||
|
|
||||||
|
def invalid_metric_fail_unless_test(node, target_metric_name, target_metric_package):
|
||||||
|
|
||||||
|
if node.resource_type == NodeType.Test:
|
||||||
|
msg = get_target_not_found_or_disabled_msg(node, target_metric_name, target_metric_package)
|
||||||
|
warn_or_error(msg, log_fmt=warning_tag("{}"))
|
||||||
|
else:
|
||||||
|
metric_target_not_found(
|
||||||
|
node,
|
||||||
|
target_metric_name,
|
||||||
|
target_metric_package,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def _check_resource_uniqueness(
|
def _check_resource_uniqueness(
|
||||||
manifest: Manifest,
|
manifest: Manifest,
|
||||||
config: RuntimeConfig,
|
config: RuntimeConfig,
|
||||||
@@ -1039,6 +1071,11 @@ def _process_docs_for_metrics(context: Dict[str, Any], metric: ParsedMetric) ->
|
|||||||
metric.description = get_rendered(metric.description, context)
|
metric.description = get_rendered(metric.description, context)
|
||||||
|
|
||||||
|
|
||||||
|
# TODO: this isn't actually referenced anywhere?
|
||||||
|
def _process_derived_metrics(context: Dict[str, Any], metric: ParsedMetric) -> None:
|
||||||
|
metric.description = get_rendered(metric.description, context)
|
||||||
|
|
||||||
|
|
||||||
def _process_refs_for_exposure(manifest: Manifest, current_project: str, exposure: ParsedExposure):
|
def _process_refs_for_exposure(manifest: Manifest, current_project: str, exposure: ParsedExposure):
|
||||||
"""Given a manifest and exposure in that manifest, process its refs"""
|
"""Given a manifest and exposure in that manifest, process its refs"""
|
||||||
for ref in exposure.refs:
|
for ref in exposure.refs:
|
||||||
@@ -1121,6 +1158,48 @@ def _process_refs_for_metric(manifest: Manifest, current_project: str, metric: P
|
|||||||
manifest.update_metric(metric)
|
manifest.update_metric(metric)
|
||||||
|
|
||||||
|
|
||||||
|
def _process_metrics_for_node(
|
||||||
|
manifest: Manifest, current_project: str, node: Union[ManifestNode, ParsedMetric]
|
||||||
|
):
|
||||||
|
"""Given a manifest and a node in that manifest, process its metrics"""
|
||||||
|
for metric in node.metrics:
|
||||||
|
target_metric: Optional[ParsedMetric] = None
|
||||||
|
target_metric_name: str
|
||||||
|
target_metric_package: Optional[str] = None
|
||||||
|
|
||||||
|
if len(metric) == 1:
|
||||||
|
target_metric_name = metric[0]
|
||||||
|
elif len(metric) == 2:
|
||||||
|
target_metric_package, target_metric_name = metric
|
||||||
|
else:
|
||||||
|
raise dbt.exceptions.InternalException(
|
||||||
|
f"Metric references should always be 1 or 2 arguments - got {len(metric)}"
|
||||||
|
)
|
||||||
|
|
||||||
|
# Resolve_ref
|
||||||
|
target_metric = manifest.resolve_metric(
|
||||||
|
target_metric_name,
|
||||||
|
target_metric_package,
|
||||||
|
current_project,
|
||||||
|
node.package_name,
|
||||||
|
)
|
||||||
|
|
||||||
|
if target_metric is None:
|
||||||
|
# This may raise. Even if it doesn't, we don't want to add
|
||||||
|
# this node to the graph b/c there is no destination node
|
||||||
|
invalid_metric_fail_unless_test(
|
||||||
|
node,
|
||||||
|
target_metric_name,
|
||||||
|
target_metric_package,
|
||||||
|
)
|
||||||
|
|
||||||
|
continue
|
||||||
|
|
||||||
|
target_metric_id = target_metric.unique_id
|
||||||
|
|
||||||
|
node.depends_on.nodes.append(target_metric_id)
|
||||||
|
|
||||||
|
|
||||||
def _process_refs_for_node(manifest: Manifest, current_project: str, node: ManifestNode):
|
def _process_refs_for_node(manifest: Manifest, current_project: str, node: ManifestNode):
|
||||||
"""Given a manifest and a node in that manifest, process its refs"""
|
"""Given a manifest and a node in that manifest, process its refs"""
|
||||||
for ref in node.refs:
|
for ref in node.refs:
|
||||||
|
|||||||
@@ -66,6 +66,9 @@ class SchemaYamlRenderer(BaseRenderer):
|
|||||||
return False
|
return False
|
||||||
elif self._is_norender_key(keypath[0:]):
|
elif self._is_norender_key(keypath[0:]):
|
||||||
return False
|
return False
|
||||||
|
elif self.key == "metrics":
|
||||||
|
if keypath[0] == "sql":
|
||||||
|
return False
|
||||||
else: # models, seeds, snapshots, analyses
|
else: # models, seeds, snapshots, analyses
|
||||||
if self._is_norender_key(keypath[0:]):
|
if self._is_norender_key(keypath[0:]):
|
||||||
return False
|
return False
|
||||||
|
|||||||
@@ -797,6 +797,7 @@ class NonSourceParser(YamlDocsReader, Generic[NonSourceTarget, Parsed]):
|
|||||||
if self.key != "macros":
|
if self.key != "macros":
|
||||||
# macros don't have the 'config' key support yet
|
# macros don't have the 'config' key support yet
|
||||||
self.normalize_meta_attribute(data, path)
|
self.normalize_meta_attribute(data, path)
|
||||||
|
self.normalize_docs_attribute(data, path)
|
||||||
node = self._target_type().from_dict(data)
|
node = self._target_type().from_dict(data)
|
||||||
except (ValidationError, JSONValidationException) as exc:
|
except (ValidationError, JSONValidationException) as exc:
|
||||||
msg = error_context(path, self.key, data, exc)
|
msg = error_context(path, self.key, data, exc)
|
||||||
@@ -804,21 +805,27 @@ class NonSourceParser(YamlDocsReader, Generic[NonSourceTarget, Parsed]):
|
|||||||
else:
|
else:
|
||||||
yield node
|
yield node
|
||||||
|
|
||||||
# We want to raise an error if 'meta' is in two places, and move 'meta'
|
# We want to raise an error if some attributes are in two places, and move them
|
||||||
# from toplevel to config if necessary
|
# from toplevel to config if necessary
|
||||||
def normalize_meta_attribute(self, data, path):
|
def normalize_attribute(self, data, path, attribute):
|
||||||
if "meta" in data:
|
if attribute in data:
|
||||||
if "config" in data and "meta" in data["config"]:
|
if "config" in data and attribute in data["config"]:
|
||||||
raise ParsingException(
|
raise ParsingException(
|
||||||
f"""
|
f"""
|
||||||
In {path}: found meta dictionary in 'config' dictionary and as top-level key.
|
In {path}: found {attribute} dictionary in 'config' dictionary and as top-level key.
|
||||||
Remove the top-level key and define it under 'config' dictionary only.
|
Remove the top-level key and define it under 'config' dictionary only.
|
||||||
""".strip()
|
""".strip()
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
if "config" not in data:
|
if "config" not in data:
|
||||||
data["config"] = {}
|
data["config"] = {}
|
||||||
data["config"]["meta"] = data.pop("meta")
|
data["config"][attribute] = data.pop(attribute)
|
||||||
|
|
||||||
|
def normalize_meta_attribute(self, data, path):
|
||||||
|
return self.normalize_attribute(data, path, "meta")
|
||||||
|
|
||||||
|
def normalize_docs_attribute(self, data, path):
|
||||||
|
return self.normalize_attribute(data, path, "docs")
|
||||||
|
|
||||||
def patch_node_config(self, node, patch):
|
def patch_node_config(self, node, patch):
|
||||||
# Get the ContextConfig that's used in calculating the config
|
# Get the ContextConfig that's used in calculating the config
|
||||||
@@ -1029,7 +1036,7 @@ class MetricParser(YamlReader):
|
|||||||
description=unparsed.description,
|
description=unparsed.description,
|
||||||
label=unparsed.label,
|
label=unparsed.label,
|
||||||
type=unparsed.type,
|
type=unparsed.type,
|
||||||
sql=unparsed.sql,
|
sql=str(unparsed.sql),
|
||||||
timestamp=unparsed.timestamp,
|
timestamp=unparsed.timestamp,
|
||||||
dimensions=unparsed.dimensions,
|
dimensions=unparsed.dimensions,
|
||||||
time_grains=unparsed.time_grains,
|
time_grains=unparsed.time_grains,
|
||||||
@@ -1044,8 +1051,17 @@ class MetricParser(YamlReader):
|
|||||||
self.schema_parser.manifest,
|
self.schema_parser.manifest,
|
||||||
package_name,
|
package_name,
|
||||||
)
|
)
|
||||||
model_ref = "{{ " + unparsed.model + " }}"
|
|
||||||
get_rendered(model_ref, ctx, parsed, capture_macros=True)
|
if parsed.model is not None:
|
||||||
|
model_ref = "{{ " + parsed.model + " }}"
|
||||||
|
get_rendered(model_ref, ctx, parsed)
|
||||||
|
|
||||||
|
parsed.sql = get_rendered(
|
||||||
|
parsed.sql,
|
||||||
|
ctx,
|
||||||
|
node=parsed,
|
||||||
|
)
|
||||||
|
|
||||||
return parsed
|
return parsed
|
||||||
|
|
||||||
def parse(self) -> Iterable[ParsedMetric]:
|
def parse(self) -> Iterable[ParsedMetric]:
|
||||||
|
|||||||
@@ -14,7 +14,7 @@ from dbt import flags
|
|||||||
from dbt.version import _get_adapter_plugin_names
|
from dbt.version import _get_adapter_plugin_names
|
||||||
from dbt.adapters.factory import load_plugin, get_include_paths
|
from dbt.adapters.factory import load_plugin, get_include_paths
|
||||||
|
|
||||||
from dbt.contracts.project import Name as ProjectName
|
from dbt.contracts.util import Identifier as ProjectName
|
||||||
|
|
||||||
from dbt.events.functions import fire_event
|
from dbt.events.functions import fire_event
|
||||||
from dbt.events.types import (
|
from dbt.events.types import (
|
||||||
|
|||||||
@@ -73,7 +73,10 @@ def run_dbt(args: List[str] = None, expect_pass=True):
|
|||||||
return res
|
return res
|
||||||
|
|
||||||
|
|
||||||
# Use this if you need to capture the command logs in a test
|
# Use this if you need to capture the command logs in a test.
|
||||||
|
# If you want the logs that are normally written to a file, you must
|
||||||
|
# start with the "--debug" flag. The structured schema log CI test
|
||||||
|
# will turn the logs into json, so you have to be prepared for that.
|
||||||
def run_dbt_and_capture(args: List[str] = None, expect_pass=True):
|
def run_dbt_and_capture(args: List[str] = None, expect_pass=True):
|
||||||
try:
|
try:
|
||||||
stringbuf = capture_stdout_logs()
|
stringbuf = capture_stdout_logs()
|
||||||
@@ -83,6 +86,11 @@ def run_dbt_and_capture(args: List[str] = None, expect_pass=True):
|
|||||||
finally:
|
finally:
|
||||||
stop_capture_stdout_logs()
|
stop_capture_stdout_logs()
|
||||||
|
|
||||||
|
# Json logs will have lots of escape characters which will
|
||||||
|
# make checks for strings in the logs fail, so remove those.
|
||||||
|
if '{"code":' in stdout:
|
||||||
|
stdout = stdout.replace("\\", "")
|
||||||
|
|
||||||
return res, stdout
|
return res, stdout
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -235,5 +235,5 @@ def _get_adapter_plugin_names() -> Iterator[str]:
|
|||||||
yield plugin_name
|
yield plugin_name
|
||||||
|
|
||||||
|
|
||||||
__version__ = "1.2.0b1"
|
__version__ = "1.3.0a1"
|
||||||
installed = get_installed_version()
|
installed = get_installed_version()
|
||||||
|
|||||||
@@ -25,7 +25,7 @@ with open(os.path.join(this_directory, "README.md")) as f:
|
|||||||
|
|
||||||
|
|
||||||
package_name = "dbt-core"
|
package_name = "dbt-core"
|
||||||
package_version = "1.2.0b1"
|
package_version = "1.3.0a1"
|
||||||
description = """With dbt, data analysts and engineers can build analytics \
|
description = """With dbt, data analysts and engineers can build analytics \
|
||||||
the way engineers build applications."""
|
the way engineers build applications."""
|
||||||
|
|
||||||
|
|||||||
@@ -14,8 +14,8 @@ FROM --platform=$build_for python:3.10.5-slim-bullseye as base
|
|||||||
# N.B. The refs updated automagically every release via bumpversion
|
# N.B. The refs updated automagically every release via bumpversion
|
||||||
# N.B. dbt-postgres is currently found in the core codebase so a value of dbt-core@<some_version> is correct
|
# N.B. dbt-postgres is currently found in the core codebase so a value of dbt-core@<some_version> is correct
|
||||||
|
|
||||||
ARG dbt_core_ref=dbt-core@v1.2.0b1
|
ARG dbt_core_ref=dbt-core@v1.3.0a1
|
||||||
ARG dbt_postgres_ref=dbt-core@v1.2.0b1
|
ARG dbt_postgres_ref=dbt-core@v1.3.0a1
|
||||||
ARG dbt_redshift_ref=dbt-redshift@v1.0.0
|
ARG dbt_redshift_ref=dbt-redshift@v1.0.0
|
||||||
ARG dbt_bigquery_ref=dbt-bigquery@v1.0.0
|
ARG dbt_bigquery_ref=dbt-bigquery@v1.0.0
|
||||||
ARG dbt_snowflake_ref=dbt-snowflake@v1.0.0
|
ARG dbt_snowflake_ref=dbt-snowflake@v1.0.0
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user