forked from repo-mirrors/dbt-core
Compare commits
109 Commits
update-ind
...
gha-postgr
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
2e876d2c70 | ||
|
|
5f357187ca | ||
|
|
afd1bf2771 | ||
|
|
715386ad3a | ||
|
|
9554d1c926 | ||
|
|
d6ad6a5477 | ||
|
|
1d3d315249 | ||
|
|
b35ad46e3f | ||
|
|
c28cb92af5 | ||
|
|
b56d96df5e | ||
|
|
37d382c8e7 | ||
|
|
9b7f4ff842 | ||
|
|
555ff8091f | ||
|
|
98fddcf54f | ||
|
|
d652359c61 | ||
|
|
f7d21e012e | ||
|
|
e1fa461186 | ||
|
|
1153597970 | ||
|
|
09f9febc25 | ||
|
|
22181409f6 | ||
|
|
f25a474f75 | ||
|
|
3c55806203 | ||
|
|
bba020fcc0 | ||
|
|
84eb0ff672 | ||
|
|
3695698e22 | ||
|
|
9ca1bc5b4c | ||
|
|
5f66678f6d | ||
|
|
63262e93cb | ||
|
|
374412af53 | ||
|
|
47848b8ea8 | ||
|
|
3d09872a56 | ||
|
|
dfa7d06526 | ||
|
|
7f57dd5a30 | ||
|
|
56bfbeaedd | ||
|
|
1dd26e79af | ||
|
|
86223609dd | ||
|
|
21a46332f1 | ||
|
|
ff2726c3b5 | ||
|
|
014444dc18 | ||
|
|
25c2042dc9 | ||
|
|
0a160fc27a | ||
|
|
c598741262 | ||
|
|
f9c2b9398f | ||
|
|
cab6dabbc7 | ||
|
|
e1621ebc54 | ||
|
|
cd90d4493c | ||
|
|
560d151dcd | ||
|
|
229c537748 | ||
|
|
79ad0a3243 | ||
|
|
c668846404 | ||
|
|
c4958de166 | ||
|
|
33161a3035 | ||
|
|
471b816dcd | ||
|
|
bef2d20c21 | ||
|
|
2a26fabfdf | ||
|
|
4c7d922a6d | ||
|
|
b03291548a | ||
|
|
a7af3b3831 | ||
|
|
6e4564ab05 | ||
|
|
1aeff2c58f | ||
|
|
601fee0d5f | ||
|
|
88b8b10df1 | ||
|
|
4ea0e1007c | ||
|
|
a309283a7c | ||
|
|
b10fa79ae8 | ||
|
|
37e2725038 | ||
|
|
37fd299ad0 | ||
|
|
a94027acea | ||
|
|
b59c9075e2 | ||
|
|
c215697a02 | ||
|
|
d936a630c1 | ||
|
|
11ee2b9c42 | ||
|
|
64c59476f4 | ||
|
|
2bae05b8ed | ||
|
|
ca163c3d6e | ||
|
|
9a796aa202 | ||
|
|
51ff85bb2d | ||
|
|
d389ff1450 | ||
|
|
4415731da4 | ||
|
|
0fdc83af9d | ||
|
|
71a8a41104 | ||
|
|
da19d7ba9f | ||
|
|
1475abb1cb | ||
|
|
27b2f965dd | ||
|
|
100352d6b4 | ||
|
|
8ee8b2560a | ||
|
|
d4a6482091 | ||
|
|
8639290108 | ||
|
|
e699f5d042 | ||
|
|
e977b3eee5 | ||
|
|
c5be8e2a93 | ||
|
|
bff116dbed | ||
|
|
4df120e40e | ||
|
|
e53420c1d0 | ||
|
|
88ccc8a447 | ||
|
|
a98059967d | ||
|
|
b680c7ae95 | ||
|
|
a677abd5e8 | ||
|
|
8c850b58cb | ||
|
|
a34267f54b | ||
|
|
155482851a | ||
|
|
81386a7a43 | ||
|
|
d8e38c1a1d | ||
|
|
3e37d77780 | ||
|
|
e0783c2922 | ||
|
|
c2d4643f9d | ||
|
|
84456f50f6 | ||
|
|
fb10bb4aea | ||
|
|
366d4ad04a |
@@ -0,0 +1,6 @@
|
||||
kind: Breaking Changes
|
||||
body: Fix changing the current working directory when using dpt deps, clean and init.
|
||||
time: 2023-12-06T19:24:42.575372+09:00
|
||||
custom:
|
||||
Author: rariyama
|
||||
Issue: "8997"
|
||||
7
.changes/unreleased/Dependencies-20240820-131909.yaml
Normal file
7
.changes/unreleased/Dependencies-20240820-131909.yaml
Normal file
@@ -0,0 +1,7 @@
|
||||
kind: Dependencies
|
||||
body: Increase supported version range for dbt-semantic-interfaces. Needed to support
|
||||
custom calendar features.
|
||||
time: 2024-08-20T13:19:09.015225-07:00
|
||||
custom:
|
||||
Author: courtneyholcomb
|
||||
Issue: "9265"
|
||||
6
.changes/unreleased/Docs-20240311-140344.yaml
Normal file
6
.changes/unreleased/Docs-20240311-140344.yaml
Normal file
@@ -0,0 +1,6 @@
|
||||
kind: Docs
|
||||
body: Enable display of unit tests
|
||||
time: 2024-03-11T14:03:44.490834-04:00
|
||||
custom:
|
||||
Author: gshank
|
||||
Issue: "501"
|
||||
6
.changes/unreleased/Docs-20240501-021050.yaml
Normal file
6
.changes/unreleased/Docs-20240501-021050.yaml
Normal file
@@ -0,0 +1,6 @@
|
||||
kind: Docs
|
||||
body: Unit tests not rendering
|
||||
time: 2024-05-01T02:10:50.987412+02:00
|
||||
custom:
|
||||
Author: aranke
|
||||
Issue: "506"
|
||||
6
.changes/unreleased/Docs-20240516-223036.yaml
Normal file
6
.changes/unreleased/Docs-20240516-223036.yaml
Normal file
@@ -0,0 +1,6 @@
|
||||
kind: Docs
|
||||
body: Add support for Saved Query node
|
||||
time: 2024-05-16T22:30:36.206492-07:00
|
||||
custom:
|
||||
Author: ChenyuLInx
|
||||
Issue: "486"
|
||||
6
.changes/unreleased/Docs-20240613-151048.yaml
Normal file
6
.changes/unreleased/Docs-20240613-151048.yaml
Normal file
@@ -0,0 +1,6 @@
|
||||
kind: Docs
|
||||
body: Fix npm security vulnerabilities as of June 2024
|
||||
time: 2024-06-13T15:10:48.301989+01:00
|
||||
custom:
|
||||
Author: aranke
|
||||
Issue: "513"
|
||||
7
.changes/unreleased/Features-20240522-000309.yaml
Normal file
7
.changes/unreleased/Features-20240522-000309.yaml
Normal file
@@ -0,0 +1,7 @@
|
||||
kind: Features
|
||||
body: Maximally parallelize dbt clone
|
||||
in clone command"
|
||||
time: 2024-05-22T00:03:09.765977-04:00
|
||||
custom:
|
||||
Author: michelleark
|
||||
Issue: "7914"
|
||||
6
.changes/unreleased/Features-20240527-124405.yaml
Normal file
6
.changes/unreleased/Features-20240527-124405.yaml
Normal file
@@ -0,0 +1,6 @@
|
||||
kind: Features
|
||||
body: Add --host flag to dbt docs serve, defaulting to '127.0.0.1'
|
||||
time: 2024-05-27T12:44:05.040843-04:00
|
||||
custom:
|
||||
Author: michelleark
|
||||
Issue: "10229"
|
||||
6
.changes/unreleased/Features-20240531-150816.yaml
Normal file
6
.changes/unreleased/Features-20240531-150816.yaml
Normal file
@@ -0,0 +1,6 @@
|
||||
kind: Features
|
||||
body: Update data_test to accept arbitrary config options
|
||||
time: 2024-05-31T15:08:16.431966-05:00
|
||||
custom:
|
||||
Author: McKnight-42
|
||||
Issue: "10197"
|
||||
6
.changes/unreleased/Features-20240606-112334.yaml
Normal file
6
.changes/unreleased/Features-20240606-112334.yaml
Normal file
@@ -0,0 +1,6 @@
|
||||
kind: Features
|
||||
body: add pre_model and post_model hook calls to data and unit tests to be able to provide extra config options
|
||||
time: 2024-06-06T11:23:34.758675-05:00
|
||||
custom:
|
||||
Author: McKnight-42
|
||||
Issue: "10198"
|
||||
6
.changes/unreleased/Features-20240617-103948.yaml
Normal file
6
.changes/unreleased/Features-20240617-103948.yaml
Normal file
@@ -0,0 +1,6 @@
|
||||
kind: Features
|
||||
body: add --empty value to jinja context as flags.EMPTY
|
||||
time: 2024-06-17T10:39:48.275801-04:00
|
||||
custom:
|
||||
Author: michelleark
|
||||
Issue: "10317"
|
||||
6
.changes/unreleased/Features-20240621-141635.yaml
Normal file
6
.changes/unreleased/Features-20240621-141635.yaml
Normal file
@@ -0,0 +1,6 @@
|
||||
kind: Features
|
||||
body: Warning message for snapshot timestamp data types
|
||||
time: 2024-06-21T14:16:35.717637-04:00
|
||||
custom:
|
||||
Author: gshank
|
||||
Issue: "10234"
|
||||
6
.changes/unreleased/Features-20240625-095107.yaml
Normal file
6
.changes/unreleased/Features-20240625-095107.yaml
Normal file
@@ -0,0 +1,6 @@
|
||||
kind: Features
|
||||
body: Support cumulative_type_params & sub-daily granularities in semantic manifest.
|
||||
time: 2024-06-25T09:51:07.983248-07:00
|
||||
custom:
|
||||
Author: courtneyholcomb
|
||||
Issue: "10360"
|
||||
6
.changes/unreleased/Features-20240627-162953.yaml
Normal file
6
.changes/unreleased/Features-20240627-162953.yaml
Normal file
@@ -0,0 +1,6 @@
|
||||
kind: Features
|
||||
body: Add time_granularity to metric spec.
|
||||
time: 2024-06-27T16:29:53.500917-07:00
|
||||
custom:
|
||||
Author: courtneyholcomb
|
||||
Issue: "10376"
|
||||
6
.changes/unreleased/Features-20240712-214546.yaml
Normal file
6
.changes/unreleased/Features-20240712-214546.yaml
Normal file
@@ -0,0 +1,6 @@
|
||||
kind: Features
|
||||
body: Support standard schema/database fields for snapshots
|
||||
time: 2024-07-12T21:45:46.06011-04:00
|
||||
custom:
|
||||
Author: gshank
|
||||
Issue: "10301"
|
||||
6
.changes/unreleased/Features-20240719-161841.yaml
Normal file
6
.changes/unreleased/Features-20240719-161841.yaml
Normal file
@@ -0,0 +1,6 @@
|
||||
kind: Features
|
||||
body: Support ref and source in foreign key constraint expressions, bump dbt-common minimum to 1.6
|
||||
time: 2024-07-19T16:18:41.434278-04:00
|
||||
custom:
|
||||
Author: michelleark
|
||||
Issue: "8062"
|
||||
6
.changes/unreleased/Features-20240722-202238.yaml
Normal file
6
.changes/unreleased/Features-20240722-202238.yaml
Normal file
@@ -0,0 +1,6 @@
|
||||
kind: Features
|
||||
body: Support new semantic layer time spine configs to enable sub-daily granularity.
|
||||
time: 2024-07-22T20:22:38.258249-07:00
|
||||
custom:
|
||||
Author: courtneyholcomb
|
||||
Issue: "10475"
|
||||
6
.changes/unreleased/Features-20240829-135320.yaml
Normal file
6
.changes/unreleased/Features-20240829-135320.yaml
Normal file
@@ -0,0 +1,6 @@
|
||||
kind: Features
|
||||
body: Add support for behavior flags
|
||||
time: 2024-08-29T13:53:20.16122-04:00
|
||||
custom:
|
||||
Author: mikealfare
|
||||
Issue: "10618"
|
||||
6
.changes/unreleased/Fixes-20240113-073615.yaml
Normal file
6
.changes/unreleased/Fixes-20240113-073615.yaml
Normal file
@@ -0,0 +1,6 @@
|
||||
kind: Fixes
|
||||
body: Convert "Skipping model due to fail_fast" message to DEBUG level
|
||||
time: 2024-01-13T07:36:15.836294-00:00
|
||||
custom:
|
||||
Author: scottgigante,nevdelap
|
||||
Issue: "8774"
|
||||
7
.changes/unreleased/Fixes-20240522-182855.yaml
Normal file
7
.changes/unreleased/Fixes-20240522-182855.yaml
Normal file
@@ -0,0 +1,7 @@
|
||||
kind: Fixes
|
||||
body: 'Fix: Order-insensitive unit test equality assertion for expected/actual with
|
||||
multiple nulls'
|
||||
time: 2024-05-22T18:28:55.91733-04:00
|
||||
custom:
|
||||
Author: michelleark
|
||||
Issue: "10167"
|
||||
6
.changes/unreleased/Fixes-20240523-204251.yaml
Normal file
6
.changes/unreleased/Fixes-20240523-204251.yaml
Normal file
@@ -0,0 +1,6 @@
|
||||
kind: Fixes
|
||||
body: Renaming or removing a contracted model should raise a BreakingChange warning/error
|
||||
time: 2024-05-23T20:42:51.033946-04:00
|
||||
custom:
|
||||
Author: michelleark
|
||||
Issue: "10116"
|
||||
6
.changes/unreleased/Fixes-20240524-131135.yaml
Normal file
6
.changes/unreleased/Fixes-20240524-131135.yaml
Normal file
@@ -0,0 +1,6 @@
|
||||
kind: Fixes
|
||||
body: prefer disabled project nodes to external node
|
||||
time: 2024-05-24T13:11:35.440443-04:00
|
||||
custom:
|
||||
Author: michelleark
|
||||
Issue: "10224"
|
||||
6
.changes/unreleased/Fixes-20240605-111652.yaml
Normal file
6
.changes/unreleased/Fixes-20240605-111652.yaml
Normal file
@@ -0,0 +1,6 @@
|
||||
kind: Fixes
|
||||
body: Fix issues with selectors and inline nodes
|
||||
time: 2024-06-05T11:16:52.187667-04:00
|
||||
custom:
|
||||
Author: gshank
|
||||
Issue: 8943 9269
|
||||
6
.changes/unreleased/Fixes-20240607-134648.yaml
Normal file
6
.changes/unreleased/Fixes-20240607-134648.yaml
Normal file
@@ -0,0 +1,6 @@
|
||||
kind: Fixes
|
||||
body: Fix snapshot config to work in yaml files
|
||||
time: 2024-06-07T13:46:48.383215-04:00
|
||||
custom:
|
||||
Author: gshank
|
||||
Issue: "4000"
|
||||
6
.changes/unreleased/Fixes-20240610-132130.yaml
Normal file
6
.changes/unreleased/Fixes-20240610-132130.yaml
Normal file
@@ -0,0 +1,6 @@
|
||||
kind: Fixes
|
||||
body: Improve handling of error when loading schema file list
|
||||
time: 2024-06-10T13:21:30.963371-04:00
|
||||
custom:
|
||||
Author: gshank
|
||||
Issue: "10284"
|
||||
6
.changes/unreleased/Fixes-20240610-200522.yaml
Normal file
6
.changes/unreleased/Fixes-20240610-200522.yaml
Normal file
@@ -0,0 +1,6 @@
|
||||
kind: Fixes
|
||||
body: Use model alias for the CTE identifier generated during ephemeral materialization
|
||||
time: 2024-06-10T20:05:22.510814008Z
|
||||
custom:
|
||||
Author: jeancochrane
|
||||
Issue: "5273"
|
||||
6
.changes/unreleased/Fixes-20240612-124256.yaml
Normal file
6
.changes/unreleased/Fixes-20240612-124256.yaml
Normal file
@@ -0,0 +1,6 @@
|
||||
kind: Fixes
|
||||
body: Saved Query node fail during skip
|
||||
time: 2024-06-12T12:42:56.329073-07:00
|
||||
custom:
|
||||
Author: ChenyuLInx
|
||||
Issue: "10029"
|
||||
6
.changes/unreleased/Fixes-20240612-152139.yaml
Normal file
6
.changes/unreleased/Fixes-20240612-152139.yaml
Normal file
@@ -0,0 +1,6 @@
|
||||
kind: Fixes
|
||||
body: Implement state:modified for saved queries
|
||||
time: 2024-06-12T15:21:39.851426-04:00
|
||||
custom:
|
||||
Author: gshank
|
||||
Issue: "10294"
|
||||
6
.changes/unreleased/Fixes-20240613-183117.yaml
Normal file
6
.changes/unreleased/Fixes-20240613-183117.yaml
Normal file
@@ -0,0 +1,6 @@
|
||||
kind: Fixes
|
||||
body: DOn't warn on `unit_test` config paths that are properly used
|
||||
time: 2024-06-13T18:31:17.486497-07:00
|
||||
custom:
|
||||
Author: QMalcolm
|
||||
Issue: "10311"
|
||||
6
.changes/unreleased/Fixes-20240624-171729.yaml
Normal file
6
.changes/unreleased/Fixes-20240624-171729.yaml
Normal file
@@ -0,0 +1,6 @@
|
||||
kind: Fixes
|
||||
body: Fix setting `silence` of `warn_error_options` via `dbt_project.yaml` flags
|
||||
time: 2024-06-24T17:17:29.464865-07:00
|
||||
custom:
|
||||
Author: QMalcolm
|
||||
Issue: "10160"
|
||||
7
.changes/unreleased/Fixes-20240625-171737.yaml
Normal file
7
.changes/unreleased/Fixes-20240625-171737.yaml
Normal file
@@ -0,0 +1,7 @@
|
||||
kind: Fixes
|
||||
body: Attempt to provide test fixture tables with all values to set types correctly
|
||||
for comparisong with source tables
|
||||
time: 2024-06-25T17:17:37.514619-07:00
|
||||
custom:
|
||||
Author: versusfacit
|
||||
Issue: "10365"
|
||||
6
.changes/unreleased/Fixes-20240627-154448.yaml
Normal file
6
.changes/unreleased/Fixes-20240627-154448.yaml
Normal file
@@ -0,0 +1,6 @@
|
||||
kind: Fixes
|
||||
body: Limit data_tests deprecation to root_project
|
||||
time: 2024-06-27T15:44:48.579869-04:00
|
||||
custom:
|
||||
Author: gshank
|
||||
Issue: "9835"
|
||||
6
.changes/unreleased/Fixes-20240709-172440.yaml
Normal file
6
.changes/unreleased/Fixes-20240709-172440.yaml
Normal file
@@ -0,0 +1,6 @@
|
||||
kind: Fixes
|
||||
body: CLI flags should take precedence over env var flags
|
||||
time: 2024-07-09T17:24:40.918977-04:00
|
||||
custom:
|
||||
Author: gshank
|
||||
Issue: "10304"
|
||||
6
.changes/unreleased/Fixes-20240714-100254.yaml
Normal file
6
.changes/unreleased/Fixes-20240714-100254.yaml
Normal file
@@ -0,0 +1,6 @@
|
||||
kind: Fixes
|
||||
body: Fix typing for artifact schemas
|
||||
time: 2024-07-14T10:02:54.452099+09:00
|
||||
custom:
|
||||
Author: nakamichiworks
|
||||
Issue: "10442"
|
||||
6
.changes/unreleased/Fixes-20240716-133703.yaml
Normal file
6
.changes/unreleased/Fixes-20240716-133703.yaml
Normal file
@@ -0,0 +1,6 @@
|
||||
kind: Fixes
|
||||
body: Fix over deletion of generated_metrics in partial parsing
|
||||
time: 2024-07-16T13:37:03.49651-04:00
|
||||
custom:
|
||||
Author: gshank
|
||||
Issue: "10450"
|
||||
6
.changes/unreleased/Fixes-20240716-171427.yaml
Normal file
6
.changes/unreleased/Fixes-20240716-171427.yaml
Normal file
@@ -0,0 +1,6 @@
|
||||
kind: Fixes
|
||||
body: Fix error constructing warn_error_options
|
||||
time: 2024-07-16T17:14:27.837171-04:00
|
||||
custom:
|
||||
Author: gshank
|
||||
Issue: "10452"
|
||||
6
.changes/unreleased/Fixes-20240728-221421.yaml
Normal file
6
.changes/unreleased/Fixes-20240728-221421.yaml
Normal file
@@ -0,0 +1,6 @@
|
||||
kind: Fixes
|
||||
body: Do not update varchar column definitions if a contract exists
|
||||
time: 2024-07-28T22:14:21.67712-04:00
|
||||
custom:
|
||||
Author: gshank
|
||||
Issue: "10362"
|
||||
6
.changes/unreleased/Fixes-20240731-095152.yaml
Normal file
6
.changes/unreleased/Fixes-20240731-095152.yaml
Normal file
@@ -0,0 +1,6 @@
|
||||
kind: Fixes
|
||||
body: fix all_constraints access, disabled node parsing of non-uniquely named resources
|
||||
time: 2024-07-31T09:51:52.751135-04:00
|
||||
custom:
|
||||
Author: michelleark gshank
|
||||
Issue: "10509"
|
||||
6
.changes/unreleased/Fixes-20240806-172110.yaml
Normal file
6
.changes/unreleased/Fixes-20240806-172110.yaml
Normal file
@@ -0,0 +1,6 @@
|
||||
kind: Fixes
|
||||
body: Propagate measure label when using create_metrics
|
||||
time: 2024-08-06T17:21:10.265494-07:00
|
||||
custom:
|
||||
Author: aliceliu
|
||||
Issue: "10536"
|
||||
6
.changes/unreleased/Fixes-20240806-194843.yaml
Normal file
6
.changes/unreleased/Fixes-20240806-194843.yaml
Normal file
@@ -0,0 +1,6 @@
|
||||
kind: Fixes
|
||||
body: respect --quiet and --warn-error-options for flag deprecations
|
||||
time: 2024-08-06T19:48:43.399453-04:00
|
||||
custom:
|
||||
Author: michelleark
|
||||
Issue: "10105"
|
||||
6
.changes/unreleased/Fixes-20240813-154235.yaml
Normal file
6
.changes/unreleased/Fixes-20240813-154235.yaml
Normal file
@@ -0,0 +1,6 @@
|
||||
kind: Fixes
|
||||
body: Fix state:modified check for exports
|
||||
time: 2024-08-13T15:42:35.471685-07:00
|
||||
custom:
|
||||
Author: aliceliu
|
||||
Issue: "10138"
|
||||
6
.changes/unreleased/Fixes-20240816-140807.yaml
Normal file
6
.changes/unreleased/Fixes-20240816-140807.yaml
Normal file
@@ -0,0 +1,6 @@
|
||||
kind: Fixes
|
||||
body: Filter out empty nodes after graph selection to support consistent selection of nodes that depend on upstream public models
|
||||
time: 2024-08-16T14:08:07.426235-07:00
|
||||
custom:
|
||||
Author: jtcohen6
|
||||
Issue: "8987"
|
||||
6
.changes/unreleased/Fixes-20240824-210903.yaml
Normal file
6
.changes/unreleased/Fixes-20240824-210903.yaml
Normal file
@@ -0,0 +1,6 @@
|
||||
kind: Fixes
|
||||
body: Late render pre- and post-hooks configs in properties / schema YAML files
|
||||
time: 2024-08-24T21:09:03.252733-06:00
|
||||
custom:
|
||||
Author: dbeatty10
|
||||
Issue: "10603"
|
||||
7
.changes/unreleased/Fixes-20240829-105701.yaml
Normal file
7
.changes/unreleased/Fixes-20240829-105701.yaml
Normal file
@@ -0,0 +1,7 @@
|
||||
kind: Fixes
|
||||
body: Allow the use of env_var function in certain macros in which it was previously
|
||||
unavailable.
|
||||
time: 2024-08-29T10:57:01.160613-04:00
|
||||
custom:
|
||||
Author: peterallenwebb
|
||||
Issue: "10609"
|
||||
6
.changes/unreleased/Fixes-20240905-180248.yaml
Normal file
6
.changes/unreleased/Fixes-20240905-180248.yaml
Normal file
@@ -0,0 +1,6 @@
|
||||
kind: Fixes
|
||||
body: 'Remove deprecation for tests: to data_tests: change'
|
||||
time: 2024-09-05T18:02:48.086421-04:00
|
||||
custom:
|
||||
Author: gshank
|
||||
Issue: "10564"
|
||||
6
.changes/unreleased/Under the Hood-20240529-102814.yaml
Normal file
6
.changes/unreleased/Under the Hood-20240529-102814.yaml
Normal file
@@ -0,0 +1,6 @@
|
||||
kind: Under the Hood
|
||||
body: Enable record filtering by type.
|
||||
time: 2024-05-29T10:28:14.547624-05:00
|
||||
custom:
|
||||
Author: emmyoop
|
||||
Issue: "10240"
|
||||
6
.changes/unreleased/Under the Hood-20240618-140652.yaml
Normal file
6
.changes/unreleased/Under the Hood-20240618-140652.yaml
Normal file
@@ -0,0 +1,6 @@
|
||||
kind: Under the Hood
|
||||
body: Remove IntermediateSnapshotNode
|
||||
time: 2024-06-18T14:06:52.618602-04:00
|
||||
custom:
|
||||
Author: gshank
|
||||
Issue: "10326"
|
||||
6
.changes/unreleased/Under the Hood-20240701-131750.yaml
Normal file
6
.changes/unreleased/Under the Hood-20240701-131750.yaml
Normal file
@@ -0,0 +1,6 @@
|
||||
kind: Under the Hood
|
||||
body: Additional logging for skipped ephemeral models
|
||||
time: 2024-07-01T13:17:50.827788-04:00
|
||||
custom:
|
||||
Author: gshank
|
||||
Issue: "10389"
|
||||
6
.changes/unreleased/Under the Hood-20240716-184859.yaml
Normal file
6
.changes/unreleased/Under the Hood-20240716-184859.yaml
Normal file
@@ -0,0 +1,6 @@
|
||||
kind: Under the Hood
|
||||
body: bump black to 24.3.0
|
||||
time: 2024-07-16T18:48:59.651834-04:00
|
||||
custom:
|
||||
Author: michelleark
|
||||
Issue: "10454"
|
||||
6
.changes/unreleased/Under the Hood-20240716-205703.yaml
Normal file
6
.changes/unreleased/Under the Hood-20240716-205703.yaml
Normal file
@@ -0,0 +1,6 @@
|
||||
kind: Under the Hood
|
||||
body: generate protos with protoc version 5.26.1
|
||||
time: 2024-07-16T20:57:03.332448-04:00
|
||||
custom:
|
||||
Author: michelleark
|
||||
Issue: "10457"
|
||||
6
.changes/unreleased/Under the Hood-20240806-155406.yaml
Normal file
6
.changes/unreleased/Under the Hood-20240806-155406.yaml
Normal file
@@ -0,0 +1,6 @@
|
||||
kind: Under the Hood
|
||||
body: Move from minimal-snowplow-tracker fork back to snowplow-tracker
|
||||
time: 2024-08-06T15:54:06.422444-04:00
|
||||
custom:
|
||||
Author: peterallenwebb
|
||||
Issue: "8409"
|
||||
7
.changes/unreleased/Under the Hood-20240807-155652.yaml
Normal file
7
.changes/unreleased/Under the Hood-20240807-155652.yaml
Normal file
@@ -0,0 +1,7 @@
|
||||
kind: Under the Hood
|
||||
body: Add group info to RunResultError, RunResultFailure, RunResultWarning log lines
|
||||
time: 2024-08-07T15:56:52.171199-05:00
|
||||
custom:
|
||||
Author: aranke
|
||||
Issue: ""
|
||||
JiraID: "364"
|
||||
6
.changes/unreleased/Under the Hood-20240809-130234.yaml
Normal file
6
.changes/unreleased/Under the Hood-20240809-130234.yaml
Normal file
@@ -0,0 +1,6 @@
|
||||
kind: Under the Hood
|
||||
body: Improve speed of tree traversal when finding children, increasing build speed for some selectors
|
||||
time: 2024-08-09T13:02:34.759905-07:00
|
||||
custom:
|
||||
Author: ttusing
|
||||
Issue: "10434"
|
||||
6
.changes/unreleased/Under the Hood-20240821-095516.yaml
Normal file
6
.changes/unreleased/Under the Hood-20240821-095516.yaml
Normal file
@@ -0,0 +1,6 @@
|
||||
kind: Under the Hood
|
||||
body: Add test for sources tables with quotes
|
||||
time: 2024-08-21T09:55:16.038101-04:00
|
||||
custom:
|
||||
Author: gshank
|
||||
Issue: "10582"
|
||||
6
.changes/unreleased/Under the Hood-20240827-105014.yaml
Normal file
6
.changes/unreleased/Under the Hood-20240827-105014.yaml
Normal file
@@ -0,0 +1,6 @@
|
||||
kind: Under the Hood
|
||||
body: Additional type hints for `core/dbt/version.py`
|
||||
time: 2024-08-27T10:50:14.047859-05:00
|
||||
custom:
|
||||
Author: QMalcolm
|
||||
Issue: "10612"
|
||||
6
.changes/unreleased/Under the Hood-20240827-113123.yaml
Normal file
6
.changes/unreleased/Under the Hood-20240827-113123.yaml
Normal file
@@ -0,0 +1,6 @@
|
||||
kind: Under the Hood
|
||||
body: Fix typing issues in core/dbt/contracts/sql.py
|
||||
time: 2024-08-27T11:31:23.749912-05:00
|
||||
custom:
|
||||
Author: QMalcolm
|
||||
Issue: "10614"
|
||||
6
.changes/unreleased/Under the Hood-20240827-114810.yaml
Normal file
6
.changes/unreleased/Under the Hood-20240827-114810.yaml
Normal file
@@ -0,0 +1,6 @@
|
||||
kind: Under the Hood
|
||||
body: Fix type errors in `dbt/core/task/clean.py`
|
||||
time: 2024-08-27T11:48:10.438173-05:00
|
||||
custom:
|
||||
Author: QMalcolm
|
||||
Issue: "10616"
|
||||
1
.flake8
1
.flake8
@@ -7,6 +7,7 @@ ignore =
|
||||
W503 # makes Flake8 work like black
|
||||
W504
|
||||
E203 # makes Flake8 work like black
|
||||
E704 # makes Flake8 work like black
|
||||
E741
|
||||
E501 # long line checking is done in black
|
||||
exclude = test/
|
||||
|
||||
18
.github/ISSUE_TEMPLATE/code-docs.yml
vendored
Normal file
18
.github/ISSUE_TEMPLATE/code-docs.yml
vendored
Normal file
@@ -0,0 +1,18 @@
|
||||
name: 📄 Code docs
|
||||
description: Report an issue for markdown files within this repo, such as README, ARCHITECTURE, etc.
|
||||
title: "[Code docs] <title>"
|
||||
labels: ["triage"]
|
||||
body:
|
||||
- type: markdown
|
||||
attributes:
|
||||
value: |
|
||||
Thanks for taking the time to fill out this code docs issue!
|
||||
- type: textarea
|
||||
attributes:
|
||||
label: Please describe the issue and your proposals.
|
||||
description: |
|
||||
Links? References? Anything that will give us more context about the issue you are encountering!
|
||||
|
||||
Tip: You can attach images by clicking this area to highlight it and then dragging files in.
|
||||
validations:
|
||||
required: false
|
||||
3
.github/ISSUE_TEMPLATE/config.yml
vendored
3
.github/ISSUE_TEMPLATE/config.yml
vendored
@@ -1,5 +1,8 @@
|
||||
blank_issues_enabled: false
|
||||
contact_links:
|
||||
- name: Documentation
|
||||
url: https://github.com/dbt-labs/docs.getdbt.com/issues/new/choose
|
||||
about: Problems and issues with dbt product documentation hosted on docs.getdbt.com. Issues for markdown files within this repo, such as README, should be opened using the "Code docs" template.
|
||||
- name: Ask the community for help
|
||||
url: https://github.com/dbt-labs/docs.getdbt.com/discussions
|
||||
about: Need help troubleshooting? Check out our guide on how to ask
|
||||
|
||||
@@ -5,7 +5,6 @@ runs:
|
||||
steps:
|
||||
- shell: bash
|
||||
run: |
|
||||
brew services start postgresql
|
||||
echo "Check PostgreSQL service is running"
|
||||
i=10
|
||||
COMMAND='pg_isready'
|
||||
|
||||
@@ -2,6 +2,8 @@ name: "Set up postgres (windows)"
|
||||
description: "Set up postgres service on windows vm for dbt integration tests"
|
||||
runs:
|
||||
using: "composite"
|
||||
env:
|
||||
PQ_LIB_DIR: 'C:\Program Files\PostgreSQL\16\lib'
|
||||
steps:
|
||||
- shell: pwsh
|
||||
run: |
|
||||
|
||||
14
.github/pull_request_template.md
vendored
14
.github/pull_request_template.md
vendored
@@ -1,7 +1,7 @@
|
||||
resolves #
|
||||
Resolves #
|
||||
|
||||
<!---
|
||||
Include the number of the issue addressed by this PR above if applicable.
|
||||
Include the number of the issue addressed by this PR above, if applicable.
|
||||
PRs for code changes without an associated issue *will not be merged*.
|
||||
See CONTRIBUTING.md for more information.
|
||||
|
||||
@@ -26,8 +26,8 @@ resolves #
|
||||
|
||||
### Checklist
|
||||
|
||||
- [ ] I have read [the contributing guide](https://github.com/dbt-labs/dbt-core/blob/main/CONTRIBUTING.md) and understand what's expected of me
|
||||
- [ ] I have run this code in development and it appears to resolve the stated issue
|
||||
- [ ] This PR includes tests, or tests are not required/relevant for this PR
|
||||
- [ ] This PR has no interface changes (e.g. macros, cli, logs, json artifacts, config files, adapter interface, etc) or this PR has already received feedback and approval from Product or DX
|
||||
- [ ] This PR includes [type annotations](https://docs.python.org/3/library/typing.html) for new and modified functions
|
||||
- [ ] I have read [the contributing guide](https://github.com/dbt-labs/dbt-core/blob/main/CONTRIBUTING.md) and understand what's expected of me.
|
||||
- [ ] I have run this code in development, and it appears to resolve the stated issue.
|
||||
- [ ] This PR includes tests, or tests are not required or relevant for this PR.
|
||||
- [ ] This PR has no interface changes (e.g., macros, CLI, logs, JSON artifacts, config files, adapter interface, etc.) or this PR has already received feedback and approval from Product or DX.
|
||||
- [ ] This PR includes [type annotations](https://docs.python.org/3/library/typing.html) for new and modified functions.
|
||||
|
||||
2
.github/workflows/check-artifact-changes.yml
vendored
2
.github/workflows/check-artifact-changes.yml
vendored
@@ -32,7 +32,7 @@ jobs:
|
||||
run: |
|
||||
echo "CI failure: Artifact changes checked in core/dbt/artifacts directory."
|
||||
echo "Files changed: ${{ steps.check_artifact_changes.outputs.artifacts_changed_files }}"
|
||||
echo "To bypass this check, confirm that the change is not breaking (https://github.com/dbt-labs/dbt-core/blob/main/core/dbt/artifacts/README.md#breaking-changes) and add the 'artifact_minor_upgrade' label to the PR."
|
||||
echo "To bypass this check, confirm that the change is not breaking (https://github.com/dbt-labs/dbt-core/blob/main/core/dbt/artifacts/README.md#breaking-changes) and add the 'artifact_minor_upgrade' label to the PR. Modifications and additions to all fields require updates to https://github.com/dbt-labs/dbt-jsonschema."
|
||||
exit 1
|
||||
|
||||
- name: CI check passed
|
||||
|
||||
13
.github/workflows/main.yml
vendored
13
.github/workflows/main.yml
vendored
@@ -186,17 +186,29 @@ jobs:
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
|
||||
- name: Install postgres 16
|
||||
uses: ikalnytskyi/action-setup-postgres@v6
|
||||
with:
|
||||
postgres-version: "16"
|
||||
id: postgres
|
||||
|
||||
- name: Set up postgres (linux)
|
||||
if: runner.os == 'Linux'
|
||||
uses: ./.github/actions/setup-postgres-linux
|
||||
env:
|
||||
CONNECTION_STR: ${{ steps.postgres.outputs.connection-uri }}
|
||||
|
||||
- name: Set up postgres (macos)
|
||||
if: runner.os == 'macOS'
|
||||
uses: ./.github/actions/setup-postgres-macos
|
||||
env:
|
||||
CONNECTION_STR: ${{ steps.postgres.outputs.connection-uri }}
|
||||
|
||||
- name: Set up postgres (windows)
|
||||
if: runner.os == 'Windows'
|
||||
uses: ./.github/actions/setup-postgres-windows
|
||||
env:
|
||||
CONNECTION_STR: ${{ steps.postgres.outputs.connection-uri }}
|
||||
|
||||
- name: Install python tools
|
||||
run: |
|
||||
@@ -213,6 +225,7 @@ jobs:
|
||||
command: tox -- --ddtrace
|
||||
env:
|
||||
PYTEST_ADDOPTS: ${{ format('--splits {0} --group {1}', env.PYTHON_INTEGRATION_TEST_WORKERS, matrix.split-group) }}
|
||||
CONNECTION_STR: ${{ steps.postgres.outputs.connection-uri }}
|
||||
|
||||
- name: Get current date
|
||||
if: always()
|
||||
|
||||
21
.github/workflows/release.yml
vendored
21
.github/workflows/release.yml
vendored
@@ -247,3 +247,24 @@ jobs:
|
||||
|
||||
secrets:
|
||||
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_DEV_CORE_ALERTS }}
|
||||
|
||||
testing-slack-notification:
|
||||
# sends notifications to #slackbot-test
|
||||
name: Testing - Slack Notification
|
||||
if: ${{ failure() && inputs.test_run && !inputs.nightly_release }}
|
||||
|
||||
needs:
|
||||
[
|
||||
bump-version-generate-changelog,
|
||||
build-test-package,
|
||||
github-release,
|
||||
pypi-release,
|
||||
docker-release,
|
||||
]
|
||||
|
||||
uses: dbt-labs/dbt-release/.github/workflows/slack-post-notification.yml@main
|
||||
with:
|
||||
status: "failure"
|
||||
|
||||
secrets:
|
||||
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_TESTING_WEBHOOK_URL }}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
[settings]
|
||||
profile=black
|
||||
extend_skip_glob=.github/*,third-party-stubs/*,scripts/*
|
||||
known_first_party=dbt,dbt_adapters,dbt_common,dbt_extractor,dbt_semantic_interface
|
||||
known_first_party=dbt,dbt_adapters,dbt_common,dbt_extractor,dbt_semantic_interfaces
|
||||
|
||||
@@ -15,16 +15,19 @@ repos:
|
||||
args: [--unsafe]
|
||||
- id: check-json
|
||||
- id: end-of-file-fixer
|
||||
exclude: schemas/dbt/manifest/
|
||||
- id: trailing-whitespace
|
||||
exclude_types:
|
||||
- "markdown"
|
||||
- id: check-case-conflict
|
||||
- repo: https://github.com/pycqa/isort
|
||||
rev: 5.12.0
|
||||
# rev must match what's in dev-requirements.txt
|
||||
rev: 5.13.2
|
||||
hooks:
|
||||
- id: isort
|
||||
- repo: https://github.com/psf/black
|
||||
rev: 22.3.0
|
||||
# rev must match what's in dev-requirements.txt
|
||||
rev: 24.3.0
|
||||
hooks:
|
||||
- id: black
|
||||
- id: black
|
||||
@@ -34,6 +37,7 @@ repos:
|
||||
- "--check"
|
||||
- "--diff"
|
||||
- repo: https://github.com/pycqa/flake8
|
||||
# rev must match what's in dev-requirements.txt
|
||||
rev: 4.0.1
|
||||
hooks:
|
||||
- id: flake8
|
||||
@@ -41,6 +45,7 @@ repos:
|
||||
alias: flake8-check
|
||||
stages: [manual]
|
||||
- repo: https://github.com/pre-commit/mirrors-mypy
|
||||
# rev must match what's in dev-requirements.txt
|
||||
rev: v1.4.1
|
||||
hooks:
|
||||
- id: mypy
|
||||
|
||||
@@ -170,9 +170,9 @@ Finally, you can also run a specific test or group of tests using [`pytest`](htt
|
||||
|
||||
```sh
|
||||
# run all unit tests in a file
|
||||
python3 -m pytest tests/unit/test_base_column.py
|
||||
python3 -m pytest tests/unit/test_invocation_id.py
|
||||
# run a specific unit test
|
||||
python3 -m pytest tests/unit/test_base_column.py::TestNumericType::test__numeric_type
|
||||
python3 -m pytest tests/unit/test_invocation_id.py::TestInvocationId::test_invocation_id
|
||||
# run specific Postgres functional tests
|
||||
python3 -m pytest tests/functional/sources
|
||||
```
|
||||
|
||||
4
Makefile
4
Makefile
@@ -144,3 +144,7 @@ help: ## Show this help message.
|
||||
@echo
|
||||
@echo 'options:'
|
||||
@echo 'use USE_DOCKER=true to run target in a docker container'
|
||||
|
||||
.PHONY: json_schema
|
||||
json_schema: ## Update generated JSON schema using code changes.
|
||||
scripts/collect-artifact-schema.py --path schemas
|
||||
|
||||
26
codecov.yml
26
codecov.yml
@@ -1,6 +1,7 @@
|
||||
ignore:
|
||||
- ".github"
|
||||
- ".changes"
|
||||
|
||||
coverage:
|
||||
status:
|
||||
project:
|
||||
@@ -11,3 +12,28 @@ coverage:
|
||||
default:
|
||||
target: auto
|
||||
threshold: 80%
|
||||
|
||||
comment:
|
||||
layout: "header, diff, flags, components" # show component info in the PR comment
|
||||
|
||||
component_management:
|
||||
default_rules: # default rules that will be inherited by all components
|
||||
statuses:
|
||||
- type: project # in this case every component that doens't have a status defined will have a project type one
|
||||
target: auto
|
||||
threshold: 0.1%
|
||||
- type: patch
|
||||
target: 80%
|
||||
individual_components:
|
||||
- component_id: unittests
|
||||
name: "Unit Tests"
|
||||
flag_regexes:
|
||||
- "unit"
|
||||
statuses:
|
||||
- type: patch
|
||||
target: 80%
|
||||
threshold: 5%
|
||||
- component_id: integrationtests
|
||||
name: "Integration Tests"
|
||||
flag_regexes:
|
||||
- "integration"
|
||||
|
||||
@@ -29,6 +29,10 @@ All existing resources are defined under `dbt/artifacts/resources/v1`.
|
||||
|
||||
## Making changes to dbt/artifacts
|
||||
|
||||
### All changes
|
||||
|
||||
All changes to any fields will require a manual update to [dbt-jsonschema](https://github.com/dbt-labs/dbt-jsonschema) to ensure live checking continues to work.
|
||||
|
||||
### Non-breaking changes
|
||||
|
||||
Freely make incremental, non-breaking changes in-place to the latest major version of any artifact (minor or patch bumps). The only changes that are fully forward and backward compatible are:
|
||||
|
||||
@@ -38,6 +38,7 @@ from dbt.artifacts.resources.v1.macro import Macro, MacroArgument, MacroDependsO
|
||||
from dbt.artifacts.resources.v1.metric import (
|
||||
ConstantPropertyInput,
|
||||
ConversionTypeParams,
|
||||
CumulativeTypeParams,
|
||||
Metric,
|
||||
MetricConfig,
|
||||
MetricInput,
|
||||
@@ -45,7 +46,7 @@ from dbt.artifacts.resources.v1.metric import (
|
||||
MetricTimeWindow,
|
||||
MetricTypeParams,
|
||||
)
|
||||
from dbt.artifacts.resources.v1.model import Model, ModelConfig
|
||||
from dbt.artifacts.resources.v1.model import Model, ModelConfig, TimeSpine
|
||||
from dbt.artifacts.resources.v1.owner import Owner
|
||||
from dbt.artifacts.resources.v1.saved_query import (
|
||||
Export,
|
||||
|
||||
@@ -10,6 +10,7 @@ from dbt_common.contracts.config.properties import AdditionalPropertiesMixin
|
||||
from dbt_common.contracts.constraints import ColumnLevelConstraint
|
||||
from dbt_common.contracts.util import Mergeable
|
||||
from dbt_common.dataclass_schema import ExtensibleDbtClassMixin, dbtClassMixin
|
||||
from dbt_semantic_interfaces.type_enums import TimeGranularity
|
||||
|
||||
NodeVersion = Union[str, float]
|
||||
|
||||
@@ -66,6 +67,7 @@ class ColumnInfo(AdditionalPropertiesMixin, ExtensibleDbtClassMixin):
|
||||
quote: Optional[bool] = None
|
||||
tags: List[str] = field(default_factory=list)
|
||||
_extra: Dict[str, Any] = field(default_factory=dict)
|
||||
granularity: Optional[TimeGranularity] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
|
||||
@@ -2,13 +2,6 @@ import time
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Dict, List, Literal, Optional
|
||||
|
||||
from dbt_semantic_interfaces.references import MeasureReference, MetricReference
|
||||
from dbt_semantic_interfaces.type_enums import (
|
||||
ConversionCalculationType,
|
||||
MetricType,
|
||||
TimeGranularity,
|
||||
)
|
||||
|
||||
from dbt.artifacts.resources.base import GraphResource
|
||||
from dbt.artifacts.resources.types import NodeType
|
||||
from dbt.artifacts.resources.v1.components import DependsOn, RefArgs
|
||||
@@ -18,6 +11,13 @@ from dbt.artifacts.resources.v1.semantic_layer_components import (
|
||||
)
|
||||
from dbt_common.contracts.config.base import BaseConfig, CompareBehavior, MergeBehavior
|
||||
from dbt_common.dataclass_schema import dbtClassMixin
|
||||
from dbt_semantic_interfaces.references import MeasureReference, MetricReference
|
||||
from dbt_semantic_interfaces.type_enums import (
|
||||
ConversionCalculationType,
|
||||
MetricType,
|
||||
PeriodAggregation,
|
||||
TimeGranularity,
|
||||
)
|
||||
|
||||
"""
|
||||
The following classes are dataclasses which are used to construct the Metric
|
||||
@@ -80,6 +80,13 @@ class ConversionTypeParams(dbtClassMixin):
|
||||
constant_properties: Optional[List[ConstantPropertyInput]] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class CumulativeTypeParams(dbtClassMixin):
|
||||
window: Optional[MetricTimeWindow] = None
|
||||
grain_to_date: Optional[TimeGranularity] = None
|
||||
period_agg: PeriodAggregation = PeriodAggregation.FIRST
|
||||
|
||||
|
||||
@dataclass
|
||||
class MetricTypeParams(dbtClassMixin):
|
||||
measure: Optional[MetricInputMeasure] = None
|
||||
@@ -91,6 +98,7 @@ class MetricTypeParams(dbtClassMixin):
|
||||
grain_to_date: Optional[TimeGranularity] = None
|
||||
metrics: Optional[List[MetricInput]] = None
|
||||
conversion_type_params: Optional[ConversionTypeParams] = None
|
||||
cumulative_type_params: Optional[CumulativeTypeParams] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -113,6 +121,7 @@ class Metric(GraphResource):
|
||||
type_params: MetricTypeParams
|
||||
filter: Optional[WhereFilterIntersection] = None
|
||||
metadata: Optional[SourceFileMetadata] = None
|
||||
time_granularity: Optional[TimeGranularity] = None
|
||||
resource_type: Literal[NodeType.Metric]
|
||||
meta: Dict[str, Any] = field(default_factory=dict, metadata=MergeBehavior.Update.meta())
|
||||
tags: List[str] = field(default_factory=list)
|
||||
|
||||
@@ -11,6 +11,7 @@ from dbt.artifacts.resources.v1.components import (
|
||||
from dbt.artifacts.resources.v1.config import NodeConfig
|
||||
from dbt_common.contracts.config.base import MergeBehavior
|
||||
from dbt_common.contracts.constraints import ModelLevelConstraint
|
||||
from dbt_common.dataclass_schema import dbtClassMixin
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -21,6 +22,11 @@ class ModelConfig(NodeConfig):
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class TimeSpine(dbtClassMixin):
|
||||
standard_granularity_column: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class Model(CompiledResource):
|
||||
resource_type: Literal[NodeType.Model]
|
||||
@@ -32,6 +38,7 @@ class Model(CompiledResource):
|
||||
deprecation_date: Optional[datetime] = None
|
||||
defer_relation: Optional[DeferRelation] = None
|
||||
primary_key: List[str] = field(default_factory=list)
|
||||
time_spine: Optional[TimeSpine] = None
|
||||
|
||||
def __post_serialize__(self, dct: Dict, context: Optional[Dict] = None):
|
||||
dct = super().__post_serialize__(dct, context)
|
||||
|
||||
@@ -4,10 +4,6 @@ import time
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Dict, List, Literal, Optional
|
||||
|
||||
from dbt_semantic_interfaces.type_enums.export_destination_type import (
|
||||
ExportDestinationType,
|
||||
)
|
||||
|
||||
from dbt.artifacts.resources.base import GraphResource
|
||||
from dbt.artifacts.resources.types import NodeType
|
||||
from dbt.artifacts.resources.v1.components import DependsOn, RefArgs
|
||||
@@ -17,6 +13,9 @@ from dbt.artifacts.resources.v1.semantic_layer_components import (
|
||||
)
|
||||
from dbt_common.contracts.config.base import BaseConfig, CompareBehavior, MergeBehavior
|
||||
from dbt_common.dataclass_schema import dbtClassMixin
|
||||
from dbt_semantic_interfaces.type_enums.export_destination_type import (
|
||||
ExportDestinationType,
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -35,6 +34,7 @@ class Export(dbtClassMixin):
|
||||
|
||||
name: str
|
||||
config: ExportConfig
|
||||
unrendered_config: Dict[str, str] = field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass
|
||||
|
||||
@@ -1,13 +1,12 @@
|
||||
from dataclasses import dataclass
|
||||
from typing import List, Sequence, Tuple
|
||||
|
||||
from dbt_common.dataclass_schema import dbtClassMixin
|
||||
from dbt_semantic_interfaces.call_parameter_sets import FilterCallParameterSets
|
||||
from dbt_semantic_interfaces.parsing.where_filter.where_filter_parser import (
|
||||
WhereFilterParser,
|
||||
)
|
||||
|
||||
from dbt_common.dataclass_schema import dbtClassMixin
|
||||
|
||||
|
||||
@dataclass
|
||||
class WhereFilter(dbtClassMixin):
|
||||
|
||||
@@ -2,6 +2,11 @@ import time
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Dict, List, Optional, Sequence
|
||||
|
||||
from dbt.artifacts.resources import SourceFileMetadata
|
||||
from dbt.artifacts.resources.base import GraphResource
|
||||
from dbt.artifacts.resources.v1.components import DependsOn, RefArgs
|
||||
from dbt_common.contracts.config.base import BaseConfig, CompareBehavior, MergeBehavior
|
||||
from dbt_common.dataclass_schema import dbtClassMixin
|
||||
from dbt_semantic_interfaces.references import (
|
||||
DimensionReference,
|
||||
EntityReference,
|
||||
@@ -17,12 +22,6 @@ from dbt_semantic_interfaces.type_enums import (
|
||||
TimeGranularity,
|
||||
)
|
||||
|
||||
from dbt.artifacts.resources import SourceFileMetadata
|
||||
from dbt.artifacts.resources.base import GraphResource
|
||||
from dbt.artifacts.resources.v1.components import DependsOn, RefArgs
|
||||
from dbt_common.contracts.config.base import BaseConfig, CompareBehavior, MergeBehavior
|
||||
from dbt_common.dataclass_schema import dbtClassMixin
|
||||
|
||||
"""
|
||||
The classes in this file are dataclasses which are used to construct the Semantic
|
||||
Model node in dbt-core. Additionally, these classes need to at a minimum support
|
||||
|
||||
@@ -18,39 +18,34 @@ class SnapshotConfig(NodeConfig):
|
||||
# Not using Optional because of serialization issues with a Union of str and List[str]
|
||||
check_cols: Union[str, List[str], None] = None
|
||||
|
||||
@classmethod
|
||||
def validate(cls, data):
|
||||
super().validate(data)
|
||||
# Note: currently you can't just set these keys in schema.yml because this validation
|
||||
# will fail when parsing the snapshot node.
|
||||
if not data.get("strategy") or not data.get("unique_key") or not data.get("target_schema"):
|
||||
def final_validate(self):
|
||||
if not self.strategy or not self.unique_key:
|
||||
raise ValidationError(
|
||||
"Snapshots must be configured with a 'strategy', 'unique_key', "
|
||||
"and 'target_schema'."
|
||||
"Snapshots must be configured with a 'strategy' and 'unique_key'."
|
||||
)
|
||||
if data.get("strategy") == "check":
|
||||
if not data.get("check_cols"):
|
||||
if self.strategy == "check":
|
||||
if not self.check_cols:
|
||||
raise ValidationError(
|
||||
"A snapshot configured with the check strategy must "
|
||||
"specify a check_cols configuration."
|
||||
)
|
||||
if isinstance(data["check_cols"], str) and data["check_cols"] != "all":
|
||||
if isinstance(self.check_cols, str) and self.check_cols != "all":
|
||||
raise ValidationError(
|
||||
f"Invalid value for 'check_cols': {data['check_cols']}. "
|
||||
f"Invalid value for 'check_cols': {self.check_cols}. "
|
||||
"Expected 'all' or a list of strings."
|
||||
)
|
||||
elif data.get("strategy") == "timestamp":
|
||||
if not data.get("updated_at"):
|
||||
elif self.strategy == "timestamp":
|
||||
if not self.updated_at:
|
||||
raise ValidationError(
|
||||
"A snapshot configured with the timestamp strategy "
|
||||
"must specify an updated_at configuration."
|
||||
)
|
||||
if data.get("check_cols"):
|
||||
if self.check_cols:
|
||||
raise ValidationError("A 'timestamp' snapshot should not have 'check_cols'")
|
||||
# If the strategy is not 'check' or 'timestamp' it's a custom strategy,
|
||||
# formerly supported with GenericSnapshotConfig
|
||||
|
||||
if data.get("materialized") and data.get("materialized") != "snapshot":
|
||||
if self.materialized and self.materialized != "snapshot":
|
||||
raise ValidationError("A snapshot must have a materialized value of 'snapshot'")
|
||||
|
||||
# Called by "calculate_node_config_dict" in ContextConfigGenerator
|
||||
|
||||
@@ -77,8 +77,11 @@ class BaseArtifactMetadata(dbtClassMixin):
|
||||
# remote-compile-result
|
||||
# remote-execution-result
|
||||
# remote-run-result
|
||||
S = TypeVar("S", bound="VersionedSchema")
|
||||
|
||||
|
||||
def schema_version(name: str, version: int):
|
||||
def inner(cls: Type[VersionedSchema]):
|
||||
def inner(cls: Type[S]):
|
||||
cls.dbt_schema_version = SchemaVersion(
|
||||
name=name,
|
||||
version=version,
|
||||
|
||||
@@ -1,2 +1,11 @@
|
||||
# alias to latest
|
||||
from dbt.artifacts.schemas.catalog.v1.catalog import * # noqa
|
||||
from dbt_common.contracts.metadata import (
|
||||
CatalogKey,
|
||||
CatalogTable,
|
||||
ColumnMap,
|
||||
ColumnMetadata,
|
||||
StatsDict,
|
||||
StatsItem,
|
||||
TableMetadata,
|
||||
)
|
||||
|
||||
@@ -1,71 +1,18 @@
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime
|
||||
from typing import Any, Dict, List, NamedTuple, Optional, Union
|
||||
from typing import Any, Dict, List, Optional, Union
|
||||
|
||||
from dbt.artifacts.schemas.base import (
|
||||
ArtifactMixin,
|
||||
BaseArtifactMetadata,
|
||||
schema_version,
|
||||
)
|
||||
from dbt_common.contracts.metadata import CatalogTable
|
||||
from dbt_common.dataclass_schema import dbtClassMixin
|
||||
from dbt_common.utils.formatting import lowercase
|
||||
|
||||
Primitive = Union[bool, str, float, None]
|
||||
PrimitiveDict = Dict[str, Primitive]
|
||||
|
||||
CatalogKey = NamedTuple(
|
||||
"CatalogKey", [("database", Optional[str]), ("schema", str), ("name", str)]
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class StatsItem(dbtClassMixin):
|
||||
id: str
|
||||
label: str
|
||||
value: Primitive
|
||||
include: bool
|
||||
description: Optional[str] = None
|
||||
|
||||
|
||||
StatsDict = Dict[str, StatsItem]
|
||||
|
||||
|
||||
@dataclass
|
||||
class ColumnMetadata(dbtClassMixin):
|
||||
type: str
|
||||
index: int
|
||||
name: str
|
||||
comment: Optional[str] = None
|
||||
|
||||
|
||||
ColumnMap = Dict[str, ColumnMetadata]
|
||||
|
||||
|
||||
@dataclass
|
||||
class TableMetadata(dbtClassMixin):
|
||||
type: str
|
||||
schema: str
|
||||
name: str
|
||||
database: Optional[str] = None
|
||||
comment: Optional[str] = None
|
||||
owner: Optional[str] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class CatalogTable(dbtClassMixin):
|
||||
metadata: TableMetadata
|
||||
columns: ColumnMap
|
||||
stats: StatsDict
|
||||
# the same table with two unique IDs will just be listed two times
|
||||
unique_id: Optional[str] = None
|
||||
|
||||
def key(self) -> CatalogKey:
|
||||
return CatalogKey(
|
||||
lowercase(self.metadata.database),
|
||||
self.metadata.schema.lower(),
|
||||
self.metadata.name.lower(),
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class CatalogMetadata(BaseArtifactMetadata):
|
||||
|
||||
@@ -158,7 +158,8 @@ class RunResultsArtifact(ExecutionResult, ArtifactMixin):
|
||||
@classmethod
|
||||
def upgrade_schema_version(cls, data):
|
||||
"""This overrides the "upgrade_schema_version" call in VersionedSchema (via
|
||||
ArtifactMixin) to modify the dictionary passed in from earlier versions of the run_results."""
|
||||
ArtifactMixin) to modify the dictionary passed in from earlier versions of the run_results.
|
||||
"""
|
||||
run_results_schema_version = get_artifact_schema_version(data)
|
||||
# If less than the current version (v5), preprocess contents to match latest schema version
|
||||
if run_results_schema_version <= 5:
|
||||
|
||||
@@ -1,7 +1,10 @@
|
||||
from typing import IO, Optional
|
||||
from typing import IO, List, Optional, Union
|
||||
|
||||
from click.exceptions import ClickException
|
||||
|
||||
from dbt.artifacts.schemas.catalog import CatalogArtifact
|
||||
from dbt.contracts.graph.manifest import Manifest
|
||||
from dbt.contracts.results import RunExecutionResult
|
||||
from dbt.utils import ExitCodes
|
||||
|
||||
|
||||
@@ -23,7 +26,7 @@ class CliException(ClickException):
|
||||
|
||||
# the typing of _file is to satisfy the signature of ClickException.show
|
||||
# overriding this method prevents click from printing any exceptions to stdout
|
||||
def show(self, _file: Optional[IO] = None) -> None:
|
||||
def show(self, _file: Optional[IO] = None) -> None: # type: ignore[type-arg]
|
||||
pass
|
||||
|
||||
|
||||
@@ -31,7 +34,17 @@ class ResultExit(CliException):
|
||||
"""This class wraps any exception that contains results while invoking dbt, or the
|
||||
results of an invocation that did not succeed but did not throw any exceptions."""
|
||||
|
||||
def __init__(self, result) -> None:
|
||||
def __init__(
|
||||
self,
|
||||
result: Union[
|
||||
bool, # debug
|
||||
CatalogArtifact, # docs generate
|
||||
List[str], # list/ls
|
||||
Manifest, # parse
|
||||
None, # clean, deps, init, source
|
||||
RunExecutionResult, # build, compile, run, seed, snapshot, test, run-operation
|
||||
] = None,
|
||||
) -> None:
|
||||
super().__init__(ExitCodes.ModelError)
|
||||
self.result = result
|
||||
|
||||
|
||||
@@ -15,7 +15,7 @@ from dbt.cli.resolvers import default_log_path, default_project_dir
|
||||
from dbt.cli.types import Command as CliCommand
|
||||
from dbt.config.project import read_project_flags
|
||||
from dbt.contracts.project import ProjectFlags
|
||||
from dbt.deprecations import renamed_env_var
|
||||
from dbt.deprecations import fire_buffered_deprecations, renamed_env_var
|
||||
from dbt.events import ALL_EVENT_NAMES
|
||||
from dbt_common import ui
|
||||
from dbt_common.clients import jinja
|
||||
@@ -57,6 +57,7 @@ def convert_config(config_name, config_value):
|
||||
ret = WarnErrorOptions(
|
||||
include=config_value.get("include", []),
|
||||
exclude=config_value.get("exclude", []),
|
||||
silence=config_value.get("silence", []),
|
||||
valid_error_names=ALL_EVENT_NAMES,
|
||||
)
|
||||
return ret
|
||||
@@ -91,6 +92,8 @@ class Flags:
|
||||
# Set the default flags.
|
||||
for key, value in FLAGS_DEFAULTS.items():
|
||||
object.__setattr__(self, key, value)
|
||||
# Use to handle duplicate params in _assign_params
|
||||
flags_defaults_list = list(FLAGS_DEFAULTS.keys())
|
||||
|
||||
if ctx is None:
|
||||
ctx = get_current_context()
|
||||
@@ -172,13 +175,29 @@ class Flags:
|
||||
old_name=dep_param.envvar,
|
||||
new_name=new_param.envvar,
|
||||
)
|
||||
# end deprecated_params
|
||||
|
||||
# Set the flag value.
|
||||
is_duplicate = hasattr(self, param_name.upper())
|
||||
is_duplicate = (
|
||||
hasattr(self, param_name.upper())
|
||||
and param_name.upper() not in flags_defaults_list
|
||||
)
|
||||
# First time through, set as though FLAGS_DEFAULTS hasn't been set, so not a duplicate.
|
||||
# Subsequent pass (to process "parent" params) should be treated as duplicates.
|
||||
if param_name.upper() in flags_defaults_list:
|
||||
flags_defaults_list.remove(param_name.upper())
|
||||
# Note: the following determines whether parameter came from click default,
|
||||
# not from FLAGS_DEFAULTS in __init__.
|
||||
is_default = ctx.get_parameter_source(param_name) == ParameterSource.DEFAULT
|
||||
is_envvar = ctx.get_parameter_source(param_name) == ParameterSource.ENVIRONMENT
|
||||
|
||||
flag_name = (new_name or param_name).upper()
|
||||
|
||||
if (is_duplicate and not is_default) or not is_duplicate:
|
||||
# envvar flags are assigned in either parent or child context if there
|
||||
# isn't an overriding cli command flag.
|
||||
# If the flag has been encountered as a child cli flag, we don't
|
||||
# want to overwrite with parent envvar, since the commandline flag takes precedence.
|
||||
if (is_duplicate and not (is_default or is_envvar)) or not is_duplicate:
|
||||
object.__setattr__(self, flag_name, param_value)
|
||||
|
||||
# Track default assigned params.
|
||||
@@ -289,6 +308,10 @@ class Flags:
|
||||
params_assigned_from_default, ["WARN_ERROR", "WARN_ERROR_OPTIONS"]
|
||||
)
|
||||
|
||||
# Handle arguments mutually exclusive with INLINE
|
||||
self._assert_mutually_exclusive(params_assigned_from_default, ["SELECT", "INLINE"])
|
||||
self._assert_mutually_exclusive(params_assigned_from_default, ["SELECTOR", "INLINE"])
|
||||
|
||||
# Support lower cased access for legacy code.
|
||||
params = set(
|
||||
x for x in dir(self) if not callable(getattr(self, x)) and not x.startswith("__")
|
||||
@@ -315,7 +338,9 @@ class Flags:
|
||||
"""
|
||||
set_flag = None
|
||||
for flag in group:
|
||||
flag_set_by_user = flag.lower() not in params_assigned_from_default
|
||||
flag_set_by_user = (
|
||||
hasattr(self, flag) and flag.lower() not in params_assigned_from_default
|
||||
)
|
||||
if flag_set_by_user and set_flag:
|
||||
raise DbtUsageException(
|
||||
f"{flag.lower()}: not allowed with argument {set_flag.lower()}"
|
||||
@@ -330,6 +355,8 @@ class Flags:
|
||||
# not get pickled when written to disk as json.
|
||||
object.__delattr__(self, "deprecated_env_var_warnings")
|
||||
|
||||
fire_buffered_deprecations()
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, command: CliCommand, args_dict: Dict[str, Any]) -> "Flags":
|
||||
command_arg_list = command_params(command, args_dict)
|
||||
|
||||
@@ -218,10 +218,9 @@ def clean(ctx, **kwargs):
|
||||
"""Delete all folders in the clean-targets list (usually the dbt_packages and target directories.)"""
|
||||
from dbt.task.clean import CleanTask
|
||||
|
||||
task = CleanTask(ctx.obj["flags"], ctx.obj["project"])
|
||||
|
||||
results = task.run()
|
||||
success = task.interpret_results(results)
|
||||
with CleanTask(ctx.obj["flags"], ctx.obj["project"]) as task:
|
||||
results = task.run()
|
||||
success = task.interpret_results(results)
|
||||
return results, success
|
||||
|
||||
|
||||
@@ -274,6 +273,7 @@ def docs_generate(ctx, **kwargs):
|
||||
@click.pass_context
|
||||
@global_flags
|
||||
@p.browser
|
||||
@p.host
|
||||
@p.port
|
||||
@p.profiles_dir
|
||||
@p.project_dir
|
||||
@@ -436,9 +436,9 @@ def deps(ctx, **kwargs):
|
||||
message=f"Version is required in --add-package when a package when source is {flags.SOURCE}",
|
||||
option_name="--add-package",
|
||||
)
|
||||
task = DepsTask(flags, ctx.obj["project"])
|
||||
results = task.run()
|
||||
success = task.interpret_results(results)
|
||||
with DepsTask(flags, ctx.obj["project"]) as task:
|
||||
results = task.run()
|
||||
success = task.interpret_results(results)
|
||||
return results, success
|
||||
|
||||
|
||||
@@ -458,10 +458,9 @@ def init(ctx, **kwargs):
|
||||
"""Initialize a new dbt project."""
|
||||
from dbt.task.init import InitTask
|
||||
|
||||
task = InitTask(ctx.obj["flags"])
|
||||
|
||||
results = task.run()
|
||||
success = task.interpret_results(results)
|
||||
with InitTask(ctx.obj["flags"]) as task:
|
||||
results = task.run()
|
||||
success = task.interpret_results(results)
|
||||
return results, success
|
||||
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
from click import Choice, ParamType
|
||||
|
||||
from dbt.config.utils import exclusive_primary_alt_value_setting, parse_cli_yaml_string
|
||||
from dbt.config.utils import normalize_warn_error_options, parse_cli_yaml_string
|
||||
from dbt.events import ALL_EVENT_NAMES
|
||||
from dbt.exceptions import OptionNotYamlDictError, ValidationError
|
||||
from dbt_common.exceptions import DbtValidationError
|
||||
@@ -51,12 +51,7 @@ class WarnErrorOptionsType(YAML):
|
||||
def convert(self, value, param, ctx):
|
||||
# this function is being used by param in click
|
||||
include_exclude = super().convert(value, param, ctx)
|
||||
exclusive_primary_alt_value_setting(
|
||||
include_exclude, "include", "error", "warn_error_options"
|
||||
)
|
||||
exclusive_primary_alt_value_setting(
|
||||
include_exclude, "exclude", "warn", "warn_error_options"
|
||||
)
|
||||
normalize_warn_error_options(include_exclude)
|
||||
|
||||
return WarnErrorOptions(
|
||||
include=include_exclude.get("include", []),
|
||||
|
||||
@@ -135,6 +135,14 @@ full_refresh = click.option(
|
||||
is_flag=True,
|
||||
)
|
||||
|
||||
host = click.option(
|
||||
"--host",
|
||||
envvar="DBT_HOST",
|
||||
help="host to serve dbt docs on",
|
||||
type=click.STRING,
|
||||
default="127.0.0.1",
|
||||
)
|
||||
|
||||
indirect_selection = click.option(
|
||||
"--indirect-selection",
|
||||
envvar="DBT_INDIRECT_SELECTION",
|
||||
|
||||
@@ -41,7 +41,13 @@ from dbt_common.events.functions import LOG_VERSION, fire_event
|
||||
from dbt_common.events.helpers import get_json_string_utcnow
|
||||
from dbt_common.exceptions import DbtBaseException as DbtException
|
||||
from dbt_common.invocation import reset_invocation_id
|
||||
from dbt_common.record import Recorder, RecorderMode, get_record_mode_from_env
|
||||
from dbt_common.record import (
|
||||
Recorder,
|
||||
RecorderMode,
|
||||
get_record_mode_from_env,
|
||||
get_record_types_from_dict,
|
||||
get_record_types_from_env,
|
||||
)
|
||||
from dbt_common.utils import cast_dict_to_dict_of_strings
|
||||
|
||||
|
||||
@@ -101,13 +107,23 @@ def preflight(func):
|
||||
|
||||
def setup_record_replay():
|
||||
rec_mode = get_record_mode_from_env()
|
||||
rec_types = get_record_types_from_env()
|
||||
|
||||
recorder: Optional[Recorder] = None
|
||||
if rec_mode == RecorderMode.REPLAY:
|
||||
recording_path = os.environ["DBT_REPLAY"]
|
||||
recorder = Recorder(RecorderMode.REPLAY, recording_path)
|
||||
previous_recording_path = os.environ.get("DBT_RECORDER_FILE_PATH")
|
||||
recorder = Recorder(
|
||||
RecorderMode.REPLAY, types=rec_types, previous_recording_path=previous_recording_path
|
||||
)
|
||||
elif rec_mode == RecorderMode.DIFF:
|
||||
previous_recording_path = os.environ.get("DBT_RECORDER_FILE_PATH")
|
||||
# ensure types match the previous recording
|
||||
types = get_record_types_from_dict(previous_recording_path)
|
||||
recorder = Recorder(
|
||||
RecorderMode.DIFF, types=types, previous_recording_path=previous_recording_path
|
||||
)
|
||||
elif rec_mode == RecorderMode.RECORD:
|
||||
recorder = Recorder(RecorderMode.RECORD)
|
||||
recorder = Recorder(RecorderMode.RECORD, types=rec_types)
|
||||
|
||||
get_invocation_context().recorder = recorder
|
||||
|
||||
@@ -116,7 +132,10 @@ def tear_down_record_replay():
|
||||
recorder = get_invocation_context().recorder
|
||||
if recorder is not None:
|
||||
if recorder.mode == RecorderMode.RECORD:
|
||||
recorder.write("recording.json")
|
||||
recorder.write()
|
||||
if recorder.mode == RecorderMode.DIFF:
|
||||
recorder.write()
|
||||
recorder.write_diffs(diff_file_name="recording_diffs.json")
|
||||
elif recorder.mode == RecorderMode.REPLAY:
|
||||
recorder.write_diffs("replay_diffs.json")
|
||||
|
||||
@@ -160,9 +179,11 @@ def postflight(func):
|
||||
process_in_blocks=rusage.ru_inblock,
|
||||
process_out_blocks=rusage.ru_oublock,
|
||||
),
|
||||
EventLevel.INFO
|
||||
if "flags" in ctx.obj and ctx.obj["flags"].SHOW_RESOURCE_REPORT
|
||||
else None,
|
||||
(
|
||||
EventLevel.INFO
|
||||
if "flags" in ctx.obj and ctx.obj["flags"].SHOW_RESOURCE_REPORT
|
||||
else None
|
||||
),
|
||||
)
|
||||
|
||||
fire_event(
|
||||
|
||||
@@ -1,11 +1,13 @@
|
||||
from typing import Any, Dict, Optional
|
||||
from typing import Any, Dict, List, Optional, Union
|
||||
|
||||
import jinja2
|
||||
|
||||
from dbt.exceptions import MacroNamespaceNotStringError
|
||||
from dbt.artifacts.resources import RefArgs
|
||||
from dbt.exceptions import MacroNamespaceNotStringError, ParsingError
|
||||
from dbt_common.clients.jinja import get_environment
|
||||
from dbt_common.exceptions.macros import MacroNameNotStringError
|
||||
from dbt_common.tests import test_caching_enabled
|
||||
from dbt_extractor import ExtractionError, py_extract_from_source # type: ignore
|
||||
|
||||
_TESTING_MACRO_CACHE: Optional[Dict[str, Any]] = {}
|
||||
|
||||
@@ -153,3 +155,39 @@ def statically_parse_adapter_dispatch(func_call, ctx, db_wrapper):
|
||||
possible_macro_calls.append(f"{package_name}.{func_name}")
|
||||
|
||||
return possible_macro_calls
|
||||
|
||||
|
||||
def statically_parse_ref_or_source(expression: str) -> Union[RefArgs, List[str]]:
|
||||
"""
|
||||
Returns a RefArgs or List[str] object, corresponding to ref or source respectively, given an input jinja expression.
|
||||
|
||||
input: str representing how input node is referenced in tested model sql
|
||||
* examples:
|
||||
- "ref('my_model_a')"
|
||||
- "ref('my_model_a', version=3)"
|
||||
- "ref('package', 'my_model_a', version=3)"
|
||||
- "source('my_source_schema', 'my_source_name')"
|
||||
|
||||
If input is not a well-formed jinja ref or source expression, a ParsingError is raised.
|
||||
"""
|
||||
ref_or_source: Union[RefArgs, List[str]]
|
||||
|
||||
try:
|
||||
statically_parsed = py_extract_from_source(f"{{{{ {expression} }}}}")
|
||||
except ExtractionError:
|
||||
raise ParsingError(f"Invalid jinja expression: {expression}")
|
||||
|
||||
if statically_parsed.get("refs"):
|
||||
raw_ref = list(statically_parsed["refs"])[0]
|
||||
ref_or_source = RefArgs(
|
||||
package=raw_ref.get("package"),
|
||||
name=raw_ref.get("name"),
|
||||
version=raw_ref.get("version"),
|
||||
)
|
||||
elif statically_parsed.get("sources"):
|
||||
source_name, source_table_name = list(statically_parsed["sources"])[0]
|
||||
ref_or_source = [source_name, source_table_name]
|
||||
else:
|
||||
raise ParsingError(f"Invalid ref or source expression: {expression}")
|
||||
|
||||
return ref_or_source
|
||||
|
||||
@@ -21,6 +21,7 @@ from dbt.contracts.graph.nodes import (
|
||||
InjectedCTE,
|
||||
ManifestNode,
|
||||
ManifestSQLNode,
|
||||
ModelNode,
|
||||
SeedNode,
|
||||
UnitTestDefinition,
|
||||
UnitTestNode,
|
||||
@@ -29,12 +30,15 @@ from dbt.events.types import FoundStats, WritingInjectedSQLForNode
|
||||
from dbt.exceptions import (
|
||||
DbtInternalError,
|
||||
DbtRuntimeError,
|
||||
ForeignKeyConstraintToSyntaxError,
|
||||
GraphDependencyNotFoundError,
|
||||
ParsingError,
|
||||
)
|
||||
from dbt.flags import get_flags
|
||||
from dbt.graph import Graph
|
||||
from dbt.node_types import ModelLanguage, NodeType
|
||||
from dbt_common.clients.system import make_directory
|
||||
from dbt_common.contracts.constraints import ConstraintType
|
||||
from dbt_common.events.contextvars import get_node_info
|
||||
from dbt_common.events.format import pluralize
|
||||
from dbt_common.events.functions import fire_event
|
||||
@@ -371,7 +375,7 @@ class Compiler:
|
||||
|
||||
_extend_prepended_ctes(prepended_ctes, new_prepended_ctes)
|
||||
|
||||
new_cte_name = self.add_ephemeral_prefix(cte_model.name)
|
||||
new_cte_name = self.add_ephemeral_prefix(cte_model.identifier)
|
||||
rendered_sql = cte_model._pre_injected_sql or cte_model.compiled_code
|
||||
sql = f" {new_cte_name} as (\n{rendered_sql}\n)"
|
||||
|
||||
@@ -437,8 +441,31 @@ class Compiler:
|
||||
relation_name = str(relation_cls.create_from(self.config, node))
|
||||
node.relation_name = relation_name
|
||||
|
||||
# Compile 'ref' and 'source' expressions in foreign key constraints
|
||||
if isinstance(node, ModelNode):
|
||||
for constraint in node.all_constraints:
|
||||
if constraint.type == ConstraintType.foreign_key and constraint.to:
|
||||
constraint.to = self._compile_relation_for_foreign_key_constraint_to(
|
||||
manifest, node, constraint.to
|
||||
)
|
||||
|
||||
return node
|
||||
|
||||
def _compile_relation_for_foreign_key_constraint_to(
|
||||
self, manifest: Manifest, node: ManifestSQLNode, to_expression: str
|
||||
) -> str:
|
||||
try:
|
||||
foreign_key_node = manifest.find_node_from_ref_or_source(to_expression)
|
||||
except ParsingError:
|
||||
raise ForeignKeyConstraintToSyntaxError(node, to_expression)
|
||||
|
||||
if not foreign_key_node:
|
||||
raise GraphDependencyNotFoundError(node, to_expression)
|
||||
|
||||
adapter = get_adapter(self.config)
|
||||
relation_name = str(adapter.Relation.create_from(self.config, foreign_key_node))
|
||||
return relation_name
|
||||
|
||||
# This method doesn't actually "compile" any of the nodes. That is done by the
|
||||
# "compile_node" method. This creates a Linker and builds the networkx graph,
|
||||
# writes out the graph.gpickle file, and prints the stats, returning a Graph object.
|
||||
@@ -520,6 +547,8 @@ class Compiler:
|
||||
the node's raw_code into compiled_code, and then calls the
|
||||
recursive method to "prepend" the ctes.
|
||||
"""
|
||||
# REVIEW: UnitTestDefinition shouldn't be possible here because of the
|
||||
# type of node, and it is likewise an invalid return type.
|
||||
if isinstance(node, UnitTestDefinition):
|
||||
return node
|
||||
|
||||
|
||||
@@ -10,7 +10,7 @@ from dbt import deprecations
|
||||
from dbt.adapters.contracts.connection import QueryComment
|
||||
from dbt.clients.yaml_helper import load_yaml_text
|
||||
from dbt.config.selectors import SelectorDict
|
||||
from dbt.config.utils import exclusive_primary_alt_value_setting
|
||||
from dbt.config.utils import normalize_warn_error_options
|
||||
from dbt.constants import (
|
||||
DBT_PROJECT_FILE_NAME,
|
||||
DEPENDENCIES_FILE_NAME,
|
||||
@@ -480,6 +480,7 @@ class PartialProject(RenderComponents):
|
||||
rendered.selectors_dict["selectors"]
|
||||
)
|
||||
dbt_cloud = cfg.dbt_cloud
|
||||
flags: Dict[str, Any] = cfg.flags
|
||||
|
||||
project = Project(
|
||||
project_name=name,
|
||||
@@ -524,6 +525,7 @@ class PartialProject(RenderComponents):
|
||||
project_env_vars=project_env_vars,
|
||||
restrict_access=cfg.restrict_access,
|
||||
dbt_cloud=dbt_cloud,
|
||||
flags=flags,
|
||||
)
|
||||
# sanity check - this means an internal issue
|
||||
project.validate()
|
||||
@@ -568,11 +570,6 @@ class PartialProject(RenderComponents):
|
||||
) = package_and_project_data_from_root(project_root)
|
||||
selectors_dict = selector_data_from_root(project_root)
|
||||
|
||||
if "flags" in project_dict:
|
||||
# We don't want to include "flags" in the Project,
|
||||
# it goes in ProjectFlags
|
||||
project_dict.pop("flags")
|
||||
|
||||
return cls.from_dicts(
|
||||
project_root=project_root,
|
||||
project_dict=project_dict,
|
||||
@@ -645,6 +642,7 @@ class Project:
|
||||
project_env_vars: Dict[str, Any]
|
||||
restrict_access: bool
|
||||
dbt_cloud: Dict[str, Any]
|
||||
flags: Dict[str, Any]
|
||||
|
||||
@property
|
||||
def all_source_paths(self) -> List[str]:
|
||||
@@ -724,6 +722,7 @@ class Project:
|
||||
"require-dbt-version": [v.to_version_string() for v in self.dbt_version],
|
||||
"restrict-access": self.restrict_access,
|
||||
"dbt-cloud": self.dbt_cloud,
|
||||
"flags": self.flags,
|
||||
}
|
||||
)
|
||||
if self.query_comment:
|
||||
@@ -821,20 +820,15 @@ def read_project_flags(project_dir: str, profiles_dir: str) -> ProjectFlags:
|
||||
|
||||
if profile_project_flags:
|
||||
# This can't use WARN_ERROR or WARN_ERROR_OPTIONS because they're in
|
||||
# the config that we're loading. Uses special "warn" method.
|
||||
deprecations.warn("project-flags-moved")
|
||||
# the config that we're loading. Uses special "buffer" method and fired after flags are initialized in preflight.
|
||||
deprecations.buffer("project-flags-moved")
|
||||
project_flags = profile_project_flags
|
||||
|
||||
if project_flags is not None:
|
||||
# handle collapsing `include` and `error` as well as collapsing `exclude` and `warn`
|
||||
# for warn_error_options
|
||||
warn_error_options = project_flags.get("warn_error_options")
|
||||
exclusive_primary_alt_value_setting(
|
||||
warn_error_options, "include", "error", "warn_error_options"
|
||||
)
|
||||
exclusive_primary_alt_value_setting(
|
||||
warn_error_options, "exclude", "warn", "warn_error_options"
|
||||
)
|
||||
warn_error_options = project_flags.get("warn_error_options", {})
|
||||
normalize_warn_error_options(warn_error_options)
|
||||
|
||||
ProjectFlags.validate(project_flags)
|
||||
return ProjectFlags.from_dict(project_flags)
|
||||
|
||||
@@ -193,6 +193,7 @@ class RuntimeConfig(Project, Profile, AdapterRequiredConfig):
|
||||
log_cache_events=log_cache_events,
|
||||
dependencies=dependencies,
|
||||
dbt_cloud=project.dbt_cloud,
|
||||
flags=project.flags,
|
||||
)
|
||||
|
||||
# Called by 'load_projects' in this class
|
||||
@@ -290,9 +291,9 @@ class RuntimeConfig(Project, Profile, AdapterRequiredConfig):
|
||||
project_name=self.project_name,
|
||||
project_id=self.hashed_name(),
|
||||
user_id=tracking.active_user.id if tracking.active_user else None,
|
||||
send_anonymous_usage_stats=get_flags().SEND_ANONYMOUS_USAGE_STATS
|
||||
if tracking.active_user
|
||||
else None,
|
||||
send_anonymous_usage_stats=(
|
||||
get_flags().SEND_ANONYMOUS_USAGE_STATS if tracking.active_user else None
|
||||
),
|
||||
adapter_type=self.credentials.type,
|
||||
)
|
||||
|
||||
|
||||
@@ -49,5 +49,18 @@ def exclusive_primary_alt_value_setting(
|
||||
f"Only `{alt}` or `{primary}` can be specified{where}, not both"
|
||||
)
|
||||
|
||||
if alt_options:
|
||||
dictionary[primary] = alt_options
|
||||
if alt in dictionary:
|
||||
alt_value = dictionary.pop(alt)
|
||||
dictionary[primary] = alt_value
|
||||
|
||||
|
||||
def normalize_warn_error_options(warn_error_options: Dict[str, Any]) -> None:
|
||||
exclusive_primary_alt_value_setting(
|
||||
warn_error_options, "include", "error", "warn_error_options"
|
||||
)
|
||||
exclusive_primary_alt_value_setting(
|
||||
warn_error_options, "exclude", "warn", "warn_error_options"
|
||||
)
|
||||
for key in ("include", "exclude", "silence"):
|
||||
if key in warn_error_options and warn_error_options[key] is None:
|
||||
warn_error_options[key] = []
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
from dbt_semantic_interfaces.type_enums import TimeGranularity
|
||||
|
||||
DEFAULT_ENV_PLACEHOLDER = "DBT_DEFAULT_PLACEHOLDER"
|
||||
|
||||
SECRET_PLACEHOLDER = "$$$DBT_SECRET_START$$${}$$$DBT_SECRET_END$$$"
|
||||
@@ -15,5 +17,8 @@ DEPENDENCIES_FILE_NAME = "dependencies.yml"
|
||||
PACKAGE_LOCK_FILE_NAME = "package-lock.yml"
|
||||
MANIFEST_FILE_NAME = "manifest.json"
|
||||
SEMANTIC_MANIFEST_FILE_NAME = "semantic_manifest.json"
|
||||
LEGACY_TIME_SPINE_MODEL_NAME = "metricflow_time_spine"
|
||||
LEGACY_TIME_SPINE_GRANULARITY = TimeGranularity.DAY
|
||||
MINIMUM_REQUIRED_TIME_SPINE_GRANULARITY = TimeGranularity.DAY
|
||||
PARTIAL_PARSE_FILE_NAME = "partial_parse.msgpack"
|
||||
PACKAGE_LOCK_HASH_KEY = "sha1_hash"
|
||||
|
||||
@@ -27,8 +27,7 @@ class ConfigSource:
|
||||
def __init__(self, project):
|
||||
self.project = project
|
||||
|
||||
def get_config_dict(self, resource_type: NodeType):
|
||||
...
|
||||
def get_config_dict(self, resource_type: NodeType): ...
|
||||
|
||||
|
||||
class UnrenderedConfig(ConfigSource):
|
||||
@@ -130,12 +129,12 @@ class BaseContextConfigGenerator(Generic[T]):
|
||||
return self._project_configs(self._active_project, fqn, resource_type)
|
||||
|
||||
@abstractmethod
|
||||
def _update_from_config(self, result: T, partial: Dict[str, Any], validate: bool = False) -> T:
|
||||
...
|
||||
def _update_from_config(
|
||||
self, result: T, partial: Dict[str, Any], validate: bool = False
|
||||
) -> T: ...
|
||||
|
||||
@abstractmethod
|
||||
def initial_result(self, resource_type: NodeType, base: bool) -> T:
|
||||
...
|
||||
def initial_result(self, resource_type: NodeType, base: bool) -> T: ...
|
||||
|
||||
def calculate_node_config(
|
||||
self,
|
||||
@@ -181,8 +180,7 @@ class BaseContextConfigGenerator(Generic[T]):
|
||||
project_name: str,
|
||||
base: bool,
|
||||
patch_config_dict: Optional[Dict[str, Any]] = None,
|
||||
) -> Dict[str, Any]:
|
||||
...
|
||||
) -> Dict[str, Any]: ...
|
||||
|
||||
|
||||
class ContextConfigGenerator(BaseContextConfigGenerator[C]):
|
||||
|
||||
@@ -8,7 +8,7 @@ from dbt.adapters.exceptions import (
|
||||
RelationWrongTypeError,
|
||||
)
|
||||
from dbt.adapters.exceptions.cache import CacheInconsistencyError
|
||||
from dbt.events.types import JinjaLogWarning
|
||||
from dbt.events.types import JinjaLogWarning, SnapshotTimestampWarning
|
||||
from dbt.exceptions import (
|
||||
AmbiguousAliasError,
|
||||
AmbiguousCatalogMatchError,
|
||||
@@ -116,6 +116,17 @@ def raise_fail_fast_error(msg, node=None) -> NoReturn:
|
||||
raise FailFastError(msg, node=node)
|
||||
|
||||
|
||||
def warn_snapshot_timestamp_data_types(
|
||||
snapshot_time_data_type: str, updated_at_data_type: str
|
||||
) -> None:
|
||||
warn_or_error(
|
||||
SnapshotTimestampWarning(
|
||||
snapshot_time_data_type=snapshot_time_data_type,
|
||||
updated_at_data_type=updated_at_data_type,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
# Update this when a new function should be added to the
|
||||
# dbt context's `exceptions` key!
|
||||
CONTEXT_EXPORTS = {
|
||||
@@ -141,6 +152,7 @@ CONTEXT_EXPORTS = {
|
||||
raise_contract_error,
|
||||
column_type_missing,
|
||||
raise_fail_fast_error,
|
||||
warn_snapshot_timestamp_data_types,
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
@@ -239,8 +239,7 @@ class BaseRefResolver(BaseResolver):
|
||||
@abc.abstractmethod
|
||||
def resolve(
|
||||
self, name: str, package: Optional[str] = None, version: Optional[NodeVersion] = None
|
||||
) -> RelationProxy:
|
||||
...
|
||||
) -> RelationProxy: ...
|
||||
|
||||
def _repack_args(
|
||||
self, name: str, package: Optional[str], version: Optional[NodeVersion]
|
||||
@@ -306,8 +305,7 @@ class BaseSourceResolver(BaseResolver):
|
||||
|
||||
class BaseMetricResolver(BaseResolver):
|
||||
@abc.abstractmethod
|
||||
def resolve(self, name: str, package: Optional[str] = None) -> MetricReference:
|
||||
...
|
||||
def resolve(self, name: str, package: Optional[str] = None) -> MetricReference: ...
|
||||
|
||||
def _repack_args(self, name: str, package: Optional[str]) -> List[str]:
|
||||
if package is None:
|
||||
@@ -341,8 +339,7 @@ class BaseMetricResolver(BaseResolver):
|
||||
|
||||
|
||||
class Config(Protocol):
|
||||
def __init__(self, model, context_config: Optional[ContextConfig]):
|
||||
...
|
||||
def __init__(self, model, context_config: Optional[ContextConfig]): ...
|
||||
|
||||
|
||||
# Implementation of "config(..)" calls in models
|
||||
@@ -977,7 +974,8 @@ class ProviderContext(ManifestContext):
|
||||
table = agate_helper.from_csv(path, text_columns=column_types, delimiter=delimiter)
|
||||
except ValueError as e:
|
||||
raise LoadAgateTableValueError(e, node=self.model)
|
||||
table.original_abspath = os.path.abspath(path)
|
||||
# this is used by some adapters
|
||||
table.original_abspath = os.path.abspath(path) # type: ignore
|
||||
return table
|
||||
|
||||
@contextproperty()
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
## Artifacts
|
||||
|
||||
### Generating JSON schemas
|
||||
A helper script, `sripts/collect-artifact-schema.py` is available to generate json schemas corresponding to versioned artifacts (`ArtifactMixin`s).
|
||||
A helper script, `scripts/collect-artifact-schema.py` is available to generate json schemas corresponding to versioned artifacts (`ArtifactMixin`s).
|
||||
|
||||
This script is necessary to run when a new artifact schema version is created, or when changes are made to existing artifact versions, and writes json schema to `schema/dbt/<artifact>/v<version>.json`.
|
||||
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user