mirror of
https://github.com/dbt-labs/dbt-core
synced 2025-12-18 22:31:28 +00:00
Compare commits
30 Commits
postgres-s
...
v1.2.5
| Author | SHA1 | Date | |
|---|---|---|---|
|
eb4ac514c5 | ||
|
4342a2a159 | ||
|
5b3fbe7c57 | ||
|
5751b7ff3a | ||
|
ba3e5a07f7 | ||
|
45d5276114 | ||
|
cfaaad5d3b | ||
|
d6d1d14466 | ||
|
|
d21f2cd090 | ||
|
|
d6b5147566 | ||
|
|
e4cbc9790b | ||
|
|
b25a09d374 | ||
|
|
111c639f03 | ||
|
|
abee0f0dc1 | ||
|
|
f903aed3e0 | ||
|
|
1aee7b3700 | ||
|
|
6278880fa0 | ||
|
|
07415ca3e7 | ||
|
|
1d6eb7dcd5 | ||
|
|
7955fe7c3b | ||
|
|
f82cddbefe | ||
|
|
39901c5155 | ||
|
|
6c21d10469 | ||
|
|
9a10de926e | ||
|
|
9c32441d05 | ||
|
|
c7652f6a33 | ||
|
|
a2a531722e | ||
|
|
7656ffb9e6 | ||
|
3f3792882c | ||
|
|
0db634d12f |
@@ -1,13 +1,15 @@
|
||||
[bumpversion]
|
||||
current_version = 1.2.0b1
|
||||
parse = (?P<major>\d+)
|
||||
\.(?P<minor>\d+)
|
||||
\.(?P<patch>\d+)
|
||||
((?P<prekind>a|b|rc)
|
||||
(?P<pre>\d+) # pre-release version num
|
||||
current_version = 1.2.5
|
||||
parse = (?P<major>[\d]+) # major version number
|
||||
\.(?P<minor>[\d]+) # minor version number
|
||||
\.(?P<patch>[\d]+) # patch version number
|
||||
(((?P<prekind>a|b|rc) # optional pre-release type
|
||||
?(?P<num>[\d]+?)) # optional pre-release version number
|
||||
\.?(?P<nightly>[a-z0-9]+\+[a-z]+)? # optional nightly release indicator
|
||||
)?
|
||||
serialize =
|
||||
{major}.{minor}.{patch}{prekind}{pre}
|
||||
{major}.{minor}.{patch}{prekind}{num}.{nightly}
|
||||
{major}.{minor}.{patch}{prekind}{num}
|
||||
{major}.{minor}.{patch}
|
||||
commit = False
|
||||
tag = False
|
||||
@@ -21,9 +23,11 @@ values =
|
||||
rc
|
||||
final
|
||||
|
||||
[bumpversion:part:pre]
|
||||
[bumpversion:part:num]
|
||||
first_value = 1
|
||||
|
||||
[bumpversion:part:nightly]
|
||||
|
||||
[bumpversion:file:core/setup.py]
|
||||
|
||||
[bumpversion:file:core/dbt/version.py]
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
## dbt-core 1.2.0-b1 - June 24, 2022
|
||||
## dbt-core 1.2.0 - July 26, 2022
|
||||
### Features
|
||||
- Add selector method when reading selector definitions ([#4821](https://github.com/dbt-labs/dbt-core/issues/4821), [#4827](https://github.com/dbt-labs/dbt-core/pull/4827))
|
||||
- Add set and zip function to contexts ([#2345](https://github.com/dbt-labs/dbt-core/issues/2345), [#5107](https://github.com/dbt-labs/dbt-core/pull/5107))
|
||||
@@ -10,6 +10,11 @@
|
||||
- Move cross-db macros from dbt-utils into dbt-core global project ([#4813](https://github.com/dbt-labs/dbt-core/issues/4813), [#5265](https://github.com/dbt-labs/dbt-core/pull/5265))
|
||||
- Prettify duration message at the end of execution ([#5253](https://github.com/dbt-labs/dbt-core/issues/5253), [#5364](https://github.com/dbt-labs/dbt-core/pull/5364))
|
||||
- Early return from dbt init if no available adapters ([#5365](https://github.com/dbt-labs/dbt-core/issues/5365), [#5366](https://github.com/dbt-labs/dbt-core/pull/5366))
|
||||
- Allow customizing `target-path` and `log-path` through environment variables and CLI flags. ([#5399](https://github.com/dbt-labs/dbt-core/issues/5399), [#5402](https://github.com/dbt-labs/dbt-core/pull/5402))
|
||||
- Move type_* macros from dbt-utils into dbt-core, with tests ([#5317](https://github.com/dbt-labs/dbt-core/issues/5317), [#5428](https://github.com/dbt-labs/dbt-core/pull/5428))
|
||||
- Add support for ratio metrics ([#4884](https://github.com/dbt-labs/dbt-core/issues/4884), [#5027](https://github.com/dbt-labs/dbt-core/pull/5027))
|
||||
- Allow users to define grants as a reasonable default in the dbt_project.yml or within each model sql or yml file combined. ([#5263](https://github.com/dbt-labs/dbt-core/issues/5263), [#5369](https://github.com/dbt-labs/dbt-core/pull/5369))
|
||||
- Add reusable function for retrying adapter connections. Utilize said function to add retries for Postgres (and Redshift). ([#5022](https://github.com/dbt-labs/dbt-core/issues/5022), [#5432](https://github.com/dbt-labs/dbt-core/pull/5432))
|
||||
### Fixes
|
||||
- Adding new cols to check_cols in snapshots ([#3146](https://github.com/dbt-labs/dbt-core/issues/3146), [#4893](https://github.com/dbt-labs/dbt-core/pull/4893))
|
||||
- Truncate relation names when appending a suffix that will result in len > 63 characters using make_temp_relation and make_backup_relation macros ([#2869](https://github.com/dbt-labs/dbt-core/issues/2869), [#4921](https://github.com/dbt-labs/dbt-core/pull/4921))
|
||||
@@ -29,7 +34,19 @@
|
||||
- Change node ancestor/descendant algo, fixes issue where downstream models aren't run when using networkx >= 2.8.1 ([#5286](https://github.com/dbt-labs/dbt-core/issues/5286), [#5326](https://github.com/dbt-labs/dbt-core/pull/5326))
|
||||
- Fixing Windows color regression ([#5191](https://github.com/dbt-labs/dbt-core/issues/5191), [#5327](https://github.com/dbt-labs/dbt-core/pull/5327))
|
||||
- Define compatibility for older manifest versions when using state: selection methods ([#5213](https://github.com/dbt-labs/dbt-core/issues/5213), [#5346](https://github.com/dbt-labs/dbt-core/pull/5346))
|
||||
- Add inheritance to materialization macro resolution ([#4646](https://github.com/dbt-labs/dbt-core/issues/4646), [#5348](https://github.com/dbt-labs/dbt-core/pull/5348))
|
||||
- Improve pluralizations for Documentation and SqlOperation NodeTypes ([#5352](https://github.com/dbt-labs/dbt-core/issues/5352), [#5356](https://github.com/dbt-labs/dbt-core/pull/5356))
|
||||
- Properly use quotes for Snowflake snapshots when checking all columns ([#2975](https://github.com/dbt-labs/dbt-core/issues/2975), [#5389](https://github.com/dbt-labs/dbt-core/pull/5389))
|
||||
- fixes handling of RESET color code with USE_COLORS=False ([#5288](https://github.com/dbt-labs/dbt-core/issues/5288), [#5394](https://github.com/dbt-labs/dbt-core/pull/5394))
|
||||
- Remove duplicate key checking introduced in 1.2.0a1 ([#5331](https://github.com/dbt-labs/dbt-core/issues/5331), [#5403](https://github.com/dbt-labs/dbt-core/pull/5403))
|
||||
- Rename try to strict for more intuitiveness ([#5475](https://github.com/dbt-labs/dbt-core/issues/5475), [#5477](https://github.com/dbt-labs/dbt-core/pull/5477))
|
||||
### Docs
|
||||
- Fixed sample SQL Code for sources when no database is defined ([#5255](https://github.com/dbt-labs/dbt-core/issues/5255), [#5446](https://github.com/dbt-labs/dbt-core/pull/5446))
|
||||
- Add support for `file:` selector in DAG viz ([#5255](https://github.com/dbt-labs/dbt-core/issues/5255), [#5446](https://github.com/dbt-labs/dbt-core/pull/5446))
|
||||
- [Snyk] Upgrade prismjs from 1.27.0 to 1.28.0 ([#5255](https://github.com/dbt-labs/dbt-core/issues/5255), [#5446](https://github.com/dbt-labs/dbt-core/pull/5446))
|
||||
- Run build and tests in CI checks ([#5255](https://github.com/dbt-labs/dbt-core/issues/5255), [#5446](https://github.com/dbt-labs/dbt-core/pull/5446))
|
||||
- Improve metrics DAG viz and documentation page ([#5255](https://github.com/dbt-labs/dbt-core/issues/5255), [#5446](https://github.com/dbt-labs/dbt-core/pull/5446))
|
||||
- Upgrade cytoscape.js fork ([#5255](https://github.com/dbt-labs/dbt-core/issues/5255), [#5446](https://github.com/dbt-labs/dbt-core/pull/5446))
|
||||
### Under the Hood
|
||||
- Migrating 005_simple_seed to the new test framework. ([#200](https://github.com/dbt-labs/dbt-core/issues/200), [#5013](https://github.com/dbt-labs/dbt-core/pull/5013))
|
||||
- Convert 029_docs_generate tests to new framework ([#5035](https://github.com/dbt-labs/dbt-core/issues/5035), [#5058](https://github.com/dbt-labs/dbt-core/pull/5058))
|
||||
@@ -46,24 +63,41 @@
|
||||
- Fix unit test test_graph_selection ([#5323](https://github.com/dbt-labs/dbt-core/issues/5323), [#5324](https://github.com/dbt-labs/dbt-core/pull/5324))
|
||||
- Update context readme + clean up context code" ([#4796](https://github.com/dbt-labs/dbt-core/issues/4796), [#5334](https://github.com/dbt-labs/dbt-core/pull/5334))
|
||||
- removed script meant for snowflake to snowflake ([#5361](https://github.com/dbt-labs/dbt-core/issues/5361), [#5362](https://github.com/dbt-labs/dbt-core/pull/5362))
|
||||
- Added the suggested RegEx to check the SemVer string within a package dependency and improved invalid version error handling. ([#5201](https://github.com/dbt-labs/dbt-core/issues/5201), [#5370](https://github.com/dbt-labs/dbt-core/pull/5370))
|
||||
- Add annotation to render_value method reimplemented in #5334 ([#4796](https://github.com/dbt-labs/dbt-core/issues/4796), [#5382](https://github.com/dbt-labs/dbt-core/pull/5382))
|
||||
- Bump manifest version to v6 ([#5417](https://github.com/dbt-labs/dbt-core/issues/5417), [#5430](https://github.com/dbt-labs/dbt-core/pull/5430))
|
||||
- Add tests for SQL grants ([#5437](https://github.com/dbt-labs/dbt-core/issues/5437), [#5447](https://github.com/dbt-labs/dbt-core/pull/5447))
|
||||
### Dependencies
|
||||
- Bump ubuntu from 20.04 to 22.04 ([#4904](https://github.com/dbt-labs/dbt-core/issues/4904), [#5141](https://github.com/dbt-labs/dbt-core/pull/5141))
|
||||
- Bumping hologram version ([#5219](https://github.com/dbt-labs/dbt-core/issues/5219), [#5218](https://github.com/dbt-labs/dbt-core/pull/5218))
|
||||
- Bump mypy from 0.942 to 0.961 ([#4904](https://github.com/dbt-labs/dbt-core/issues/4904), [#5337](https://github.com/dbt-labs/dbt-core/pull/5337))
|
||||
- Bump python from 3.10.3-slim-bullseye to 3.10.5-slim-bullseye in /docker ([#4904](https://github.com/dbt-labs/dbt-core/issues/4904), [#5367](https://github.com/dbt-labs/dbt-core/pull/5367))
|
||||
- Update colorama requirement from <0.4.5,>=0.3.9 to >=0.3.9,<0.4.6 in /core ([#4904](https://github.com/dbt-labs/dbt-core/issues/4904), [#5388](https://github.com/dbt-labs/dbt-core/pull/5388))
|
||||
- Bump black from 22.3.0 to 22.6.0 ([#4904](https://github.com/dbt-labs/dbt-core/issues/4904), [#5420](https://github.com/dbt-labs/dbt-core/pull/5420))
|
||||
### Security
|
||||
- Move string interpolation of "secret" env vars outside of Jinja context. Update "contexts" README ([#4796](https://github.com/dbt-labs/dbt-core/issues/4796), [#5334](https://github.com/dbt-labs/dbt-core/pull/5334))
|
||||
|
||||
### Contributors
|
||||
- [@GtheSheep](https://github.com/GtheSheep) ([#4893](https://github.com/dbt-labs/dbt-core/pull/4893))
|
||||
- [@NicolasPA](https://github.com/NicolasPA) ([#5211](https://github.com/dbt-labs/dbt-core/pull/5211))
|
||||
- [@adamantike](https://github.com/adamantike) ([#5207](https://github.com/dbt-labs/dbt-core/pull/5207))
|
||||
- [@alexrosenfeld10](https://github.com/alexrosenfeld10) ([#5184](https://github.com/dbt-labs/dbt-core/pull/5184))
|
||||
- [@b-per](https://github.com/b-per) ([#5446](https://github.com/dbt-labs/dbt-core/pull/5446))
|
||||
- [@bd3dowling](https://github.com/bd3dowling) ([#5140](https://github.com/dbt-labs/dbt-core/pull/5140))
|
||||
- [@callum-mcdata](https://github.com/callum-mcdata) ([#5027](https://github.com/dbt-labs/dbt-core/pull/5027), [#5446](https://github.com/dbt-labs/dbt-core/pull/5446))
|
||||
- [@danieldiamond](https://github.com/danieldiamond) ([#4827](https://github.com/dbt-labs/dbt-core/pull/4827))
|
||||
- [@darin-reify](https://github.com/darin-reify) ([#5394](https://github.com/dbt-labs/dbt-core/pull/5394))
|
||||
- [@dbeatty10](https://github.com/dbeatty10) ([#5265](https://github.com/dbt-labs/dbt-core/pull/5265), [#5077](https://github.com/dbt-labs/dbt-core/pull/5077))
|
||||
- [@dependabot[bot]](https://github.com/dependabot[bot]) ([#5141](https://github.com/dbt-labs/dbt-core/pull/5141), [#5367](https://github.com/dbt-labs/dbt-core/pull/5367))
|
||||
- [@drewbanin](https://github.com/drewbanin) ([#5027](https://github.com/dbt-labs/dbt-core/pull/5027), [#5446](https://github.com/dbt-labs/dbt-core/pull/5446), [#5446](https://github.com/dbt-labs/dbt-core/pull/5446), [#5446](https://github.com/dbt-labs/dbt-core/pull/5446))
|
||||
- [@epapineau](https://github.com/epapineau) ([#4921](https://github.com/dbt-labs/dbt-core/pull/4921))
|
||||
- [@fivetran-joemarkiewicz](https://github.com/fivetran-joemarkiewicz) ([#5370](https://github.com/dbt-labs/dbt-core/pull/5370))
|
||||
- [@groodt](https://github.com/groodt) ([#5304](https://github.com/dbt-labs/dbt-core/pull/5304))
|
||||
- [@isidentical](https://github.com/isidentical) ([#5402](https://github.com/dbt-labs/dbt-core/pull/5402))
|
||||
- [@jared-rimmer](https://github.com/jared-rimmer) ([#5364](https://github.com/dbt-labs/dbt-core/pull/5364))
|
||||
- [@jeremyyeo](https://github.com/jeremyyeo) ([#5107](https://github.com/dbt-labs/dbt-core/pull/5107), [#5146](https://github.com/dbt-labs/dbt-core/pull/5146), [#5403](https://github.com/dbt-labs/dbt-core/pull/5403))
|
||||
- [@jeremyyeo](https://github.com/jeremyyeo) ([#5107](https://github.com/dbt-labs/dbt-core/pull/5107), [#5146](https://github.com/dbt-labs/dbt-core/pull/5146), [#5403](https://github.com/dbt-labs/dbt-core/pull/5403), [#5477](https://github.com/dbt-labs/dbt-core/pull/5477))
|
||||
- [@jwills](https://github.com/jwills) ([#5241](https://github.com/dbt-labs/dbt-core/pull/5241), [#5269](https://github.com/dbt-labs/dbt-core/pull/5269))
|
||||
- [@tomasfarias](https://github.com/tomasfarias) ([#5209](https://github.com/dbt-labs/dbt-core/pull/5209))
|
||||
- [@pdebelak](https://github.com/pdebelak) ([#5356](https://github.com/dbt-labs/dbt-core/pull/5356))
|
||||
- [@pquadri](https://github.com/pquadri) ([#5389](https://github.com/dbt-labs/dbt-core/pull/5389))
|
||||
- [@tomasfarias](https://github.com/tomasfarias) ([#5432](https://github.com/dbt-labs/dbt-core/pull/5432), [#5209](https://github.com/dbt-labs/dbt-core/pull/5209))
|
||||
- [@ulisesojeda](https://github.com/ulisesojeda) ([#5366](https://github.com/dbt-labs/dbt-core/pull/5366))
|
||||
- [@volkangurel](https://github.com/volkangurel) ([#5348](https://github.com/dbt-labs/dbt-core/pull/5348))
|
||||
@@ -1,7 +0,0 @@
|
||||
kind: Dependencies
|
||||
body: "Bump ubuntu from 20.04 to 22.04"
|
||||
time: 2022-04-27T19:51:28.000000-05:00
|
||||
custom:
|
||||
Author: dependabot[bot]
|
||||
Issue: "4904"
|
||||
PR: "5141"
|
||||
@@ -1,7 +0,0 @@
|
||||
kind: Dependencies
|
||||
body: "Bumping hologram version"
|
||||
time: 2022-05-06T16:09:07.000000-05:00
|
||||
custom:
|
||||
Author: leahwicz
|
||||
Issue: "5219"
|
||||
PR: "5218"
|
||||
@@ -1,7 +0,0 @@
|
||||
kind: Dependencies
|
||||
body: "Bump python from 3.10.3-slim-bullseye to 3.10.5-slim-bullseye in /docker"
|
||||
time: 2022-06-13T00:14:56.000000-05:00
|
||||
custom:
|
||||
Author: dependabot[bot]
|
||||
Issue: "4904"
|
||||
PR: "5367"
|
||||
@@ -1,7 +0,0 @@
|
||||
kind: Features
|
||||
body: Add selector method when reading selector definitions
|
||||
time: 2022-04-08T11:26:10.713088+10:00
|
||||
custom:
|
||||
Author: danieldiamond
|
||||
Issue: "4821"
|
||||
PR: "4827"
|
||||
@@ -1,7 +0,0 @@
|
||||
kind: Features
|
||||
body: Add set and zip function to contexts
|
||||
time: 2022-04-23T23:17:56.851793+12:00
|
||||
custom:
|
||||
Author: jeremyyeo
|
||||
Issue: "2345"
|
||||
PR: "5107"
|
||||
@@ -1,7 +0,0 @@
|
||||
kind: Features
|
||||
body: Adds itertools to modules Jinja namespace
|
||||
time: 2022-04-24T13:26:55.008246+01:00
|
||||
custom:
|
||||
Author: bd3dowling
|
||||
Issue: "5130"
|
||||
PR: "5140"
|
||||
@@ -1,7 +0,0 @@
|
||||
kind: Features
|
||||
body: allow target as an option in profile_template.yml
|
||||
time: 2022-04-28T06:56:44.511519-04:00
|
||||
custom:
|
||||
Author: alexrosenfeld10
|
||||
Issue: "5179"
|
||||
PR: "5184"
|
||||
@@ -1,7 +0,0 @@
|
||||
kind: Features
|
||||
body: 'seed: Add new macro get_csv_sql'
|
||||
time: 2022-05-03T14:29:34.847959075Z
|
||||
custom:
|
||||
Author: adamantike
|
||||
Issue: "5206"
|
||||
PR: "5207"
|
||||
@@ -1,7 +0,0 @@
|
||||
kind: Features
|
||||
body: Grants as Node Configs
|
||||
time: 2022-05-10T20:49:49.197999-04:00
|
||||
custom:
|
||||
Author: gshank
|
||||
Issue: "5189"
|
||||
PR: "5230"
|
||||
@@ -1,7 +0,0 @@
|
||||
kind: Features
|
||||
body: Adds file selectors and support for file selectors in the default method selector
|
||||
time: 2022-05-12T21:57:48.289674-07:00
|
||||
custom:
|
||||
Author: jwills
|
||||
Issue: "5240"
|
||||
PR: "5241"
|
||||
@@ -1,7 +0,0 @@
|
||||
kind: Features
|
||||
body: Move cross-db macros from dbt-utils into dbt-core global project
|
||||
time: 2022-05-18T11:46:04.557104+02:00
|
||||
custom:
|
||||
Author: jtcohen6 dbeatty10
|
||||
Issue: "4813"
|
||||
PR: "5265"
|
||||
@@ -1,7 +0,0 @@
|
||||
kind: Features
|
||||
body: Prettify duration message at the end of execution
|
||||
time: 2022-06-11T16:39:31.725960083+01:00
|
||||
custom:
|
||||
Author: jared-rimmer
|
||||
Issue: "5253"
|
||||
PR: "5364"
|
||||
@@ -1,7 +0,0 @@
|
||||
kind: Features
|
||||
body: Early return from dbt init if no available adapters
|
||||
time: 2022-06-14T08:20:51.096872718+02:00
|
||||
custom:
|
||||
Author: ulisesojeda
|
||||
Issue: "5365"
|
||||
PR: "5366"
|
||||
@@ -1,7 +0,0 @@
|
||||
kind: Fixes
|
||||
body: Adding new cols to check_cols in snapshots
|
||||
time: 2022-03-17T21:09:16.977086+01:00
|
||||
custom:
|
||||
Author: GtheSheep
|
||||
Issue: "3146"
|
||||
PR: "4893"
|
||||
@@ -1,8 +0,0 @@
|
||||
kind: Fixes
|
||||
body: Truncate relation names when appending a suffix that will result in len > 63
|
||||
characters using make_temp_relation and make_backup_relation macros
|
||||
time: 2022-03-22T17:37:53.320082-07:00
|
||||
custom:
|
||||
Author: epapineau
|
||||
Issue: "2869"
|
||||
PR: "4921"
|
||||
@@ -1,7 +0,0 @@
|
||||
kind: Fixes
|
||||
body: Restore ability to utilize `updated_at` for check_cols snapshots
|
||||
time: 2022-04-15T11:29:27.063462-06:00
|
||||
custom:
|
||||
Author: dbeatty10
|
||||
Issue: "5076"
|
||||
PR: "5077"
|
||||
@@ -1,7 +0,0 @@
|
||||
kind: Fixes
|
||||
body: Fix retry logic to return values after initial try
|
||||
time: 2022-04-22T13:12:27.239055-05:00
|
||||
custom:
|
||||
Author: emmyoop
|
||||
Issue: "5023"
|
||||
PR: "5137"
|
||||
@@ -1,7 +0,0 @@
|
||||
kind: Fixes
|
||||
body: Use yaml renderer (with target context) for rendering selectors
|
||||
time: 2022-04-22T13:56:45.147893-04:00
|
||||
custom:
|
||||
Author: gshank
|
||||
Issue: "5131"
|
||||
PR: "5136"
|
||||
@@ -1,7 +0,0 @@
|
||||
kind: Fixes
|
||||
body: Scrub secret env vars from CommandError in exception stacktrace
|
||||
time: 2022-04-25T20:39:24.365495+02:00
|
||||
custom:
|
||||
Author: jtcohen6
|
||||
Issue: "5151"
|
||||
PR: "5152"
|
||||
@@ -1,7 +0,0 @@
|
||||
kind: Fixes
|
||||
body: Ensure the metric name does not contain spaces
|
||||
time: 2022-04-26T20:21:04.360693-04:00
|
||||
custom:
|
||||
Author: gshank
|
||||
Issue: "4572"
|
||||
PR: "5173"
|
||||
@@ -1,7 +0,0 @@
|
||||
kind: Fixes
|
||||
body: When parsing 'all_sources' should be a list of unique dirs
|
||||
time: 2022-04-27T10:26:48.648388-04:00
|
||||
custom:
|
||||
Author: gshank
|
||||
Issue: "5120"
|
||||
PR: "5176"
|
||||
@@ -1,7 +0,0 @@
|
||||
kind: Fixes
|
||||
body: Add warning if yaml contains duplicate keys
|
||||
time: 2022-04-28T10:01:57.893956+12:00
|
||||
custom:
|
||||
Author: jeremyyeo
|
||||
Issue: "5114"
|
||||
PR: "5146"
|
||||
@@ -1,8 +0,0 @@
|
||||
kind: Fixes
|
||||
body: Modifying the drop_test_schema to work better with Redshift issues around locked
|
||||
tables and current transactions
|
||||
time: 2022-04-29T16:07:42.750046-05:00
|
||||
custom:
|
||||
Author: Mcknight-42
|
||||
Issue: "5200"
|
||||
PR: "5198"
|
||||
@@ -1,8 +0,0 @@
|
||||
kind: Fixes
|
||||
body: Fix column comparison in snapshot_check_all_get_existing_columns for check-strategy
|
||||
snapshots with explicit check_cols defined
|
||||
time: 2022-05-09T13:00:21.649028+02:00
|
||||
custom:
|
||||
Author: jtcohen6
|
||||
Issue: "5222"
|
||||
PR: "5223"
|
||||
@@ -1,8 +0,0 @@
|
||||
kind: Fixes
|
||||
body: Changed how `--select state:modified` detects changes for macros nodes depend
|
||||
on
|
||||
time: 2022-05-09T13:13:12.889074-05:00
|
||||
custom:
|
||||
Author: stu-k
|
||||
Issue: "5202"
|
||||
PR: "5224"
|
||||
@@ -1,7 +0,0 @@
|
||||
kind: Fixes
|
||||
body: Fix column comparison in snapshot_check_all_get_existing_columns to use adapter.get_columns_in_relation
|
||||
time: 2022-05-11T12:32:38.313321+02:00
|
||||
custom:
|
||||
Author: jtcohen6
|
||||
Issue: "5222"
|
||||
PR: "5232"
|
||||
@@ -1,7 +0,0 @@
|
||||
kind: Fixes
|
||||
body: Remove docs file from manifest when removing doc node
|
||||
time: 2022-05-18T13:46:10.167143-04:00
|
||||
custom:
|
||||
Author: gshank
|
||||
Issue: "4146"
|
||||
PR: "5270"
|
||||
@@ -1,7 +0,0 @@
|
||||
kind: Fixes
|
||||
body: Remove duplicate dbt script entry
|
||||
time: 2022-06-01T08:13:14.067001+10:00
|
||||
custom:
|
||||
Author: groodt
|
||||
Issue: "5314"
|
||||
PR: "5304"
|
||||
@@ -1,8 +0,0 @@
|
||||
kind: Fixes
|
||||
body: Change node ancestor/descendant algo, fixes issue where downstream models aren't
|
||||
run when using networkx >= 2.8.1
|
||||
time: 2022-06-01T13:59:08.886215-05:00
|
||||
custom:
|
||||
Author: iknox-fa
|
||||
Issue: "5286"
|
||||
PR: "5326"
|
||||
@@ -1,7 +0,0 @@
|
||||
kind: Fixes
|
||||
body: Fixing Windows color regression
|
||||
time: 2022-06-01T19:42:34.263009-04:00
|
||||
custom:
|
||||
Author: leahwicz
|
||||
Issue: "5191"
|
||||
PR: "5327"
|
||||
@@ -1,8 +0,0 @@
|
||||
kind: Fixes
|
||||
body: 'Define compatibility for older manifest versions when using state: selection
|
||||
methods'
|
||||
time: 2022-06-08T08:09:14.321735+02:00
|
||||
custom:
|
||||
Author: jtcohen6
|
||||
Issue: "5213"
|
||||
PR: "5346"
|
||||
@@ -1,7 +0,0 @@
|
||||
kind: Fixes
|
||||
body: Remove duplicate key checking introduced in 1.2.0a1
|
||||
time: 2022-06-23T08:24:31.900647+12:00
|
||||
custom:
|
||||
Author: jeremyyeo
|
||||
Issue: "5331"
|
||||
PR: "5403"
|
||||
@@ -1,7 +0,0 @@
|
||||
kind: Under the Hood
|
||||
body: Migrating 005_simple_seed to the new test framework.
|
||||
time: 2022-04-09T04:05:39.20045-07:00
|
||||
custom:
|
||||
Author: versusfacit
|
||||
Issue: "200"
|
||||
PR: "5013"
|
||||
@@ -1,7 +0,0 @@
|
||||
kind: Under the Hood
|
||||
body: Convert 029_docs_generate tests to new framework
|
||||
time: 2022-04-13T18:30:14.706391-04:00
|
||||
custom:
|
||||
Author: gshank
|
||||
Issue: "5035"
|
||||
PR: "5058"
|
||||
@@ -1,7 +0,0 @@
|
||||
kind: Under the Hood
|
||||
body: Move package deprecation check outside of package cache
|
||||
time: 2022-04-14T13:22:06.157579-05:00
|
||||
custom:
|
||||
Author: emmyoop
|
||||
Issue: "5068"
|
||||
PR: "5069"
|
||||
@@ -1,7 +0,0 @@
|
||||
kind: Under the Hood
|
||||
body: removal of scaffold first attempt and create_adapter_plugin.py as they are deprecated new scaffold can be found https://github.com/dbt-labs/dbt-database-adapter-scaffold
|
||||
time: 2022-04-20T12:00:25.171923-05:00
|
||||
custom:
|
||||
Author: McKnight-42
|
||||
Issue: "4980"
|
||||
PR: "5117"
|
||||
@@ -1,7 +0,0 @@
|
||||
kind: Under the Hood
|
||||
body: Mypy -> 0.942 + fixed import logic to allow for full mypy coverage
|
||||
time: 2022-04-27T11:21:27.499359-05:00
|
||||
custom:
|
||||
Author: iknox-fa
|
||||
Issue: "4805"
|
||||
PR: "5171"
|
||||
@@ -1,7 +0,0 @@
|
||||
kind: Under the Hood
|
||||
body: Converted dbt list tests to pytest
|
||||
time: 2022-04-27T14:06:28.882908-05:00
|
||||
custom:
|
||||
Author: stu-k
|
||||
Issue: "5049"
|
||||
PR: "5178"
|
||||
@@ -1,7 +0,0 @@
|
||||
kind: Under the Hood
|
||||
body: 'Fix: Call str and repr for UnsetProfileConfig without a RuntimeException'
|
||||
time: 2022-05-03T19:52:12.793729384+02:00
|
||||
custom:
|
||||
Author: tomasfarias
|
||||
Issue: "5081"
|
||||
PR: "5209"
|
||||
@@ -1,7 +0,0 @@
|
||||
kind: Under the Hood
|
||||
body: Improve tracking error logging message
|
||||
time: 2022-05-04T01:00:31.60387036+02:00
|
||||
custom:
|
||||
Author: NicolasPA
|
||||
Issue: "5197"
|
||||
PR: "5211"
|
||||
@@ -1,8 +0,0 @@
|
||||
kind: Under the Hood
|
||||
body: 'Clean up materialization logic: more consistent relation names, loading from
|
||||
cache'
|
||||
time: 2022-05-09T09:26:28.551068+02:00
|
||||
custom:
|
||||
Author: jtcohen6
|
||||
Issue: "2869"
|
||||
PR: "4921"
|
||||
@@ -1,8 +0,0 @@
|
||||
kind: Under the Hood
|
||||
body: Use the default Python version for local dev and test instead of requiring Python
|
||||
3.8
|
||||
time: 2022-05-18T09:51:44.603193-07:00
|
||||
custom:
|
||||
Author: jwills
|
||||
Issue: "5257"
|
||||
PR: "5269"
|
||||
@@ -1,7 +0,0 @@
|
||||
kind: Under the Hood
|
||||
body: Fix test for context set function
|
||||
time: 2022-05-18T14:55:22.554316-04:00
|
||||
custom:
|
||||
Author: gshank
|
||||
Issue: "5266"
|
||||
PR: "5272"
|
||||
@@ -1,7 +0,0 @@
|
||||
kind: Under the Hood
|
||||
body: Fix pip upgrade step in CI for Windows
|
||||
time: 2022-06-01T10:52:45.872931-04:00
|
||||
custom:
|
||||
Author: gshank
|
||||
Issue: "5321"
|
||||
PR: "5320"
|
||||
@@ -1,7 +0,0 @@
|
||||
kind: Under the Hood
|
||||
body: Fix unit test test_graph_selection
|
||||
time: 2022-06-01T11:26:48.725831-04:00
|
||||
custom:
|
||||
Author: gshank
|
||||
Issue: "5323"
|
||||
PR: "5324"
|
||||
@@ -1,7 +0,0 @@
|
||||
kind: Under the Hood
|
||||
body: Update context readme + clean up context code"
|
||||
time: 2022-06-06T23:03:53.022568+02:00
|
||||
custom:
|
||||
Author: jtcohen6
|
||||
Issue: "4796"
|
||||
PR: "5334"
|
||||
@@ -1,7 +0,0 @@
|
||||
kind: Under the Hood
|
||||
body: removed script meant for snowflake to snowflake
|
||||
time: 2022-06-10T11:10:56.080236-05:00
|
||||
custom:
|
||||
Author: McKnight-42
|
||||
Issue: "5361"
|
||||
PR: "5362"
|
||||
9
.changes/1.2.1.md
Normal file
9
.changes/1.2.1.md
Normal file
@@ -0,0 +1,9 @@
|
||||
## dbt-core 1.2.1 - August 25, 2022
|
||||
### Fixes
|
||||
- Fix handling of top-level exceptions ([#5564](https://github.com/dbt-labs/dbt-core/issues/5564), [#5560](https://github.com/dbt-labs/dbt-core/pull/5560))
|
||||
- Fix error rendering docs block in metrics description ([#5585](https://github.com/dbt-labs/dbt-core/issues/5585), [#5603](https://github.com/dbt-labs/dbt-core/pull/5603))
|
||||
- Use sys.exit instead of exit ([#5621](https://github.com/dbt-labs/dbt-core/issues/5621), [#5627](https://github.com/dbt-labs/dbt-core/pull/5627))
|
||||
- Finishing logic upgrade to Redshift for name truncation collisions. ([#5586](https://github.com/dbt-labs/dbt-core/issues/5586), [#5656](https://github.com/dbt-labs/dbt-core/pull/5656))
|
||||
|
||||
### Contributors
|
||||
- [@varun-dc](https://github.com/varun-dc) ([#5627](https://github.com/dbt-labs/dbt-core/pull/5627))
|
||||
9
.changes/1.2.2.md
Normal file
9
.changes/1.2.2.md
Normal file
@@ -0,0 +1,9 @@
|
||||
## dbt-core 1.2.2 - October 03, 2022
|
||||
### Features
|
||||
- This conditionally no-ops warehouse connection at compile depending on an env var, disabling introspection/queries during compilation only. This is a temporary solution to more complex permissions requirements for the semantic layer. ([#5936](https://github.com/dbt-labs/dbt-core/issues/5936), [#5926](https://github.com/dbt-labs/dbt-core/pull/5926))
|
||||
### Fixes
|
||||
- Fix race condition when invoking dbt via lib.py concurrently ([#5919](https://github.com/dbt-labs/dbt-core/issues/5919), [#5921](https://github.com/dbt-labs/dbt-core/pull/5921))
|
||||
|
||||
### Contributors
|
||||
- [@drewbanin](https://github.com/drewbanin) ([#5921](https://github.com/dbt-labs/dbt-core/pull/5921))
|
||||
- [@racheldaniel](https://github.com/racheldaniel) ([#5926](https://github.com/dbt-labs/dbt-core/pull/5926))
|
||||
6
.changes/1.2.3.md
Normal file
6
.changes/1.2.3.md
Normal file
@@ -0,0 +1,6 @@
|
||||
## dbt-core 1.2.3 - November 16, 2022
|
||||
### Features
|
||||
- This pulls the profile name from args when constructing a RuntimeConfig in lib.py, enabling the dbt-server to override the value that's in the dbt_project.yml ([#6201](https://github.com/dbt-labs/dbt-core/issues/6201), [#6202](https://github.com/dbt-labs/dbt-core/pull/6202))
|
||||
|
||||
### Contributors
|
||||
- [@racheldaniel](https://github.com/racheldaniel) ([#6202](https://github.com/dbt-labs/dbt-core/pull/6202))
|
||||
3
.changes/1.2.4.md
Normal file
3
.changes/1.2.4.md
Normal file
@@ -0,0 +1,3 @@
|
||||
## dbt-core 1.2.4 - January 05, 2023
|
||||
### Fixes
|
||||
- Bug when partial parsing with an empty schema file ([#4850](https://github.com/dbt-labs/dbt-core/issues/4850), [#<no value>](https://github.com/dbt-labs/dbt-core/pull/<no value>))
|
||||
8
.changes/1.2.5.md
Normal file
8
.changes/1.2.5.md
Normal file
@@ -0,0 +1,8 @@
|
||||
## dbt-core 1.2.5 - February 28, 2023
|
||||
|
||||
### Fixes
|
||||
|
||||
- add pytz dependency ([#7077](https://github.com/dbt-labs/dbt-core/issues/7077))
|
||||
|
||||
### Contributors
|
||||
- [@sdebruyn](https://github.com/sdebruyn) ([#7077](https://github.com/dbt-labs/dbt-core/issues/7077))
|
||||
@@ -1,7 +0,0 @@
|
||||
kind: Dependencies
|
||||
body: "Bump mypy from 0.942 to 0.961"
|
||||
time: 2022-06-07T00:09:25.000000-05:00
|
||||
custom:
|
||||
Author: dependabot[bot]
|
||||
Issue: "4904"
|
||||
PR: "5337"
|
||||
@@ -1,7 +0,0 @@
|
||||
kind: Dependencies
|
||||
body: "Update colorama requirement from <0.4.5,>=0.3.9 to >=0.3.9,<0.4.6 in /core"
|
||||
time: 2022-06-17T00:18:49.000000-05:00
|
||||
custom:
|
||||
Author: dependabot[bot]
|
||||
Issue: "4904"
|
||||
PR: "5388"
|
||||
@@ -1,7 +0,0 @@
|
||||
kind: Dependencies
|
||||
body: "Bump black from 22.3.0 to 22.6.0"
|
||||
time: 2022-06-29T00:08:31.000000-05:00
|
||||
custom:
|
||||
Author: dependabot[bot]
|
||||
Issue: "4904"
|
||||
PR: "5420"
|
||||
@@ -1,7 +0,0 @@
|
||||
kind: Docs
|
||||
body: Fixed sample SQL Code for sources when no database is defined
|
||||
time: 2022-07-06T14:20:29.7347-05:00
|
||||
custom:
|
||||
Author: b-per
|
||||
Issue: "5255"
|
||||
PR: "5446"
|
||||
@@ -1,7 +0,0 @@
|
||||
kind: Docs
|
||||
body: Add support for `file:` selector in DAG viz
|
||||
time: 2022-07-06T14:21:42.544814-05:00
|
||||
custom:
|
||||
Author: drewbanin
|
||||
Issue: "5255"
|
||||
PR: "5446"
|
||||
@@ -1,7 +0,0 @@
|
||||
kind: Docs
|
||||
body: '[Snyk] Upgrade prismjs from 1.27.0 to 1.28.0'
|
||||
time: 2022-07-06T14:22:26.039137-05:00
|
||||
custom:
|
||||
Author: emmyoop
|
||||
Issue: "5255"
|
||||
PR: "5446"
|
||||
@@ -1,7 +0,0 @@
|
||||
kind: Docs
|
||||
body: Run build and tests in CI checks
|
||||
time: 2022-07-06T14:23:00.842859-05:00
|
||||
custom:
|
||||
Author: drewbanin
|
||||
Issue: "5255"
|
||||
PR: "5446"
|
||||
@@ -1,7 +0,0 @@
|
||||
kind: Docs
|
||||
body: Improve metrics DAG viz and documentation page
|
||||
time: 2022-07-06T14:24:10.120545-05:00
|
||||
custom:
|
||||
Author: drewbanin callum-mcdata
|
||||
Issue: "5255"
|
||||
PR: "5446"
|
||||
@@ -1,7 +0,0 @@
|
||||
kind: Docs
|
||||
body: Upgrade cytoscape.js fork
|
||||
time: 2022-07-06T14:25:05.575755-05:00
|
||||
custom:
|
||||
Author: emmyoop
|
||||
Issue: "5255"
|
||||
PR: "5446"
|
||||
@@ -1,8 +0,0 @@
|
||||
kind: Features
|
||||
body: 'Allow customizing `target-path` and `log-path` through
|
||||
environment variables and CLI flags.'
|
||||
time: 2022-06-22T19:23:29.748577478+03:00
|
||||
custom:
|
||||
Author: isidentical
|
||||
Issue: "5399"
|
||||
PR: "5402"
|
||||
@@ -1,7 +0,0 @@
|
||||
kind: Features
|
||||
body: Move type_* macros from dbt-utils into dbt-core, with tests
|
||||
time: 2022-06-30T13:54:52.165139+02:00
|
||||
custom:
|
||||
Author: jtcohen6
|
||||
Issue: "5317"
|
||||
PR: "5428"
|
||||
@@ -1,7 +0,0 @@
|
||||
kind: Features
|
||||
body: Add support for ratio metrics
|
||||
time: 2022-07-05T08:30:26.494837-05:00
|
||||
custom:
|
||||
Author: drewbanin callum-mcdata
|
||||
Issue: "4884"
|
||||
PR: "5027"
|
||||
@@ -1,8 +0,0 @@
|
||||
kind: Features
|
||||
body: Allow users to define grants as a reasonable default in the dbt_project.yml
|
||||
or within each model sql or yml file combined.
|
||||
time: 2022-07-11T11:15:14.695386-05:00
|
||||
custom:
|
||||
Author: McKnight-42
|
||||
Issue: "5263"
|
||||
PR: "5369"
|
||||
@@ -1,7 +0,0 @@
|
||||
kind: Fixes
|
||||
body: Add inheritance to materialization macro resolution
|
||||
time: 2022-06-08T10:42:51.839945-04:00
|
||||
custom:
|
||||
Author: volkangurel
|
||||
Issue: "4646"
|
||||
PR: "5348"
|
||||
@@ -1,7 +0,0 @@
|
||||
kind: Fixes
|
||||
body: Improve pluralizations for Documentation and SqlOperation NodeTypes
|
||||
time: 2022-06-09T11:52:46.578469-05:00
|
||||
custom:
|
||||
Author: pdebelak
|
||||
Issue: "5352"
|
||||
PR: "5356"
|
||||
@@ -1,7 +0,0 @@
|
||||
kind: Fixes
|
||||
body: Properly use quotes for Snowflake snapshots when checking all columns
|
||||
time: 2022-06-17T11:44:43.978834+02:00
|
||||
custom:
|
||||
Author: pquadri
|
||||
Issue: "2975"
|
||||
PR: "5389"
|
||||
@@ -1,7 +0,0 @@
|
||||
kind: Fixes
|
||||
body: fixes handling of RESET color code with USE_COLORS=False
|
||||
time: 2022-06-17T16:00:27.038058-04:00
|
||||
custom:
|
||||
Author: darin-reify
|
||||
Issue: "5288"
|
||||
PR: "5394"
|
||||
@@ -1,7 +0,0 @@
|
||||
kind: Security
|
||||
body: Move string interpolation of "secret" env vars outside of Jinja context. Update "contexts" README
|
||||
time: 2022-06-06T23:03:53.022568+02:00
|
||||
custom:
|
||||
Author: jtcohen6
|
||||
Issue: "4796"
|
||||
PR: "5334"
|
||||
@@ -1,8 +0,0 @@
|
||||
kind: Under the Hood
|
||||
body: Added the suggested RegEx to check the SemVer string within a package dependency
|
||||
and improved invalid version error handling.
|
||||
time: 2022-06-13T17:07:03.773668-05:00
|
||||
custom:
|
||||
Author: fivetran-joemarkiewicz
|
||||
Issue: "5201"
|
||||
PR: "5370"
|
||||
@@ -1,7 +0,0 @@
|
||||
kind: Under the Hood
|
||||
body: 'Add annotation to render_value method reimplemented in #5334'
|
||||
time: 2022-06-16T12:05:16.474078+02:00
|
||||
custom:
|
||||
Author: jtcohen6
|
||||
Issue: "4796"
|
||||
PR: "5382"
|
||||
@@ -1,7 +0,0 @@
|
||||
kind: Under the Hood
|
||||
body: Bump manifest version to v6
|
||||
time: 2022-07-05T14:21:20.66768-04:00
|
||||
custom:
|
||||
Author: leahwicz
|
||||
Issue: "5417"
|
||||
PR: "5430"
|
||||
@@ -1,7 +0,0 @@
|
||||
kind: Under the Hood
|
||||
body: Add tests for SQL grants
|
||||
time: 2022-07-06T21:50:01.498562-04:00
|
||||
custom:
|
||||
Author: gshank
|
||||
Issue: "5437"
|
||||
PR: "5447"
|
||||
130
.changie.yaml
130
.changie.yaml
@@ -6,56 +6,128 @@ changelogPath: CHANGELOG.md
|
||||
versionExt: md
|
||||
versionFormat: '## dbt-core {{.Version}} - {{.Time.Format "January 02, 2006"}}'
|
||||
kindFormat: '### {{.Kind}}'
|
||||
changeFormat: '- {{.Body}} ([#{{.Custom.Issue}}](https://github.com/dbt-labs/dbt-core/issues/{{.Custom.Issue}}), [#{{.Custom.PR}}](https://github.com/dbt-labs/dbt-core/pull/{{.Custom.PR}}))'
|
||||
changeFormat: |-
|
||||
{{- $IssueList := list }}
|
||||
{{- $changes := splitList " " $.Custom.Issue }}
|
||||
{{- range $issueNbr := $changes }}
|
||||
{{- $changeLink := "[#nbr](https://github.com/dbt-labs/dbt-core/issues/nbr)" | replace "nbr" $issueNbr }}
|
||||
{{- $IssueList = append $IssueList $changeLink }}
|
||||
{{- end -}}
|
||||
- {{.Body}} ({{ range $index, $element := $IssueList }}{{if $index}}, {{end}}{{$element}}{{end}})
|
||||
|
||||
kinds:
|
||||
- label: Breaking Changes
|
||||
- label: Features
|
||||
- label: Fixes
|
||||
- label: Docs
|
||||
- label: Under the Hood
|
||||
- label: Dependencies
|
||||
- label: Security
|
||||
- label: Breaking Changes
|
||||
- label: Features
|
||||
- label: Fixes
|
||||
- label: Docs
|
||||
changeFormat: |-
|
||||
{{- $IssueList := list }}
|
||||
{{- $changes := splitList " " $.Custom.Issue }}
|
||||
{{- range $issueNbr := $changes }}
|
||||
{{- $changeLink := "[dbt-docs/#nbr](https://github.com/dbt-labs/dbt-docs/issues/nbr)" | replace "nbr" $issueNbr }}
|
||||
{{- $IssueList = append $IssueList $changeLink }}
|
||||
{{- end -}}
|
||||
- {{.Body}} ({{ range $index, $element := $IssueList }}{{if $index}}, {{end}}{{$element}}{{end}})
|
||||
- label: Under the Hood
|
||||
- label: Dependencies
|
||||
changeFormat: |-
|
||||
{{- $PRList := list }}
|
||||
{{- $changes := splitList " " $.Custom.PR }}
|
||||
{{- range $pullrequest := $changes }}
|
||||
{{- $changeLink := "[#nbr](https://github.com/dbt-labs/dbt-core/pull/nbr)" | replace "nbr" $pullrequest }}
|
||||
{{- $PRList = append $PRList $changeLink }}
|
||||
{{- end -}}
|
||||
- {{.Body}} ({{ range $index, $element := $PRList }}{{if $index}}, {{end}}{{$element}}{{end}})
|
||||
skipGlobalChoices: true
|
||||
additionalChoices:
|
||||
- key: Author
|
||||
label: GitHub Username(s) (separated by a single space if multiple)
|
||||
type: string
|
||||
minLength: 3
|
||||
- key: PR
|
||||
label: GitHub Pull Request Number (separated by a single space if multiple)
|
||||
type: string
|
||||
minLength: 1
|
||||
- label: Security
|
||||
changeFormat: |-
|
||||
{{- $PRList := list }}
|
||||
{{- $changes := splitList " " $.Custom.PR }}
|
||||
{{- range $pullrequest := $changes }}
|
||||
{{- $changeLink := "[#nbr](https://github.com/dbt-labs/dbt-core/pull/nbr)" | replace "nbr" $pullrequest }}
|
||||
{{- $PRList = append $PRList $changeLink }}
|
||||
{{- end -}}
|
||||
- {{.Body}} ({{ range $index, $element := $PRList }}{{if $index}}, {{end}}{{$element}}{{end}})
|
||||
skipGlobalChoices: true
|
||||
additionalChoices:
|
||||
- key: Author
|
||||
label: GitHub Username(s) (separated by a single space if multiple)
|
||||
type: string
|
||||
minLength: 3
|
||||
- key: PR
|
||||
label: GitHub Pull Request Number (separated by a single space if multiple)
|
||||
type: string
|
||||
minLength: 1
|
||||
|
||||
newlines:
|
||||
afterChangelogHeader: 1
|
||||
afterKind: 1
|
||||
afterChangelogVersion: 1
|
||||
beforeKind: 1
|
||||
endOfVersion: 1
|
||||
|
||||
custom:
|
||||
- key: Author
|
||||
label: GitHub Username(s) (separated by a single space if multiple)
|
||||
type: string
|
||||
minLength: 3
|
||||
- key: Issue
|
||||
label: GitHub Issue Number
|
||||
type: int
|
||||
minLength: 4
|
||||
- key: PR
|
||||
label: GitHub Pull Request Number
|
||||
type: int
|
||||
minLength: 4
|
||||
label: GitHub Issue Number (separated by a single space if multiple)
|
||||
type: string
|
||||
minLength: 1
|
||||
|
||||
footerFormat: |
|
||||
{{- $contributorDict := dict }}
|
||||
{{- /* any names added to this list should be all lowercase for later matching purposes */}}
|
||||
{{- $core_team := list "emmyoop" "nathaniel-may" "gshank" "leahwicz" "chenyulinx" "stu-k" "iknox-fa" "versusfacit" "mcknight-42" "jtcohen6" "dependabot" }}
|
||||
{{- $core_team := list "michelleark" "peterallenwebb" "emmyoop" "nathaniel-may" "gshank" "leahwicz" "chenyulinx" "stu-k" "iknox-fa" "versusfacit" "mcknight-42" "jtcohen6" "aranke" "dependabot[bot]" "snyk-bot" "colin-rogers-dbt" }}
|
||||
{{- range $change := .Changes }}
|
||||
{{- $authorList := splitList " " $change.Custom.Author }}
|
||||
{{- /* loop through all authors for a PR */}}
|
||||
{{- /* loop through all authors for a single changelog */}}
|
||||
{{- range $author := $authorList }}
|
||||
{{- $authorLower := lower $author }}
|
||||
{{- /* we only want to include non-core team contributors */}}
|
||||
{{- if not (has $authorLower $core_team)}}
|
||||
{{- $pr := $change.Custom.PR }}
|
||||
{{- /* check if this contributor has other PRs associated with them already */}}
|
||||
{{- if hasKey $contributorDict $author }}
|
||||
{{- $prList := get $contributorDict $author }}
|
||||
{{- $prList = append $prList $pr }}
|
||||
{{- $contributorDict := set $contributorDict $author $prList }}
|
||||
{{- else }}
|
||||
{{- $prList := list $change.Custom.PR }}
|
||||
{{- $contributorDict := set $contributorDict $author $prList }}
|
||||
{{- end }}
|
||||
{{- end}}
|
||||
{{- $changeList := splitList " " $change.Custom.Author }}
|
||||
{{- $IssueList := list }}
|
||||
{{- $changeLink := $change.Kind }}
|
||||
{{- if or (eq $change.Kind "Dependencies") (eq $change.Kind "Security") }}
|
||||
{{- $changes := splitList " " $change.Custom.PR }}
|
||||
{{- range $issueNbr := $changes }}
|
||||
{{- $changeLink := "[#nbr](https://github.com/dbt-labs/dbt-core/pull/nbr)" | replace "nbr" $issueNbr }}
|
||||
{{- $IssueList = append $IssueList $changeLink }}
|
||||
{{- end -}}
|
||||
{{- else }}
|
||||
{{- $changes := splitList " " $change.Custom.Issue }}
|
||||
{{- range $issueNbr := $changes }}
|
||||
{{- $changeLink := "[#nbr](https://github.com/dbt-labs/dbt-core/issues/nbr)" | replace "nbr" $issueNbr }}
|
||||
{{- $IssueList = append $IssueList $changeLink }}
|
||||
{{- end -}}
|
||||
{{- end }}
|
||||
{{- /* check if this contributor has other changes associated with them already */}}
|
||||
{{- if hasKey $contributorDict $author }}
|
||||
{{- $contributionList := get $contributorDict $author }}
|
||||
{{- $contributionList = concat $contributionList $IssueList }}
|
||||
{{- $contributorDict := set $contributorDict $author $contributionList }}
|
||||
{{- else }}
|
||||
{{- $contributionList := $IssueList }}
|
||||
{{- $contributorDict := set $contributorDict $author $contributionList }}
|
||||
{{- end }}
|
||||
{{- end}}
|
||||
{{- end}}
|
||||
{{- end }}
|
||||
{{- /* no indentation here for formatting so the final markdown doesn't have unneeded indentations */}}
|
||||
{{- if $contributorDict}}
|
||||
### Contributors
|
||||
{{- range $k,$v := $contributorDict }}
|
||||
- [@{{$k}}](https://github.com/{{$k}}) ({{ range $index, $element := $v }}{{if $index}}, {{end}}[#{{$element}}](https://github.com/dbt-labs/dbt-core/pull/{{$element}}){{end}})
|
||||
- [@{{$k}}](https://github.com/{{$k}}) ({{ range $index, $element := $v }}{{if $index}}, {{end}}{{$element}}{{end}})
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
61
.github/workflows/bot-changelog.yml
vendored
Normal file
61
.github/workflows/bot-changelog.yml
vendored
Normal file
@@ -0,0 +1,61 @@
|
||||
# **what?**
|
||||
# When bots create a PR, this action will add a corresponding changie yaml file to that
|
||||
# PR when a specific label is added.
|
||||
#
|
||||
# The file is created off a template:
|
||||
#
|
||||
# kind: <per action matrix>
|
||||
# body: <PR title>
|
||||
# time: <current timestamp>
|
||||
# custom:
|
||||
# Author: <PR User Login (generally the bot)>
|
||||
# Issue: 4904
|
||||
# PR: <PR number>
|
||||
#
|
||||
# **why?**
|
||||
# Automate changelog generation for more visability with automated bot PRs.
|
||||
#
|
||||
# **when?**
|
||||
# Once a PR is created, label should be added to PR before or after creation. You can also
|
||||
# manually trigger this by adding the appropriate label at any time.
|
||||
#
|
||||
# **how to add another bot?**
|
||||
# Add the label and changie kind to the include matrix. That's it!
|
||||
#
|
||||
|
||||
name: Bot Changelog
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
# catch when the PR is opened with the label or when the label is added
|
||||
types: [labeled]
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
pull-requests: read
|
||||
|
||||
jobs:
|
||||
generate_changelog:
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- label: "dependencies"
|
||||
changie_kind: "Dependencies"
|
||||
- label: "snyk"
|
||||
changie_kind: "Security"
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
|
||||
- name: Create and commit changelog on bot PR
|
||||
if: ${{ contains(github.event.pull_request.labels.*.name, matrix.label) }}
|
||||
id: bot_changelog
|
||||
uses: emmyoop/changie_bot@v1.0.1
|
||||
with:
|
||||
GITHUB_TOKEN: ${{ secrets.FISHTOWN_BOT_PAT }}
|
||||
commit_author_name: "Github Build Bot"
|
||||
commit_author_email: "<buildbot@fishtownanalytics.com>"
|
||||
commit_message: "Add automated changelog yaml from template for bot PR"
|
||||
changie_kind: ${{ matrix.changie_kind }}
|
||||
label: ${{ matrix.label }}
|
||||
custom_changelog_string: "custom:\n Author: ${{ github.event.pull_request.user.login }}\n PR: ${{ github.event.pull_request.number }}"
|
||||
20
.github/workflows/main.yml
vendored
20
.github/workflows/main.yml
vendored
@@ -37,14 +37,16 @@ jobs:
|
||||
code-quality:
|
||||
name: code-quality
|
||||
|
||||
runs-on: ubuntu-latest
|
||||
runs-on: ubuntu-20.04
|
||||
|
||||
steps:
|
||||
- name: Check out the repository
|
||||
uses: actions/checkout@v2
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v2
|
||||
uses: actions/setup-python@v4.3.0
|
||||
with:
|
||||
python-version: '3.8'
|
||||
|
||||
- name: Install python dependencies
|
||||
run: |
|
||||
@@ -64,7 +66,7 @@ jobs:
|
||||
unit:
|
||||
name: unit test / python ${{ matrix.python-version }}
|
||||
|
||||
runs-on: ubuntu-latest
|
||||
runs-on: ubuntu-20.04
|
||||
|
||||
strategy:
|
||||
fail-fast: false
|
||||
@@ -80,7 +82,7 @@ jobs:
|
||||
uses: actions/checkout@v2
|
||||
|
||||
- name: Set up Python ${{ matrix.python-version }}
|
||||
uses: actions/setup-python@v2
|
||||
uses: actions/setup-python@v4.3.0
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
|
||||
@@ -114,7 +116,7 @@ jobs:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
python-version: ["3.7", "3.8", "3.9", "3.10"]
|
||||
os: [ubuntu-latest]
|
||||
os: [ubuntu-20.04]
|
||||
include:
|
||||
- python-version: 3.8
|
||||
os: windows-latest
|
||||
@@ -134,7 +136,7 @@ jobs:
|
||||
uses: actions/checkout@v2
|
||||
|
||||
- name: Set up Python ${{ matrix.python-version }}
|
||||
uses: actions/setup-python@v2
|
||||
uses: actions/setup-python@v4.3.0
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
|
||||
@@ -180,16 +182,16 @@ jobs:
|
||||
build:
|
||||
name: build packages
|
||||
|
||||
runs-on: ubuntu-latest
|
||||
runs-on: ubuntu-20.04
|
||||
|
||||
steps:
|
||||
- name: Check out the repository
|
||||
uses: actions/checkout@v2
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v2
|
||||
uses: actions/setup-python@v4.3.0
|
||||
with:
|
||||
python-version: 3.8
|
||||
python-version: '3.8'
|
||||
|
||||
- name: Install python dependencies
|
||||
run: |
|
||||
|
||||
109
.github/workflows/nightly-release.yml
vendored
Normal file
109
.github/workflows/nightly-release.yml
vendored
Normal file
@@ -0,0 +1,109 @@
|
||||
# **what?**
|
||||
# Nightly releases to GitHub and PyPI. This workflow produces the following outcome:
|
||||
# - generate and validate data for night release (commit SHA, version number, release branch);
|
||||
# - pass data to release workflow;
|
||||
# - night release will be pushed to GitHub as a draft release;
|
||||
# - night build will be pushed to test PyPI;
|
||||
#
|
||||
# **why?**
|
||||
# Ensure an automated and tested release process for nightly builds
|
||||
#
|
||||
# **when?**
|
||||
# This workflow runs on schedule or can be run manually on demand.
|
||||
|
||||
name: Nightly Test Release to GitHub and PyPI
|
||||
|
||||
on:
|
||||
workflow_dispatch: # for manual triggering
|
||||
schedule:
|
||||
- cron: 0 9 * * *
|
||||
|
||||
permissions:
|
||||
contents: write # this is the permission that allows creating a new release
|
||||
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
|
||||
env:
|
||||
RELEASE_BRANCH: "main"
|
||||
|
||||
jobs:
|
||||
aggregate-release-data:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
outputs:
|
||||
commit_sha: ${{ steps.resolve-commit-sha.outputs.release_commit }}
|
||||
version_number: ${{ steps.nightly-release-version.outputs.number }}
|
||||
release_branch: ${{ steps.release-branch.outputs.name }}
|
||||
|
||||
steps:
|
||||
- name: "Checkout ${{ github.repository }} Branch ${{ env.RELEASE_BRANCH }}"
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
ref: ${{ env.RELEASE_BRANCH }}
|
||||
|
||||
- name: "Resolve Commit To Release"
|
||||
id: resolve-commit-sha
|
||||
run: |
|
||||
commit_sha=$(git rev-parse HEAD)
|
||||
echo "release_commit=$commit_sha" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: "Get Current Version Number"
|
||||
id: version-number-sources
|
||||
run: |
|
||||
current_version=`awk -F"current_version = " '{print $2}' .bumpversion.cfg | tr '\n' ' '`
|
||||
echo "current_version=$current_version" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: "Audit Version And Parse Into Parts"
|
||||
id: semver
|
||||
uses: dbt-labs/actions/parse-semver@v1.1.0
|
||||
with:
|
||||
version: ${{ steps.version-number-sources.outputs.current_version }}
|
||||
|
||||
- name: "Get Current Date"
|
||||
id: current-date
|
||||
run: echo "date=$(date +'%m%d%Y')" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: "Generate Nightly Release Version Number"
|
||||
id: nightly-release-version
|
||||
run: |
|
||||
number="${{ steps.semver.outputs.version }}.dev${{ steps.current-date.outputs.date }}+nightly"
|
||||
echo "number=$number" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: "Audit Nightly Release Version And Parse Into Parts"
|
||||
uses: dbt-labs/actions/parse-semver@v1.1.0
|
||||
with:
|
||||
version: ${{ steps.nightly-release-version.outputs.number }}
|
||||
|
||||
- name: "Set Release Branch"
|
||||
id: release-branch
|
||||
run: |
|
||||
echo "name=${{ env.RELEASE_BRANCH }}" >> $GITHUB_OUTPUT
|
||||
|
||||
log-outputs-aggregate-release-data:
|
||||
runs-on: ubuntu-latest
|
||||
needs: [aggregate-release-data]
|
||||
|
||||
steps:
|
||||
- name: "[DEBUG] Log Outputs"
|
||||
run: |
|
||||
echo commit_sha : ${{ needs.aggregate-release-data.outputs.commit_sha }}
|
||||
echo version_number: ${{ needs.aggregate-release-data.outputs.version_number }}
|
||||
echo release_branch: ${{ needs.aggregate-release-data.outputs.release_branch }}
|
||||
|
||||
release-github-pypi:
|
||||
needs: [aggregate-release-data]
|
||||
|
||||
uses: ./.github/workflows/release.yml
|
||||
with:
|
||||
sha: ${{ needs.aggregate-release-data.outputs.commit_sha }}
|
||||
target_branch: ${{ needs.aggregate-release-data.outputs.release-branch }}
|
||||
version_number: ${{ needs.aggregate-release-data.outputs.version_number }}
|
||||
build_script_path: "scripts/build-dist.sh"
|
||||
env_setup_script_path: "scripts/env-setup.sh"
|
||||
s3_bucket_name: "core-team-artifacts"
|
||||
package_test_command: "dbt --version"
|
||||
test_run: true
|
||||
nightly_release: true
|
||||
secrets: inherit
|
||||
340
.github/workflows/release.yml
vendored
340
.github/workflows/release.yml
vendored
@@ -1,199 +1,227 @@
|
||||
# **what?**
|
||||
# Take the given commit, run unit tests specifically on that sha, build and
|
||||
# package it, and then release to GitHub and PyPi with that specific build
|
||||
|
||||
# Release workflow provides the following steps:
|
||||
# - checkout the given commit;
|
||||
# - validate version in sources and changelog file for given version;
|
||||
# - bump the version and generate a changelog if needed;
|
||||
# - merge all changes to the target branch if needed;
|
||||
# - run unit and integration tests against given commit;
|
||||
# - build and package that SHA;
|
||||
# - release it to GitHub and PyPI with that specific build;
|
||||
#
|
||||
# **why?**
|
||||
# Ensure an automated and tested release process
|
||||
|
||||
#
|
||||
# **when?**
|
||||
# This will only run manually with a given sha and version
|
||||
# This workflow can be run manually on demand or can be called by other workflows
|
||||
|
||||
name: Release to GitHub and PyPi
|
||||
name: Release to GitHub and PyPI
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
sha:
|
||||
description: 'The last commit sha in the release'
|
||||
required: true
|
||||
description: "The last commit sha in the release"
|
||||
type: string
|
||||
required: true
|
||||
target_branch:
|
||||
description: "The branch to release from"
|
||||
type: string
|
||||
required: true
|
||||
version_number:
|
||||
description: 'The release version number (i.e. 1.0.0b1)'
|
||||
required: true
|
||||
description: "The release version number (i.e. 1.0.0b1)"
|
||||
type: string
|
||||
required: true
|
||||
build_script_path:
|
||||
description: "Build script path"
|
||||
type: string
|
||||
default: "scripts/build-dist.sh"
|
||||
required: true
|
||||
env_setup_script_path:
|
||||
description: "Environment setup script path"
|
||||
type: string
|
||||
default: "scripts/env-setup.sh"
|
||||
required: false
|
||||
s3_bucket_name:
|
||||
description: "AWS S3 bucket name"
|
||||
type: string
|
||||
default: "core-team-artifacts"
|
||||
required: true
|
||||
package_test_command:
|
||||
description: "Package test command"
|
||||
type: string
|
||||
default: "dbt --version"
|
||||
required: true
|
||||
test_run:
|
||||
description: "Test run (Publish release as draft)"
|
||||
type: boolean
|
||||
default: true
|
||||
required: false
|
||||
nightly_release:
|
||||
description: "Nightly release to dev environment"
|
||||
type: boolean
|
||||
default: false
|
||||
required: false
|
||||
workflow_call:
|
||||
inputs:
|
||||
sha:
|
||||
description: "The last commit sha in the release"
|
||||
type: string
|
||||
required: true
|
||||
target_branch:
|
||||
description: "The branch to release from"
|
||||
type: string
|
||||
required: true
|
||||
version_number:
|
||||
description: "The release version number (i.e. 1.0.0b1)"
|
||||
type: string
|
||||
required: true
|
||||
build_script_path:
|
||||
description: "Build script path"
|
||||
type: string
|
||||
default: "scripts/build-dist.sh"
|
||||
required: true
|
||||
env_setup_script_path:
|
||||
description: "Environment setup script path"
|
||||
type: string
|
||||
default: "scripts/env-setup.sh"
|
||||
required: false
|
||||
s3_bucket_name:
|
||||
description: "AWS S3 bucket name"
|
||||
type: string
|
||||
default: "core-team-artifacts"
|
||||
required: true
|
||||
package_test_command:
|
||||
description: "Package test command"
|
||||
type: string
|
||||
default: "dbt --version"
|
||||
required: true
|
||||
test_run:
|
||||
description: "Test run (Publish release as draft)"
|
||||
type: boolean
|
||||
default: true
|
||||
required: false
|
||||
nightly_release:
|
||||
description: "Nightly release to dev environment"
|
||||
type: boolean
|
||||
default: false
|
||||
required: false
|
||||
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
|
||||
jobs:
|
||||
unit:
|
||||
name: Unit test
|
||||
|
||||
log-inputs:
|
||||
name: Log Inputs
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
env:
|
||||
TOXENV: "unit"
|
||||
|
||||
steps:
|
||||
- name: Check out the repository
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
persist-credentials: false
|
||||
ref: ${{ github.event.inputs.sha }}
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v2
|
||||
with:
|
||||
python-version: 3.8
|
||||
|
||||
- name: Install python dependencies
|
||||
- name: "[DEBUG] Print Variables"
|
||||
run: |
|
||||
pip install --user --upgrade pip
|
||||
pip install tox
|
||||
pip --version
|
||||
tox --version
|
||||
echo The last commit sha in the release: ${{ inputs.sha }}
|
||||
echo The branch to release from: ${{ inputs.target_branch }}
|
||||
echo The release version number: ${{ inputs.version_number }}
|
||||
echo Build script path: ${{ inputs.build_script_path }}
|
||||
echo Environment setup script path: ${{ inputs.env_setup_script_path }}
|
||||
echo AWS S3 bucket name: ${{ inputs.s3_bucket_name }}
|
||||
echo Package test command: ${{ inputs.package_test_command }}
|
||||
echo Test run: ${{ inputs.test_run }}
|
||||
echo Nightly release: ${{ inputs.nightly_release }}
|
||||
|
||||
- name: Run tox
|
||||
run: tox
|
||||
bump-version-generate-changelog:
|
||||
name: Bump package version, Generate changelog
|
||||
|
||||
build:
|
||||
name: build packages
|
||||
uses: dbt-labs/dbt-release/.github/workflows/release-prep.yml@main
|
||||
|
||||
with:
|
||||
sha: ${{ inputs.sha }}
|
||||
version_number: ${{ inputs.version_number }}
|
||||
target_branch: ${{ inputs.target_branch }}
|
||||
env_setup_script_path: ${{ inputs.env_setup_script_path }}
|
||||
test_run: ${{ inputs.test_run }}
|
||||
nightly_release: ${{ inputs.nightly_release }}
|
||||
|
||||
secrets:
|
||||
FISHTOWN_BOT_PAT: ${{ secrets.FISHTOWN_BOT_PAT }}
|
||||
|
||||
log-outputs-bump-version-generate-changelog:
|
||||
name: "[Log output] Bump package version, Generate changelog"
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
|
||||
needs: [bump-version-generate-changelog]
|
||||
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Check out the repository
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
persist-credentials: false
|
||||
ref: ${{ github.event.inputs.sha }}
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v2
|
||||
with:
|
||||
python-version: 3.8
|
||||
|
||||
- name: Install python dependencies
|
||||
- name: Print variables
|
||||
run: |
|
||||
pip install --user --upgrade pip
|
||||
pip install --upgrade setuptools wheel twine check-wheel-contents
|
||||
pip --version
|
||||
echo Final SHA : ${{ needs.bump-version-generate-changelog.outputs.final_sha }}
|
||||
echo Changelog path: ${{ needs.bump-version-generate-changelog.outputs.changelog_path }}
|
||||
|
||||
- name: Build distributions
|
||||
run: ./scripts/build-dist.sh
|
||||
build-test-package:
|
||||
name: Build, Test, Package
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
needs: [bump-version-generate-changelog]
|
||||
|
||||
- name: Show distributions
|
||||
run: ls -lh dist/
|
||||
uses: dbt-labs/dbt-release/.github/workflows/build.yml@main
|
||||
|
||||
- name: Check distribution descriptions
|
||||
run: |
|
||||
twine check dist/*
|
||||
with:
|
||||
sha: ${{ needs.bump-version-generate-changelog.outputs.final_sha }}
|
||||
version_number: ${{ inputs.version_number }}
|
||||
changelog_path: ${{ needs.bump-version-generate-changelog.outputs.changelog_path }}
|
||||
build_script_path: ${{ inputs.build_script_path }}
|
||||
s3_bucket_name: ${{ inputs.s3_bucket_name }}
|
||||
package_test_command: ${{ inputs.package_test_command }}
|
||||
test_run: ${{ inputs.test_run }}
|
||||
nightly_release: ${{ inputs.nightly_release }}
|
||||
|
||||
- name: Check wheel contents
|
||||
run: |
|
||||
check-wheel-contents dist/*.whl --ignore W007,W008
|
||||
|
||||
- uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: dist
|
||||
path: |
|
||||
dist/
|
||||
!dist/dbt-${{github.event.inputs.version_number}}.tar.gz
|
||||
|
||||
test-build:
|
||||
name: verify packages
|
||||
|
||||
needs: [build, unit]
|
||||
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v2
|
||||
with:
|
||||
python-version: 3.8
|
||||
|
||||
- name: Install python dependencies
|
||||
run: |
|
||||
pip install --user --upgrade pip
|
||||
pip install --upgrade wheel
|
||||
pip --version
|
||||
|
||||
- uses: actions/download-artifact@v2
|
||||
with:
|
||||
name: dist
|
||||
path: dist/
|
||||
|
||||
- name: Show distributions
|
||||
run: ls -lh dist/
|
||||
|
||||
- name: Install wheel distributions
|
||||
run: |
|
||||
find ./dist/*.whl -maxdepth 1 -type f | xargs pip install --force-reinstall --find-links=dist/
|
||||
|
||||
- name: Check wheel distributions
|
||||
run: |
|
||||
dbt --version
|
||||
|
||||
- name: Install source distributions
|
||||
run: |
|
||||
find ./dist/*.gz -maxdepth 1 -type f | xargs pip install --force-reinstall --find-links=dist/
|
||||
|
||||
- name: Check source distributions
|
||||
run: |
|
||||
dbt --version
|
||||
secrets:
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
||||
|
||||
github-release:
|
||||
name: GitHub Release
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
|
||||
needs: test-build
|
||||
needs: [bump-version-generate-changelog, build-test-package]
|
||||
|
||||
runs-on: ubuntu-latest
|
||||
uses: dbt-labs/dbt-release/.github/workflows/github-release.yml@main
|
||||
|
||||
steps:
|
||||
- uses: actions/download-artifact@v2
|
||||
with:
|
||||
name: dist
|
||||
path: '.'
|
||||
|
||||
# Need to set an output variable because env variables can't be taken as input
|
||||
# This is needed for the next step with releasing to GitHub
|
||||
- name: Find release type
|
||||
id: release_type
|
||||
env:
|
||||
IS_PRERELEASE: ${{ contains(github.event.inputs.version_number, 'rc') || contains(github.event.inputs.version_number, 'b') }}
|
||||
run: |
|
||||
echo ::set-output name=isPrerelease::$IS_PRERELEASE
|
||||
|
||||
- name: Creating GitHub Release
|
||||
uses: softprops/action-gh-release@v1
|
||||
with:
|
||||
name: dbt-core v${{github.event.inputs.version_number}}
|
||||
tag_name: v${{github.event.inputs.version_number}}
|
||||
prerelease: ${{ steps.release_type.outputs.isPrerelease }}
|
||||
target_commitish: ${{github.event.inputs.sha}}
|
||||
body: |
|
||||
[Release notes](https://github.com/dbt-labs/dbt-core/blob/main/CHANGELOG.md)
|
||||
files: |
|
||||
dbt_postgres-${{github.event.inputs.version_number}}-py3-none-any.whl
|
||||
dbt_core-${{github.event.inputs.version_number}}-py3-none-any.whl
|
||||
dbt-postgres-${{github.event.inputs.version_number}}.tar.gz
|
||||
dbt-core-${{github.event.inputs.version_number}}.tar.gz
|
||||
with:
|
||||
sha: ${{ needs.bump-version-generate-changelog.outputs.final_sha }}
|
||||
version_number: ${{ inputs.version_number }}
|
||||
changelog_path: ${{ needs.bump-version-generate-changelog.outputs.changelog_path }}
|
||||
test_run: ${{ inputs.test_run }}
|
||||
|
||||
pypi-release:
|
||||
name: Pypi release
|
||||
name: PyPI Release
|
||||
|
||||
runs-on: ubuntu-latest
|
||||
needs: [github-release]
|
||||
|
||||
needs: github-release
|
||||
uses: dbt-labs/dbt-release/.github/workflows/pypi-release.yml@main
|
||||
|
||||
environment: PypiProd
|
||||
steps:
|
||||
- uses: actions/download-artifact@v2
|
||||
with:
|
||||
name: dist
|
||||
path: 'dist'
|
||||
with:
|
||||
version_number: ${{ inputs.version_number }}
|
||||
test_run: ${{ inputs.test_run }}
|
||||
|
||||
- name: Publish distribution to PyPI
|
||||
uses: pypa/gh-action-pypi-publish@v1.4.2
|
||||
with:
|
||||
password: ${{ secrets.PYPI_API_TOKEN }}
|
||||
secrets:
|
||||
PYPI_API_TOKEN: ${{ secrets.PYPI_API_TOKEN }}
|
||||
TEST_PYPI_API_TOKEN: ${{ secrets.TEST_PYPI_API_TOKEN }}
|
||||
|
||||
slack-notification:
|
||||
name: Slack Notification
|
||||
if: ${{ failure() && (!inputs.test_run || inputs.nightly_release) }}
|
||||
|
||||
needs:
|
||||
[
|
||||
bump-version-generate-changelog,
|
||||
build-test-package,
|
||||
github-release,
|
||||
pypi-release,
|
||||
]
|
||||
|
||||
uses: dbt-labs/dbt-release/.github/workflows/slack-post-notification.yml@main
|
||||
with:
|
||||
status: "failure"
|
||||
|
||||
secrets:
|
||||
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_DEV_CORE_ALERTS }}
|
||||
|
||||
@@ -22,7 +22,7 @@ jobs:
|
||||
# run the performance measurements on the current or default branch
|
||||
test-schema:
|
||||
name: Test Log Schema
|
||||
runs-on: ubuntu-latest
|
||||
runs-on: ubuntu-20.04
|
||||
env:
|
||||
# turns warnings into errors
|
||||
RUSTFLAGS: "-D warnings"
|
||||
|
||||
120
.github/workflows/version-bump.yml
vendored
120
.github/workflows/version-bump.yml
vendored
@@ -1,18 +1,15 @@
|
||||
# **what?**
|
||||
# This workflow will take a version number and a dry run flag. With that
|
||||
# This workflow will take the new version number to bump to. With that
|
||||
# it will run versionbump to update the version number everywhere in the
|
||||
# code base and then generate an update Docker requirements file. If this
|
||||
# is a dry run, a draft PR will open with the changes. If this isn't a dry
|
||||
# run, the changes will be committed to the branch this is run on.
|
||||
# code base and then run changie to create the corresponding changelog.
|
||||
# A PR will be created with the changes that can be reviewed before committing.
|
||||
|
||||
# **why?**
|
||||
# This is to aid in releasing dbt and making sure we have updated
|
||||
# the versions and Docker requirements in all places.
|
||||
# the version in all places and generated the changelog.
|
||||
|
||||
# **when?**
|
||||
# This is triggered either manually OR
|
||||
# from the repository_dispatch event "version-bump" which is sent from
|
||||
# the dbt-release repo Action
|
||||
# This is triggered manually
|
||||
|
||||
name: Version Bump
|
||||
|
||||
@@ -20,35 +17,21 @@ on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
version_number:
|
||||
description: 'The version number to bump to'
|
||||
description: 'The version number to bump to (ex. 1.2.0, 1.3.0b1)'
|
||||
required: true
|
||||
is_dry_run:
|
||||
description: 'Creates a draft PR to allow testing instead of committing to a branch'
|
||||
required: true
|
||||
default: 'true'
|
||||
repository_dispatch:
|
||||
types: [version-bump]
|
||||
|
||||
jobs:
|
||||
bump:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: "[DEBUG] Print Variables"
|
||||
run: |
|
||||
echo "all variables defined as inputs"
|
||||
echo The version_number: ${{ github.event.inputs.version_number }}
|
||||
|
||||
- name: Check out the repository
|
||||
uses: actions/checkout@v2
|
||||
|
||||
- name: Set version and dry run values
|
||||
id: variables
|
||||
env:
|
||||
VERSION_NUMBER: "${{ github.event.client_payload.version_number == '' && github.event.inputs.version_number || github.event.client_payload.version_number }}"
|
||||
IS_DRY_RUN: "${{ github.event.client_payload.is_dry_run == '' && github.event.inputs.is_dry_run || github.event.client_payload.is_dry_run }}"
|
||||
run: |
|
||||
echo Repository dispatch event version: ${{ github.event.client_payload.version_number }}
|
||||
echo Repository dispatch event dry run: ${{ github.event.client_payload.is_dry_run }}
|
||||
echo Workflow dispatch event version: ${{ github.event.inputs.version_number }}
|
||||
echo Workflow dispatch event dry run: ${{ github.event.inputs.is_dry_run }}
|
||||
echo ::set-output name=VERSION_NUMBER::$VERSION_NUMBER
|
||||
echo ::set-output name=IS_DRY_RUN::$IS_DRY_RUN
|
||||
|
||||
- uses: actions/setup-python@v2
|
||||
with:
|
||||
python-version: "3.8"
|
||||
@@ -59,53 +42,80 @@ jobs:
|
||||
source env/bin/activate
|
||||
pip install --upgrade pip
|
||||
|
||||
- name: Create PR branch
|
||||
if: ${{ steps.variables.outputs.IS_DRY_RUN == 'true' }}
|
||||
- name: Add Homebrew to PATH
|
||||
run: |
|
||||
git checkout -b bumping-version/${{steps.variables.outputs.VERSION_NUMBER}}_$GITHUB_RUN_ID
|
||||
git push origin bumping-version/${{steps.variables.outputs.VERSION_NUMBER}}_$GITHUB_RUN_ID
|
||||
git branch --set-upstream-to=origin/bumping-version/${{steps.variables.outputs.VERSION_NUMBER}}_$GITHUB_RUN_ID bumping-version/${{steps.variables.outputs.VERSION_NUMBER}}_$GITHUB_RUN_ID
|
||||
echo "/home/linuxbrew/.linuxbrew/bin:/home/linuxbrew/.linuxbrew/sbin" >> $GITHUB_PATH
|
||||
|
||||
# - name: Generate Docker requirements
|
||||
# run: |
|
||||
# source env/bin/activate
|
||||
# pip install -r requirements.txt
|
||||
# pip freeze -l > docker/requirements/requirements.txt
|
||||
# git status
|
||||
- name: Install Homebrew packages
|
||||
run: |
|
||||
brew install pre-commit
|
||||
brew tap miniscruff/changie https://github.com/miniscruff/changie
|
||||
brew install changie
|
||||
|
||||
- name: Audit Version and Parse Into Parts
|
||||
id: semver
|
||||
uses: dbt-labs/actions/parse-semver@v1
|
||||
with:
|
||||
version: ${{ github.event.inputs.version_number }}
|
||||
|
||||
- name: Set branch value
|
||||
id: variables
|
||||
run: |
|
||||
echo "::set-output name=BRANCH_NAME::prep-release/${{ github.event.inputs.version_number }}_$GITHUB_RUN_ID"
|
||||
|
||||
- name: Create PR branch
|
||||
run: |
|
||||
git checkout -b ${{ steps.variables.outputs.BRANCH_NAME }}
|
||||
git push origin ${{ steps.variables.outputs.BRANCH_NAME }}
|
||||
git branch --set-upstream-to=origin/${{ steps.variables.outputs.BRANCH_NAME }} ${{ steps.variables.outputs.BRANCH_NAME }}
|
||||
|
||||
- name: Bump version
|
||||
run: |
|
||||
source env/bin/activate
|
||||
pip install -r dev-requirements.txt
|
||||
env/bin/bumpversion --allow-dirty --new-version ${{steps.variables.outputs.VERSION_NUMBER}} major
|
||||
env/bin/bumpversion --allow-dirty --new-version ${{ github.event.inputs.version_number }} major
|
||||
git status
|
||||
|
||||
- name: Commit version bump directly
|
||||
uses: EndBug/add-and-commit@v7
|
||||
if: ${{ steps.variables.outputs.IS_DRY_RUN == 'false' }}
|
||||
with:
|
||||
author_name: 'Github Build Bot'
|
||||
author_email: 'buildbot@fishtownanalytics.com'
|
||||
message: 'Bumping version to ${{steps.variables.outputs.VERSION_NUMBER}}'
|
||||
- name: Run changie
|
||||
run: |
|
||||
if [[ ${{ steps.semver.outputs.is-pre-release }} -eq 1 ]]
|
||||
then
|
||||
changie batch ${{ steps.semver.outputs.base-version }} --move-dir '${{ steps.semver.outputs.base-version }}' --prerelease '${{ steps.semver.outputs.pre-release }}'
|
||||
else
|
||||
changie batch ${{ steps.semver.outputs.base-version }} --include '${{ steps.semver.outputs.base-version }}' --remove-prereleases
|
||||
fi
|
||||
changie merge
|
||||
git status
|
||||
|
||||
# this step will fail on whitespace errors but also correct them
|
||||
- name: Remove trailing whitespace
|
||||
continue-on-error: true
|
||||
run: |
|
||||
pre-commit run trailing-whitespace --files .bumpversion.cfg CHANGELOG.md .changes/*
|
||||
git status
|
||||
|
||||
# this step will fail on newline errors but also correct them
|
||||
- name: Removing extra newlines
|
||||
continue-on-error: true
|
||||
run: |
|
||||
pre-commit run end-of-file-fixer --files .bumpversion.cfg CHANGELOG.md .changes/*
|
||||
git status
|
||||
|
||||
- name: Commit version bump to branch
|
||||
uses: EndBug/add-and-commit@v7
|
||||
if: ${{ steps.variables.outputs.IS_DRY_RUN == 'true' }}
|
||||
with:
|
||||
author_name: 'Github Build Bot'
|
||||
author_email: 'buildbot@fishtownanalytics.com'
|
||||
message: 'Bumping version to ${{steps.variables.outputs.VERSION_NUMBER}}'
|
||||
branch: 'bumping-version/${{steps.variables.outputs.VERSION_NUMBER}}_${{GITHUB.RUN_ID}}'
|
||||
push: 'origin origin/bumping-version/${{steps.variables.outputs.VERSION_NUMBER}}_${{GITHUB.RUN_ID}}'
|
||||
message: 'Bumping version to ${{ github.event.inputs.version_number }} and generate CHANGELOG'
|
||||
branch: '${{ steps.variables.outputs.BRANCH_NAME }}'
|
||||
push: 'origin origin/${{ steps.variables.outputs.BRANCH_NAME }}'
|
||||
|
||||
- name: Create Pull Request
|
||||
uses: peter-evans/create-pull-request@v3
|
||||
if: ${{ steps.variables.outputs.IS_DRY_RUN == 'true' }}
|
||||
with:
|
||||
author: 'Github Build Bot <buildbot@fishtownanalytics.com>'
|
||||
draft: true
|
||||
base: ${{github.ref}}
|
||||
title: 'Bumping version to ${{steps.variables.outputs.VERSION_NUMBER}}'
|
||||
branch: 'bumping-version/${{steps.variables.outputs.VERSION_NUMBER}}_${{GITHUB.RUN_ID}}'
|
||||
title: 'Bumping version to ${{ github.event.inputs.version_number }} and generate changelog'
|
||||
branch: '${{ steps.variables.outputs.BRANCH_NAME }}'
|
||||
labels: |
|
||||
Skip Changelog
|
||||
|
||||
@@ -35,7 +35,7 @@ repos:
|
||||
- "--target-version=py38"
|
||||
- "--check"
|
||||
- "--diff"
|
||||
- repo: https://gitlab.com/pycqa/flake8
|
||||
- repo: https://github.com/pycqa/flake8
|
||||
rev: 4.0.1
|
||||
hooks:
|
||||
- id: flake8
|
||||
|
||||
83
CHANGELOG.md
83
CHANGELOG.md
@@ -5,8 +5,48 @@
|
||||
- "Breaking changes" listed under a version may require action from end users or external maintainers when upgrading to that version.
|
||||
- Do not edit this file directly. This file is auto-generated using [changie](https://github.com/miniscruff/changie). For details on how to document a change, see [the contributing guide](https://github.com/dbt-labs/dbt-core/blob/main/CONTRIBUTING.md#adding-changelog-entry)
|
||||
|
||||
## dbt-core 1.2.5 - February 28, 2023
|
||||
|
||||
## dbt-core 1.2.0-b1 - June 24, 2022
|
||||
### Fixes
|
||||
|
||||
- add pytz dependency ([#7077](https://github.com/dbt-labs/dbt-core/issues/7077))
|
||||
|
||||
### Contributors
|
||||
- [@sdebruyn](https://github.com/sdebruyn) ([#7077](https://github.com/dbt-labs/dbt-core/issues/7077))
|
||||
|
||||
|
||||
## dbt-core 1.2.4 - January 05, 2023
|
||||
### Fixes
|
||||
- Bug when partial parsing with an empty schema file ([#4850](https://github.com/dbt-labs/dbt-core/issues/4850), [#<no value>](https://github.com/dbt-labs/dbt-core/pull/<no value>))
|
||||
|
||||
## dbt-core 1.2.3 - November 16, 2022
|
||||
### Features
|
||||
- This pulls the profile name from args when constructing a RuntimeConfig in lib.py, enabling the dbt-server to override the value that's in the dbt_project.yml ([#6201](https://github.com/dbt-labs/dbt-core/issues/6201), [#6202](https://github.com/dbt-labs/dbt-core/pull/6202))
|
||||
|
||||
### Contributors
|
||||
- [@racheldaniel](https://github.com/racheldaniel) ([#6202](https://github.com/dbt-labs/dbt-core/pull/6202))
|
||||
|
||||
## dbt-core 1.2.2 - October 03, 2022
|
||||
### Features
|
||||
- This conditionally no-ops warehouse connection at compile depending on an env var, disabling introspection/queries during compilation only. This is a temporary solution to more complex permissions requirements for the semantic layer. ([#5936](https://github.com/dbt-labs/dbt-core/issues/5936), [#5926](https://github.com/dbt-labs/dbt-core/pull/5926))
|
||||
### Fixes
|
||||
- Fix race condition when invoking dbt via lib.py concurrently ([#5919](https://github.com/dbt-labs/dbt-core/issues/5919), [#5921](https://github.com/dbt-labs/dbt-core/pull/5921))
|
||||
|
||||
### Contributors
|
||||
- [@drewbanin](https://github.com/drewbanin) ([#5921](https://github.com/dbt-labs/dbt-core/pull/5921))
|
||||
- [@racheldaniel](https://github.com/racheldaniel) ([#5926](https://github.com/dbt-labs/dbt-core/pull/5926))
|
||||
|
||||
## dbt-core 1.2.1 - August 25, 2022
|
||||
### Fixes
|
||||
- Fix handling of top-level exceptions ([#5564](https://github.com/dbt-labs/dbt-core/issues/5564), [#5560](https://github.com/dbt-labs/dbt-core/pull/5560))
|
||||
- Fix error rendering docs block in metrics description ([#5585](https://github.com/dbt-labs/dbt-core/issues/5585), [#5603](https://github.com/dbt-labs/dbt-core/pull/5603))
|
||||
- Use sys.exit instead of exit ([#5621](https://github.com/dbt-labs/dbt-core/issues/5621), [#5627](https://github.com/dbt-labs/dbt-core/pull/5627))
|
||||
- Finishing logic upgrade to Redshift for name truncation collisions. ([#5586](https://github.com/dbt-labs/dbt-core/issues/5586), [#5656](https://github.com/dbt-labs/dbt-core/pull/5656))
|
||||
|
||||
### Contributors
|
||||
- [@varun-dc](https://github.com/varun-dc) ([#5627](https://github.com/dbt-labs/dbt-core/pull/5627))
|
||||
|
||||
## dbt-core 1.2.0 - July 26, 2022
|
||||
### Features
|
||||
- Add selector method when reading selector definitions ([#4821](https://github.com/dbt-labs/dbt-core/issues/4821), [#4827](https://github.com/dbt-labs/dbt-core/pull/4827))
|
||||
- Add set and zip function to contexts ([#2345](https://github.com/dbt-labs/dbt-core/issues/2345), [#5107](https://github.com/dbt-labs/dbt-core/pull/5107))
|
||||
@@ -18,6 +58,11 @@
|
||||
- Move cross-db macros from dbt-utils into dbt-core global project ([#4813](https://github.com/dbt-labs/dbt-core/issues/4813), [#5265](https://github.com/dbt-labs/dbt-core/pull/5265))
|
||||
- Prettify duration message at the end of execution ([#5253](https://github.com/dbt-labs/dbt-core/issues/5253), [#5364](https://github.com/dbt-labs/dbt-core/pull/5364))
|
||||
- Early return from dbt init if no available adapters ([#5365](https://github.com/dbt-labs/dbt-core/issues/5365), [#5366](https://github.com/dbt-labs/dbt-core/pull/5366))
|
||||
- Allow customizing `target-path` and `log-path` through environment variables and CLI flags. ([#5399](https://github.com/dbt-labs/dbt-core/issues/5399), [#5402](https://github.com/dbt-labs/dbt-core/pull/5402))
|
||||
- Move type_* macros from dbt-utils into dbt-core, with tests ([#5317](https://github.com/dbt-labs/dbt-core/issues/5317), [#5428](https://github.com/dbt-labs/dbt-core/pull/5428))
|
||||
- Add support for ratio metrics ([#4884](https://github.com/dbt-labs/dbt-core/issues/4884), [#5027](https://github.com/dbt-labs/dbt-core/pull/5027))
|
||||
- Allow users to define grants as a reasonable default in the dbt_project.yml or within each model sql or yml file combined. ([#5263](https://github.com/dbt-labs/dbt-core/issues/5263), [#5369](https://github.com/dbt-labs/dbt-core/pull/5369))
|
||||
- Add reusable function for retrying adapter connections. Utilize said function to add retries for Postgres (and Redshift). ([#5022](https://github.com/dbt-labs/dbt-core/issues/5022), [#5432](https://github.com/dbt-labs/dbt-core/pull/5432))
|
||||
### Fixes
|
||||
- Adding new cols to check_cols in snapshots ([#3146](https://github.com/dbt-labs/dbt-core/issues/3146), [#4893](https://github.com/dbt-labs/dbt-core/pull/4893))
|
||||
- Truncate relation names when appending a suffix that will result in len > 63 characters using make_temp_relation and make_backup_relation macros ([#2869](https://github.com/dbt-labs/dbt-core/issues/2869), [#4921](https://github.com/dbt-labs/dbt-core/pull/4921))
|
||||
@@ -37,7 +82,19 @@
|
||||
- Change node ancestor/descendant algo, fixes issue where downstream models aren't run when using networkx >= 2.8.1 ([#5286](https://github.com/dbt-labs/dbt-core/issues/5286), [#5326](https://github.com/dbt-labs/dbt-core/pull/5326))
|
||||
- Fixing Windows color regression ([#5191](https://github.com/dbt-labs/dbt-core/issues/5191), [#5327](https://github.com/dbt-labs/dbt-core/pull/5327))
|
||||
- Define compatibility for older manifest versions when using state: selection methods ([#5213](https://github.com/dbt-labs/dbt-core/issues/5213), [#5346](https://github.com/dbt-labs/dbt-core/pull/5346))
|
||||
- Add inheritance to materialization macro resolution ([#4646](https://github.com/dbt-labs/dbt-core/issues/4646), [#5348](https://github.com/dbt-labs/dbt-core/pull/5348))
|
||||
- Improve pluralizations for Documentation and SqlOperation NodeTypes ([#5352](https://github.com/dbt-labs/dbt-core/issues/5352), [#5356](https://github.com/dbt-labs/dbt-core/pull/5356))
|
||||
- Properly use quotes for Snowflake snapshots when checking all columns ([#2975](https://github.com/dbt-labs/dbt-core/issues/2975), [#5389](https://github.com/dbt-labs/dbt-core/pull/5389))
|
||||
- fixes handling of RESET color code with USE_COLORS=False ([#5288](https://github.com/dbt-labs/dbt-core/issues/5288), [#5394](https://github.com/dbt-labs/dbt-core/pull/5394))
|
||||
- Remove duplicate key checking introduced in 1.2.0a1 ([#5331](https://github.com/dbt-labs/dbt-core/issues/5331), [#5403](https://github.com/dbt-labs/dbt-core/pull/5403))
|
||||
- Rename try to strict for more intuitiveness ([#5475](https://github.com/dbt-labs/dbt-core/issues/5475), [#5477](https://github.com/dbt-labs/dbt-core/pull/5477))
|
||||
### Docs
|
||||
- Fixed sample SQL Code for sources when no database is defined ([#5255](https://github.com/dbt-labs/dbt-core/issues/5255), [#5446](https://github.com/dbt-labs/dbt-core/pull/5446))
|
||||
- Add support for `file:` selector in DAG viz ([#5255](https://github.com/dbt-labs/dbt-core/issues/5255), [#5446](https://github.com/dbt-labs/dbt-core/pull/5446))
|
||||
- [Snyk] Upgrade prismjs from 1.27.0 to 1.28.0 ([#5255](https://github.com/dbt-labs/dbt-core/issues/5255), [#5446](https://github.com/dbt-labs/dbt-core/pull/5446))
|
||||
- Run build and tests in CI checks ([#5255](https://github.com/dbt-labs/dbt-core/issues/5255), [#5446](https://github.com/dbt-labs/dbt-core/pull/5446))
|
||||
- Improve metrics DAG viz and documentation page ([#5255](https://github.com/dbt-labs/dbt-core/issues/5255), [#5446](https://github.com/dbt-labs/dbt-core/pull/5446))
|
||||
- Upgrade cytoscape.js fork ([#5255](https://github.com/dbt-labs/dbt-core/issues/5255), [#5446](https://github.com/dbt-labs/dbt-core/pull/5446))
|
||||
### Under the Hood
|
||||
- Migrating 005_simple_seed to the new test framework. ([#200](https://github.com/dbt-labs/dbt-core/issues/200), [#5013](https://github.com/dbt-labs/dbt-core/pull/5013))
|
||||
- Convert 029_docs_generate tests to new framework ([#5035](https://github.com/dbt-labs/dbt-core/issues/5035), [#5058](https://github.com/dbt-labs/dbt-core/pull/5058))
|
||||
@@ -54,28 +111,44 @@
|
||||
- Fix unit test test_graph_selection ([#5323](https://github.com/dbt-labs/dbt-core/issues/5323), [#5324](https://github.com/dbt-labs/dbt-core/pull/5324))
|
||||
- Update context readme + clean up context code" ([#4796](https://github.com/dbt-labs/dbt-core/issues/4796), [#5334](https://github.com/dbt-labs/dbt-core/pull/5334))
|
||||
- removed script meant for snowflake to snowflake ([#5361](https://github.com/dbt-labs/dbt-core/issues/5361), [#5362](https://github.com/dbt-labs/dbt-core/pull/5362))
|
||||
- Added the suggested RegEx to check the SemVer string within a package dependency and improved invalid version error handling. ([#5201](https://github.com/dbt-labs/dbt-core/issues/5201), [#5370](https://github.com/dbt-labs/dbt-core/pull/5370))
|
||||
- Add annotation to render_value method reimplemented in #5334 ([#4796](https://github.com/dbt-labs/dbt-core/issues/4796), [#5382](https://github.com/dbt-labs/dbt-core/pull/5382))
|
||||
- Bump manifest version to v6 ([#5417](https://github.com/dbt-labs/dbt-core/issues/5417), [#5430](https://github.com/dbt-labs/dbt-core/pull/5430))
|
||||
- Add tests for SQL grants ([#5437](https://github.com/dbt-labs/dbt-core/issues/5437), [#5447](https://github.com/dbt-labs/dbt-core/pull/5447))
|
||||
### Dependencies
|
||||
- Bump ubuntu from 20.04 to 22.04 ([#4904](https://github.com/dbt-labs/dbt-core/issues/4904), [#5141](https://github.com/dbt-labs/dbt-core/pull/5141))
|
||||
- Bumping hologram version ([#5219](https://github.com/dbt-labs/dbt-core/issues/5219), [#5218](https://github.com/dbt-labs/dbt-core/pull/5218))
|
||||
- Bump mypy from 0.942 to 0.961 ([#4904](https://github.com/dbt-labs/dbt-core/issues/4904), [#5337](https://github.com/dbt-labs/dbt-core/pull/5337))
|
||||
- Bump python from 3.10.3-slim-bullseye to 3.10.5-slim-bullseye in /docker ([#4904](https://github.com/dbt-labs/dbt-core/issues/4904), [#5367](https://github.com/dbt-labs/dbt-core/pull/5367))
|
||||
- Update colorama requirement from <0.4.5,>=0.3.9 to >=0.3.9,<0.4.6 in /core ([#4904](https://github.com/dbt-labs/dbt-core/issues/4904), [#5388](https://github.com/dbt-labs/dbt-core/pull/5388))
|
||||
- Bump black from 22.3.0 to 22.6.0 ([#4904](https://github.com/dbt-labs/dbt-core/issues/4904), [#5420](https://github.com/dbt-labs/dbt-core/pull/5420))
|
||||
### Security
|
||||
- Move string interpolation of "secret" env vars outside of Jinja context. Update "contexts" README ([#4796](https://github.com/dbt-labs/dbt-core/issues/4796), [#5334](https://github.com/dbt-labs/dbt-core/pull/5334))
|
||||
|
||||
### Contributors
|
||||
- [@GtheSheep](https://github.com/GtheSheep) ([#4893](https://github.com/dbt-labs/dbt-core/pull/4893))
|
||||
- [@NicolasPA](https://github.com/NicolasPA) ([#5211](https://github.com/dbt-labs/dbt-core/pull/5211))
|
||||
- [@adamantike](https://github.com/adamantike) ([#5207](https://github.com/dbt-labs/dbt-core/pull/5207))
|
||||
- [@alexrosenfeld10](https://github.com/alexrosenfeld10) ([#5184](https://github.com/dbt-labs/dbt-core/pull/5184))
|
||||
- [@b-per](https://github.com/b-per) ([#5446](https://github.com/dbt-labs/dbt-core/pull/5446))
|
||||
- [@bd3dowling](https://github.com/bd3dowling) ([#5140](https://github.com/dbt-labs/dbt-core/pull/5140))
|
||||
- [@callum-mcdata](https://github.com/callum-mcdata) ([#5027](https://github.com/dbt-labs/dbt-core/pull/5027), [#5446](https://github.com/dbt-labs/dbt-core/pull/5446))
|
||||
- [@danieldiamond](https://github.com/danieldiamond) ([#4827](https://github.com/dbt-labs/dbt-core/pull/4827))
|
||||
- [@darin-reify](https://github.com/darin-reify) ([#5394](https://github.com/dbt-labs/dbt-core/pull/5394))
|
||||
- [@dbeatty10](https://github.com/dbeatty10) ([#5265](https://github.com/dbt-labs/dbt-core/pull/5265), [#5077](https://github.com/dbt-labs/dbt-core/pull/5077))
|
||||
- [@dependabot[bot]](https://github.com/dependabot[bot]) ([#5141](https://github.com/dbt-labs/dbt-core/pull/5141), [#5367](https://github.com/dbt-labs/dbt-core/pull/5367))
|
||||
- [@drewbanin](https://github.com/drewbanin) ([#5027](https://github.com/dbt-labs/dbt-core/pull/5027), [#5446](https://github.com/dbt-labs/dbt-core/pull/5446), [#5446](https://github.com/dbt-labs/dbt-core/pull/5446), [#5446](https://github.com/dbt-labs/dbt-core/pull/5446))
|
||||
- [@epapineau](https://github.com/epapineau) ([#4921](https://github.com/dbt-labs/dbt-core/pull/4921))
|
||||
- [@fivetran-joemarkiewicz](https://github.com/fivetran-joemarkiewicz) ([#5370](https://github.com/dbt-labs/dbt-core/pull/5370))
|
||||
- [@groodt](https://github.com/groodt) ([#5304](https://github.com/dbt-labs/dbt-core/pull/5304))
|
||||
- [@isidentical](https://github.com/isidentical) ([#5402](https://github.com/dbt-labs/dbt-core/pull/5402))
|
||||
- [@jared-rimmer](https://github.com/jared-rimmer) ([#5364](https://github.com/dbt-labs/dbt-core/pull/5364))
|
||||
- [@jeremyyeo](https://github.com/jeremyyeo) ([#5107](https://github.com/dbt-labs/dbt-core/pull/5107), [#5146](https://github.com/dbt-labs/dbt-core/pull/5146), [#5403](https://github.com/dbt-labs/dbt-core/pull/5403))
|
||||
- [@jeremyyeo](https://github.com/jeremyyeo) ([#5107](https://github.com/dbt-labs/dbt-core/pull/5107), [#5146](https://github.com/dbt-labs/dbt-core/pull/5146), [#5403](https://github.com/dbt-labs/dbt-core/pull/5403), [#5477](https://github.com/dbt-labs/dbt-core/pull/5477))
|
||||
- [@jwills](https://github.com/jwills) ([#5241](https://github.com/dbt-labs/dbt-core/pull/5241), [#5269](https://github.com/dbt-labs/dbt-core/pull/5269))
|
||||
- [@tomasfarias](https://github.com/tomasfarias) ([#5209](https://github.com/dbt-labs/dbt-core/pull/5209))
|
||||
- [@pdebelak](https://github.com/pdebelak) ([#5356](https://github.com/dbt-labs/dbt-core/pull/5356))
|
||||
- [@pquadri](https://github.com/pquadri) ([#5389](https://github.com/dbt-labs/dbt-core/pull/5389))
|
||||
- [@tomasfarias](https://github.com/tomasfarias) ([#5432](https://github.com/dbt-labs/dbt-core/pull/5432), [#5209](https://github.com/dbt-labs/dbt-core/pull/5209))
|
||||
- [@ulisesojeda](https://github.com/ulisesojeda) ([#5366](https://github.com/dbt-labs/dbt-core/pull/5366))
|
||||
|
||||
- [@volkangurel](https://github.com/volkangurel) ([#5348](https://github.com/dbt-labs/dbt-core/pull/5348))
|
||||
|
||||
## Previous Releases
|
||||
|
||||
|
||||
@@ -174,13 +174,47 @@ python3 -m pytest tests/functional/sources
|
||||
|
||||
> See [pytest usage docs](https://docs.pytest.org/en/6.2.x/usage.html) for an overview of useful command-line options.
|
||||
|
||||
<<<<<<< HEAD
|
||||
## Adding CHANGELOG Entry
|
||||
=======
|
||||
### Unit, Integration, Functional?
|
||||
|
||||
Here are some general rules for adding tests:
|
||||
* unit tests (`test/unit` & `tests/unit`) don’t need to access a database; "pure Python" tests should be written as unit tests
|
||||
* functional tests (`test/integration` & `tests/functional`) cover anything that interacts with a database, namely adapter
|
||||
* *everything in* `test/*` *is being steadily migrated to* `tests/*`
|
||||
|
||||
## Debugging
|
||||
|
||||
1. The logs for a `dbt run` have stack traces and other information for debugging errors (in `logs/dbt.log` in your project directory).
|
||||
2. Try using a debugger, like `ipdb`. For pytest: `--pdb --pdbcls=IPython.terminal.debugger:pdb`
|
||||
3. Sometimes, it’s easier to debug on a single thread: `dbt --single-threaded run`
|
||||
4. To make print statements from Jinja macros: `{{ log(msg, info=true) }}`
|
||||
5. You can also add `{{ debug() }}` statements, which will drop you into some auto-generated code that the macro wrote.
|
||||
6. The dbt “artifacts” are written out to the ‘target’ directory of your dbt project. They are in unformatted json, which can be hard to read. Format them with:
|
||||
> python -m json.tool target/run_results.json > run_results.json
|
||||
|
||||
### Assorted development tips
|
||||
* Append `# type: ignore` to the end of a line if you need to disable `mypy` on that line.
|
||||
* Sometimes flake8 complains about lines that are actually fine, in which case you can put a comment on the line such as: # noqa or # noqa: ANNN, where ANNN is the error code that flake8 issues.
|
||||
* To collect output for `CProfile`, run dbt with the `-r` option and the name of an output file, i.e. `dbt -r dbt.cprof run`. If you just want to profile parsing, you can do: `dbt -r dbt.cprof parse`. `pip` install `snakeviz` to view the output. Run `snakeviz dbt.cprof` and output will be rendered in a browser window.
|
||||
|
||||
## Adding or modifying a CHANGELOG Entry
|
||||
>>>>>>> 0fbbc896b (Remove PR from most changelog kinds (#6374))
|
||||
|
||||
We use [changie](https://changie.dev) to generate `CHANGELOG` entries. **Note:** Do not edit the `CHANGELOG.md` directly. Your modifications will be lost.
|
||||
|
||||
Follow the steps to [install `changie`](https://changie.dev/guide/installation/) for your system.
|
||||
|
||||
Once changie is installed and your PR is created, simply run `changie new` and changie will walk you through the process of creating a changelog entry. Commit the file that's created and your changelog entry is complete!
|
||||
Once changie is installed and your PR is created for a new feature, simply run the following command and changie will walk you through the process of creating a changelog entry:
|
||||
|
||||
```shell
|
||||
changie new
|
||||
```
|
||||
|
||||
Commit the file that's created and your changelog entry is complete!
|
||||
|
||||
If you are contributing to a feature already in progress, you will modify the changie yaml file in dbt/.changes/unreleased/ related to your change. If you need help finding this file, please ask within the discussion for the pull request!
|
||||
|
||||
You don't need to worry about which `dbt-core` version your change will go into. Just create the changelog entry with `changie`, and open your PR against the `main` branch. All merged changes will be included in the next minor version of `dbt-core`. The Core maintainers _may_ choose to "backport" specific changes in order to patch older minor versions. In that case, a maintainer will take care of that backport after merging your PR, before releasing the new version of `dbt-core`.
|
||||
|
||||
|
||||
@@ -1,10 +1,24 @@
|
||||
import abc
|
||||
import os
|
||||
from time import sleep
|
||||
import sys
|
||||
|
||||
# multiprocessing.RLock is a function returning this type
|
||||
from multiprocessing.synchronize import RLock
|
||||
from threading import get_ident
|
||||
from typing import Dict, Tuple, Hashable, Optional, ContextManager, List
|
||||
from typing import (
|
||||
Any,
|
||||
Dict,
|
||||
Tuple,
|
||||
Hashable,
|
||||
Optional,
|
||||
ContextManager,
|
||||
List,
|
||||
Type,
|
||||
Union,
|
||||
Iterable,
|
||||
Callable,
|
||||
)
|
||||
|
||||
import agate
|
||||
|
||||
@@ -21,6 +35,7 @@ from dbt.contracts.graph.manifest import Manifest
|
||||
from dbt.adapters.base.query_headers import (
|
||||
MacroQueryStringSetter,
|
||||
)
|
||||
from dbt.events import AdapterLogger
|
||||
from dbt.events.functions import fire_event
|
||||
from dbt.events.types import (
|
||||
NewConnection,
|
||||
@@ -34,6 +49,9 @@ from dbt.events.types import (
|
||||
)
|
||||
from dbt import flags
|
||||
|
||||
SleepTime = Union[int, float] # As taken by time.sleep.
|
||||
AdapterHandle = Any # Adapter connection handle objects can be any class.
|
||||
|
||||
|
||||
class BaseConnectionManager(metaclass=abc.ABCMeta):
|
||||
"""Methods to implement:
|
||||
@@ -159,6 +177,94 @@ class BaseConnectionManager(metaclass=abc.ABCMeta):
|
||||
conn.name = conn_name
|
||||
return conn
|
||||
|
||||
@classmethod
|
||||
def retry_connection(
|
||||
cls,
|
||||
connection: Connection,
|
||||
connect: Callable[[], AdapterHandle],
|
||||
logger: AdapterLogger,
|
||||
retryable_exceptions: Iterable[Type[Exception]],
|
||||
retry_limit: int = 1,
|
||||
retry_timeout: Union[Callable[[int], SleepTime], SleepTime] = 1,
|
||||
_attempts: int = 0,
|
||||
) -> Connection:
|
||||
"""Given a Connection, set its handle by calling connect.
|
||||
|
||||
The calls to connect will be retried up to retry_limit times to deal with transient
|
||||
connection errors. By default, one retry will be attempted if retryable_exceptions is set.
|
||||
|
||||
:param Connection connection: An instance of a Connection that needs a handle to be set,
|
||||
usually when attempting to open it.
|
||||
:param connect: A callable that returns the appropiate connection handle for a
|
||||
given adapter. This callable will be retried retry_limit times if a subclass of any
|
||||
Exception in retryable_exceptions is raised by connect.
|
||||
:type connect: Callable[[], AdapterHandle]
|
||||
:param AdapterLogger logger: A logger to emit messages on retry attempts or errors. When
|
||||
handling expected errors, we call debug, and call warning on unexpected errors or when
|
||||
all retry attempts have been exhausted.
|
||||
:param retryable_exceptions: An iterable of exception classes that if raised by
|
||||
connect should trigger a retry.
|
||||
:type retryable_exceptions: Iterable[Type[Exception]]
|
||||
:param int retry_limit: How many times to retry the call to connect. If this limit
|
||||
is exceeded before a successful call, a FailedToConnectException will be raised.
|
||||
Must be non-negative.
|
||||
:param retry_timeout: Time to wait between attempts to connect. Can also take a
|
||||
Callable that takes the number of attempts so far, beginning at 0, and returns an int
|
||||
or float to be passed to time.sleep.
|
||||
:type retry_timeout: Union[Callable[[int], SleepTime], SleepTime] = 1
|
||||
:param int _attempts: Parameter used to keep track of the number of attempts in calling the
|
||||
connect function across recursive calls. Passed as an argument to retry_timeout if it
|
||||
is a Callable. This parameter should not be set by the initial caller.
|
||||
:raises dbt.exceptions.FailedToConnectException: Upon exhausting all retry attempts without
|
||||
successfully acquiring a handle.
|
||||
:return: The given connection with its appropriate state and handle attributes set
|
||||
depending on whether we successfully acquired a handle or not.
|
||||
"""
|
||||
timeout = retry_timeout(_attempts) if callable(retry_timeout) else retry_timeout
|
||||
if timeout < 0:
|
||||
raise dbt.exceptions.FailedToConnectException(
|
||||
"retry_timeout cannot be negative or return a negative time."
|
||||
)
|
||||
|
||||
if retry_limit < 0 or retry_limit > sys.getrecursionlimit():
|
||||
# This guard is not perfect others may add to the recursion limit (e.g. built-ins).
|
||||
connection.handle = None
|
||||
connection.state = ConnectionState.FAIL
|
||||
raise dbt.exceptions.FailedToConnectException("retry_limit cannot be negative")
|
||||
|
||||
try:
|
||||
connection.handle = connect()
|
||||
connection.state = ConnectionState.OPEN
|
||||
return connection
|
||||
|
||||
except tuple(retryable_exceptions) as e:
|
||||
if retry_limit <= 0:
|
||||
connection.handle = None
|
||||
connection.state = ConnectionState.FAIL
|
||||
raise dbt.exceptions.FailedToConnectException(str(e))
|
||||
|
||||
logger.debug(
|
||||
f"Got a retryable error when attempting to open a {cls.TYPE} connection.\n"
|
||||
f"{retry_limit} attempts remaining. Retrying in {timeout} seconds.\n"
|
||||
f"Error:\n{e}"
|
||||
)
|
||||
|
||||
sleep(timeout)
|
||||
return cls.retry_connection(
|
||||
connection=connection,
|
||||
connect=connect,
|
||||
logger=logger,
|
||||
retry_limit=retry_limit - 1,
|
||||
retry_timeout=retry_timeout,
|
||||
retryable_exceptions=retryable_exceptions,
|
||||
_attempts=_attempts + 1,
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
connection.handle = None
|
||||
connection.state = ConnectionState.FAIL
|
||||
raise dbt.exceptions.FailedToConnectException(str(e))
|
||||
|
||||
@abc.abstractmethod
|
||||
def cancel_open(self) -> Optional[List[str]]:
|
||||
"""Cancel all open connections on the adapter. (passable)"""
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import re
|
||||
import threading
|
||||
from copy import deepcopy
|
||||
from typing import Any, Dict, Iterable, List, Optional, Set, Tuple
|
||||
@@ -413,9 +414,24 @@ class RelationsCache:
|
||||
:raises InternalError: If the new key is already present.
|
||||
"""
|
||||
if new_key in self.relations:
|
||||
# Tell user when collision caused by model names truncated during
|
||||
# materialization.
|
||||
match = re.search("__dbt_backup|__dbt_tmp$", new_key.identifier)
|
||||
if match:
|
||||
truncated_model_name_prefix = new_key.identifier[: match.start()]
|
||||
message_addendum = (
|
||||
"\n\nName collisions can occur when the length of two "
|
||||
"models' names approach your database's builtin limit. "
|
||||
"Try restructuring your project such that no two models "
|
||||
"share the prefix '{}'.".format(truncated_model_name_prefix)
|
||||
+ " Then, clean your warehouse of any removed models."
|
||||
)
|
||||
else:
|
||||
message_addendum = ""
|
||||
|
||||
dbt.exceptions.raise_cache_inconsistent(
|
||||
"in rename, new key {} already in cache: {}".format(
|
||||
new_key, list(self.relations.keys())
|
||||
"in rename, new key {} already in cache: {}{}".format(
|
||||
new_key, list(self.relations.keys()), message_addendum
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@@ -3,31 +3,42 @@ import os
|
||||
from copy import deepcopy
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Dict, Any, Optional, Mapping, Iterator, Iterable, Tuple, List, MutableSet, Type
|
||||
from typing import (
|
||||
Any,
|
||||
Dict,
|
||||
Iterable,
|
||||
Iterator,
|
||||
List,
|
||||
Mapping,
|
||||
MutableSet,
|
||||
Optional,
|
||||
Tuple,
|
||||
Type,
|
||||
Union,
|
||||
)
|
||||
|
||||
from .profile import Profile
|
||||
from .project import Project
|
||||
from .renderer import DbtProjectYamlRenderer, ProfileRenderer
|
||||
from .utils import parse_cli_vars
|
||||
from dbt import flags
|
||||
from dbt.adapters.factory import get_relation_class_by_name, get_include_paths
|
||||
from dbt.helper_types import FQNPath, PathSet, DictDefaultEmptyStr
|
||||
from dbt.adapters.factory import get_include_paths, get_relation_class_by_name
|
||||
from dbt.config.profile import read_user_config
|
||||
from dbt.contracts.connection import AdapterRequiredConfig, Credentials
|
||||
from dbt.contracts.graph.manifest import ManifestMetadata
|
||||
from dbt.contracts.relation import ComponentName
|
||||
from dbt.ui import warning_tag
|
||||
|
||||
from dbt.contracts.project import Configuration, UserConfig
|
||||
from dbt.contracts.relation import ComponentName
|
||||
from dbt.dataclass_schema import ValidationError
|
||||
from dbt.exceptions import (
|
||||
RuntimeException,
|
||||
DbtProjectError,
|
||||
RuntimeException,
|
||||
raise_compiler_error,
|
||||
validator_error_message,
|
||||
warn_or_error,
|
||||
raise_compiler_error,
|
||||
)
|
||||
from dbt.helper_types import DictDefaultEmptyStr, FQNPath, PathSet
|
||||
from dbt.ui import warning_tag
|
||||
|
||||
from dbt.dataclass_schema import ValidationError
|
||||
from .profile import Profile
|
||||
from .project import Project, PartialProject
|
||||
from .renderer import DbtProjectYamlRenderer, ProfileRenderer
|
||||
from .utils import parse_cli_vars
|
||||
|
||||
|
||||
def _project_quoting_dict(proj: Project, profile: Profile) -> Dict[ComponentName, bool]:
|
||||
@@ -188,28 +199,52 @@ class RuntimeConfig(Project, Profile, AdapterRequiredConfig):
|
||||
|
||||
@classmethod
|
||||
def collect_parts(cls: Type["RuntimeConfig"], args: Any) -> Tuple[Project, Profile]:
|
||||
# profile_name from the project
|
||||
project_root = args.project_dir if args.project_dir else os.getcwd()
|
||||
version_check = bool(flags.VERSION_CHECK)
|
||||
partial = Project.partial_load(project_root, verify_version=version_check)
|
||||
|
||||
# build the profile using the base renderer and the one fact we know
|
||||
# Note: only the named profile section is rendered. The rest of the
|
||||
# profile is ignored.
|
||||
cli_vars: Dict[str, Any] = parse_cli_vars(getattr(args, "vars", "{}"))
|
||||
|
||||
profile = cls.collect_profile(args=args)
|
||||
project_renderer = DbtProjectYamlRenderer(profile, cli_vars)
|
||||
project = cls.collect_project(args=args, project_renderer=project_renderer)
|
||||
assert type(project) is Project
|
||||
return (project, profile)
|
||||
|
||||
@classmethod
|
||||
def collect_profile(
|
||||
cls: Type["RuntimeConfig"], args: Any, profile_name: Optional[str] = None
|
||||
) -> Profile:
|
||||
|
||||
cli_vars: Dict[str, Any] = parse_cli_vars(getattr(args, "vars", "{}"))
|
||||
profile_renderer = ProfileRenderer(cli_vars)
|
||||
profile_name = partial.render_profile_name(profile_renderer)
|
||||
|
||||
# build the profile using the base renderer and the one fact we know
|
||||
if profile_name is None:
|
||||
# Note: only the named profile section is rendered here. The rest of the
|
||||
# profile is ignored.
|
||||
partial = cls.collect_project(args)
|
||||
assert type(partial) is PartialProject
|
||||
profile_name = partial.render_profile_name(profile_renderer)
|
||||
|
||||
profile = cls._get_rendered_profile(args, profile_renderer, profile_name)
|
||||
# Save env_vars encountered in rendering for partial parsing
|
||||
profile.profile_env_vars = profile_renderer.ctx_obj.env_vars
|
||||
return profile
|
||||
|
||||
# get a new renderer using our target information and render the
|
||||
# project
|
||||
project_renderer = DbtProjectYamlRenderer(profile, cli_vars)
|
||||
project = partial.render(project_renderer)
|
||||
# Save env_vars encountered in rendering for partial parsing
|
||||
project.project_env_vars = project_renderer.ctx_obj.env_vars
|
||||
return (project, profile)
|
||||
@classmethod
|
||||
def collect_project(
|
||||
cls: Type["RuntimeConfig"],
|
||||
args: Any,
|
||||
project_renderer: Optional[DbtProjectYamlRenderer] = None,
|
||||
) -> Union[Project, PartialProject]:
|
||||
|
||||
project_root = args.project_dir if args.project_dir else os.getcwd()
|
||||
version_check = bool(flags.VERSION_CHECK)
|
||||
partial = Project.partial_load(project_root, verify_version=version_check)
|
||||
if project_renderer is None:
|
||||
return partial
|
||||
else:
|
||||
project = partial.render(project_renderer)
|
||||
project.project_env_vars = project_renderer.ctx_obj.env_vars
|
||||
return project
|
||||
|
||||
# Called in main.py, lib.py, task/base.py
|
||||
@classmethod
|
||||
|
||||
@@ -474,19 +474,17 @@ class BaseContext(metaclass=ContextMeta):
|
||||
|
||||
@contextmember
|
||||
@staticmethod
|
||||
def try_set(value: Iterable[Any]) -> Set[Any]:
|
||||
"""The `try_set` context method can be used to convert any iterable
|
||||
def set_strict(value: Iterable[Any]) -> Set[Any]:
|
||||
"""The `set_strict` context method can be used to convert any iterable
|
||||
to a sequence of iterable elements that are unique (a set). The
|
||||
difference to the `set` context method is that the `try_set` method
|
||||
difference to the `set` context method is that the `set_strict` method
|
||||
will raise an exception on a TypeError.
|
||||
|
||||
:param value: The iterable
|
||||
:param default: A default value to return if the `value` argument
|
||||
is not an iterable
|
||||
|
||||
Usage:
|
||||
{% set my_list = [1, 2, 2, 3] %}
|
||||
{% set my_set = try_set(my_list) %}
|
||||
{% set my_set = set_strict(my_list) %}
|
||||
{% do log(my_set) %} {# {1, 2, 3} #}
|
||||
"""
|
||||
try:
|
||||
@@ -497,7 +495,7 @@ class BaseContext(metaclass=ContextMeta):
|
||||
@contextmember("zip")
|
||||
@staticmethod
|
||||
def _zip(*args: Iterable[Any], default: Any = None) -> Optional[Iterable[Any]]:
|
||||
"""The `try_zip` context method can be used to used to return
|
||||
"""The `zip` context method can be used to used to return
|
||||
an iterator of tuples, where the i-th tuple contains the i-th
|
||||
element from each of the argument iterables.
|
||||
|
||||
@@ -518,21 +516,19 @@ class BaseContext(metaclass=ContextMeta):
|
||||
|
||||
@contextmember
|
||||
@staticmethod
|
||||
def try_zip(*args: Iterable[Any]) -> Iterable[Any]:
|
||||
"""The `try_zip` context method can be used to used to return
|
||||
def zip_strict(*args: Iterable[Any]) -> Iterable[Any]:
|
||||
"""The `zip_strict` context method can be used to used to return
|
||||
an iterator of tuples, where the i-th tuple contains the i-th
|
||||
element from each of the argument iterables. The difference to the
|
||||
`zip` context method is that the `try_zip` method will raise an
|
||||
`zip` context method is that the `zip_strict` method will raise an
|
||||
exception on a TypeError.
|
||||
|
||||
:param *args: Any number of iterables
|
||||
:param default: A default value to return if `*args` is not
|
||||
iterable
|
||||
|
||||
Usage:
|
||||
{% set my_list_a = [1, 2] %}
|
||||
{% set my_list_b = ['alice', 'bob'] %}
|
||||
{% set my_zip = try_zip(my_list_a, my_list_b) | list %}
|
||||
{% set my_zip = zip_strict(my_list_a, my_list_b) | list %}
|
||||
{% do log(my_set) %} {# [(1, 'alice'), (2, 'bob')] #}
|
||||
"""
|
||||
try:
|
||||
@@ -635,9 +631,8 @@ class BaseContext(metaclass=ContextMeta):
|
||||
{% endif %}
|
||||
|
||||
This supports all flags defined in flags submodule (core/dbt/flags.py)
|
||||
TODO: Replace with object that provides read-only access to flag values
|
||||
"""
|
||||
return flags
|
||||
return flags.get_flag_obj()
|
||||
|
||||
@contextmember
|
||||
@staticmethod
|
||||
|
||||
@@ -105,6 +105,8 @@ class ErrorLevel(EventSerialization, Event):
|
||||
|
||||
|
||||
# prevents an event from going to the file
|
||||
# This should rarely be used in core code. It is currently
|
||||
# only used in integration tests and for the 'clean' command.
|
||||
class NoFile:
|
||||
pass
|
||||
|
||||
|
||||
@@ -218,7 +218,7 @@ def create_log_line(e: T_Event, file_output=False) -> Optional[str]:
|
||||
return create_info_text_log_line(e) # console output
|
||||
|
||||
|
||||
# allows for resuse of this obnoxious if else tree.
|
||||
# allows for reuse of this obnoxious if else tree.
|
||||
# do not use for exceptions, it doesn't pass along exc_info, stack_info, or extra
|
||||
def send_to_logger(l: Union[Logger, logbook.Logger], level_tag: str, log_line: str):
|
||||
if not log_line:
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
from dataclasses import dataclass
|
||||
from .types import InfoLevel, DebugLevel, WarnLevel, ErrorLevel, ShowException, NoFile
|
||||
from dbt.events.types import InfoLevel, DebugLevel, WarnLevel, ErrorLevel, ShowException
|
||||
from dbt.events.base_types import NoFile
|
||||
|
||||
|
||||
# Keeping log messages for testing separate since they are used for debugging.
|
||||
|
||||
@@ -94,7 +94,7 @@ class AdapterEventError(ErrorLevel, AdapterEventBase, ShowException):
|
||||
|
||||
|
||||
@dataclass
|
||||
class MainKeyboardInterrupt(InfoLevel, NoFile):
|
||||
class MainKeyboardInterrupt(InfoLevel):
|
||||
code: str = "Z001"
|
||||
|
||||
def message(self) -> str:
|
||||
@@ -102,7 +102,7 @@ class MainKeyboardInterrupt(InfoLevel, NoFile):
|
||||
|
||||
|
||||
@dataclass
|
||||
class MainEncounteredError(ErrorLevel, NoFile):
|
||||
class MainEncounteredError(ErrorLevel):
|
||||
e: BaseException
|
||||
code: str = "Z002"
|
||||
|
||||
@@ -111,7 +111,7 @@ class MainEncounteredError(ErrorLevel, NoFile):
|
||||
|
||||
|
||||
@dataclass
|
||||
class MainStackTrace(DebugLevel, NoFile):
|
||||
class MainStackTrace(ErrorLevel):
|
||||
stack_trace: str
|
||||
code: str = "Z003"
|
||||
|
||||
@@ -1353,6 +1353,8 @@ class NodeConnectionReleaseError(ShowException, DebugLevel):
|
||||
return "Error releasing connection for node {}: {!s}".format(self.node_name, self.exc)
|
||||
|
||||
|
||||
# We don't write "clean" events to the log, because the clean command
|
||||
# may have removed the log directory.
|
||||
@dataclass
|
||||
class CheckCleanPath(InfoLevel, NoFile):
|
||||
path: str
|
||||
@@ -2382,7 +2384,7 @@ class SendEventFailure(DebugLevel):
|
||||
|
||||
|
||||
@dataclass
|
||||
class FlushEvents(DebugLevel, NoFile):
|
||||
class FlushEvents(DebugLevel):
|
||||
code: str = "Z042"
|
||||
|
||||
def message(self) -> str:
|
||||
@@ -2390,7 +2392,7 @@ class FlushEvents(DebugLevel, NoFile):
|
||||
|
||||
|
||||
@dataclass
|
||||
class FlushEventsFailure(DebugLevel, NoFile):
|
||||
class FlushEventsFailure(DebugLevel):
|
||||
code: str = "Z043"
|
||||
|
||||
def message(self) -> str:
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
import os
|
||||
# Do not import the os package because we expose this package in jinja
|
||||
from os import name as os_name, path as os_path, getenv as os_getenv
|
||||
import multiprocessing
|
||||
from argparse import Namespace
|
||||
|
||||
if os.name != "nt":
|
||||
if os_name != "nt":
|
||||
# https://bugs.python.org/issue41567
|
||||
import multiprocessing.popen_spawn_posix # type: ignore
|
||||
from pathlib import Path
|
||||
@@ -10,8 +12,8 @@ from typing import Optional
|
||||
# PROFILES_DIR must be set before the other flags
|
||||
# It also gets set in main.py and in set_from_args because the rpc server
|
||||
# doesn't go through exactly the same main arg processing.
|
||||
DEFAULT_PROFILES_DIR = os.path.join(os.path.expanduser("~"), ".dbt")
|
||||
PROFILES_DIR = os.path.expanduser(os.getenv("DBT_PROFILES_DIR", DEFAULT_PROFILES_DIR))
|
||||
DEFAULT_PROFILES_DIR = os_path.join(os_path.expanduser("~"), ".dbt")
|
||||
PROFILES_DIR = os_path.expanduser(os_getenv("DBT_PROFILES_DIR", DEFAULT_PROFILES_DIR))
|
||||
|
||||
STRICT_MODE = False # Only here for backwards compatibility
|
||||
FULL_REFRESH = False # subcommand
|
||||
@@ -84,7 +86,7 @@ def env_set_truthy(key: str) -> Optional[str]:
|
||||
"""Return the value if it was set to a "truthy" string value or None
|
||||
otherwise.
|
||||
"""
|
||||
value = os.getenv(key)
|
||||
value = os_getenv(key)
|
||||
if not value or value.lower() in ("0", "false", "f"):
|
||||
return None
|
||||
return value
|
||||
@@ -97,7 +99,7 @@ def env_set_bool(env_value):
|
||||
|
||||
|
||||
def env_set_path(key: str) -> Optional[Path]:
|
||||
value = os.getenv(key)
|
||||
value = os_getenv(key)
|
||||
if value is None:
|
||||
return value
|
||||
else:
|
||||
@@ -177,7 +179,7 @@ def get_flag_value(flag, args, user_config):
|
||||
if flag in ["PRINTER_WIDTH", "EVENT_BUFFER_SIZE"]: # must be ints
|
||||
flag_value = int(flag_value)
|
||||
if flag == "PROFILES_DIR":
|
||||
flag_value = os.path.abspath(flag_value)
|
||||
flag_value = os_path.abspath(flag_value)
|
||||
|
||||
return flag_value
|
||||
|
||||
@@ -201,7 +203,7 @@ def _load_flag_value(flag, args, user_config):
|
||||
def _get_flag_value_from_env(flag):
|
||||
# Environment variables use pattern 'DBT_{flag name}'
|
||||
env_flag = _get_env_flag(flag)
|
||||
env_value = os.getenv(env_flag)
|
||||
env_value = os_getenv(env_flag)
|
||||
if env_value is None or env_value == "":
|
||||
return None
|
||||
|
||||
@@ -238,4 +240,21 @@ def get_flag_dict():
|
||||
"event_buffer_size": EVENT_BUFFER_SIZE,
|
||||
"quiet": QUIET,
|
||||
"no_print": NO_PRINT,
|
||||
"cache_selected_only": CACHE_SELECTED_ONLY,
|
||||
"target_path": TARGET_PATH,
|
||||
"log_path": LOG_PATH,
|
||||
}
|
||||
|
||||
|
||||
# This is used by core/dbt/context/base.py to return a flag object
|
||||
# in Jinja.
|
||||
def get_flag_obj():
|
||||
new_flags = Namespace()
|
||||
for k, v in get_flag_dict().items():
|
||||
setattr(new_flags, k.upper(), v)
|
||||
# The following 3 are CLI arguments only so they're not full-fledged flags,
|
||||
# but we put in flags for users.
|
||||
setattr(new_flags, "FULL_REFRESH", FULL_REFRESH)
|
||||
setattr(new_flags, "STORE_FAILURES", STORE_FAILURES)
|
||||
setattr(new_flags, "WHICH", WHICH)
|
||||
return new_flags
|
||||
|
||||
124
core/dbt/lib.py
124
core/dbt/lib.py
@@ -1,10 +1,66 @@
|
||||
# TODO: this file is one big TODO
|
||||
import os
|
||||
from dbt.config.project import Project
|
||||
from dbt.config.renderer import DbtProjectYamlRenderer
|
||||
from dbt.contracts.results import RunningStatus, collect_timing_info
|
||||
from dbt.events.functions import fire_event
|
||||
from dbt.events.types import NodeCompiling, NodeExecuting
|
||||
from dbt.exceptions import RuntimeException
|
||||
from dbt import flags
|
||||
from collections import namedtuple
|
||||
from dbt.task.sql import SqlCompileRunner
|
||||
from dataclasses import dataclass
|
||||
|
||||
RuntimeArgs = namedtuple("RuntimeArgs", "project_dir profiles_dir single_threaded profile target")
|
||||
|
||||
@dataclass
|
||||
class RuntimeArgs:
|
||||
project_dir: str
|
||||
profiles_dir: str
|
||||
single_threaded: bool
|
||||
profile: str
|
||||
target: str
|
||||
|
||||
|
||||
class SqlCompileRunnerNoIntrospection(SqlCompileRunner):
|
||||
def compile_and_execute(self, manifest, ctx):
|
||||
"""
|
||||
This version of this method does not connect to the data warehouse.
|
||||
As a result, introspective queries at compilation will not be supported
|
||||
and will throw an error.
|
||||
|
||||
TODO: This is a temporary solution to more complex permissions requirements
|
||||
for the semantic layer, and thus largely duplicates the code in the parent class
|
||||
method. Once conditional credential usage is enabled, this should be removed.
|
||||
"""
|
||||
result = None
|
||||
ctx.node._event_status["node_status"] = RunningStatus.Compiling
|
||||
fire_event(
|
||||
NodeCompiling(
|
||||
node_info=ctx.node.node_info,
|
||||
unique_id=ctx.node.unique_id,
|
||||
)
|
||||
)
|
||||
with collect_timing_info("compile") as timing_info:
|
||||
# if we fail here, we still have a compiled node to return
|
||||
# this has the benefit of showing a build path for the errant
|
||||
# model
|
||||
ctx.node = self.compile(manifest)
|
||||
ctx.timing.append(timing_info)
|
||||
|
||||
# for ephemeral nodes, we only want to compile, not run
|
||||
if not ctx.node.is_ephemeral_model:
|
||||
ctx.node._event_status["node_status"] = RunningStatus.Executing
|
||||
fire_event(
|
||||
NodeExecuting(
|
||||
node_info=ctx.node.node_info,
|
||||
unique_id=ctx.node.unique_id,
|
||||
)
|
||||
)
|
||||
with collect_timing_info("execute") as timing_info:
|
||||
result = self.run(ctx.node, manifest)
|
||||
ctx.node = result.node
|
||||
|
||||
ctx.timing.append(timing_info)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def get_dbt_config(project_dir, args=None, single_threaded=False):
|
||||
@@ -17,27 +73,36 @@ def get_dbt_config(project_dir, args=None, single_threaded=False):
|
||||
else:
|
||||
profiles_dir = os.path.expanduser("~/.dbt")
|
||||
|
||||
profile = args.profile if hasattr(args, "profile") else None
|
||||
target = args.target if hasattr(args, "target") else None
|
||||
profile_name = getattr(args, "profile", None)
|
||||
|
||||
# Construct a phony config
|
||||
config = RuntimeConfig.from_args(
|
||||
RuntimeArgs(project_dir, profiles_dir, single_threaded, profile, target)
|
||||
runtime_args = RuntimeArgs(
|
||||
project_dir=project_dir,
|
||||
profiles_dir=profiles_dir,
|
||||
single_threaded=single_threaded,
|
||||
profile=profile_name,
|
||||
target=getattr(args, "target", None),
|
||||
)
|
||||
# Clear previously registered adapters--
|
||||
# this fixes cacheing behavior on the dbt-server
|
||||
|
||||
profile = RuntimeConfig.collect_profile(args=runtime_args, profile_name=profile_name)
|
||||
project_renderer = DbtProjectYamlRenderer(profile, None)
|
||||
project = RuntimeConfig.collect_project(args=runtime_args, project_renderer=project_renderer)
|
||||
assert type(project) is Project
|
||||
|
||||
config = RuntimeConfig.from_parts(project, profile, runtime_args)
|
||||
|
||||
# Set global flags from arguments
|
||||
flags.set_from_args(args, config)
|
||||
dbt.adapters.factory.reset_adapters()
|
||||
# Load the relevant adapter
|
||||
|
||||
# This is idempotent, so we can call it repeatedly
|
||||
dbt.adapters.factory.register_adapter(config)
|
||||
# Set invocation id
|
||||
|
||||
# Make sure we have a valid invocation_id
|
||||
dbt.events.functions.set_invocation_id()
|
||||
|
||||
return config
|
||||
|
||||
|
||||
def get_task_by_type(type):
|
||||
# TODO: we need to tell dbt-server what tasks are available
|
||||
from dbt.task.run import RunTask
|
||||
from dbt.task.list import ListTask
|
||||
from dbt.task.seed import SeedTask
|
||||
@@ -70,16 +135,13 @@ def create_task(type, args, manifest, config):
|
||||
def no_op(*args, **kwargs):
|
||||
pass
|
||||
|
||||
# TODO: yuck, let's rethink tasks a little
|
||||
task = task(args, config)
|
||||
|
||||
# Wow! We can monkeypatch taskCls.load_manifest to return _our_ manifest
|
||||
task.load_manifest = no_op
|
||||
task.manifest = manifest
|
||||
return task
|
||||
|
||||
|
||||
def _get_operation_node(manifest, project_path, sql):
|
||||
def _get_operation_node(manifest, project_path, sql, node_name):
|
||||
from dbt.parser.manifest import process_node
|
||||
from dbt.parser.sql import SqlBlockParser
|
||||
import dbt.adapters.factory
|
||||
@@ -92,26 +154,33 @@ def _get_operation_node(manifest, project_path, sql):
|
||||
)
|
||||
|
||||
adapter = dbt.adapters.factory.get_adapter(config)
|
||||
# TODO : This needs a real name?
|
||||
sql_node = block_parser.parse_remote(sql, "name")
|
||||
sql_node = block_parser.parse_remote(sql, node_name)
|
||||
process_node(config, manifest, sql_node)
|
||||
return config, sql_node, adapter
|
||||
|
||||
|
||||
def compile_sql(manifest, project_path, sql):
|
||||
from dbt.task.sql import SqlCompileRunner
|
||||
def compile_sql(manifest, project_path, sql, node_name="query"):
|
||||
config, node, adapter = _get_operation_node(manifest, project_path, sql, node_name)
|
||||
allow_introspection = str(os.environ.get("__DBT_ALLOW_INTROSPECTION", "1")).lower() in (
|
||||
"true",
|
||||
"1",
|
||||
"on",
|
||||
)
|
||||
|
||||
config, node, adapter = _get_operation_node(manifest, project_path, sql)
|
||||
runner = SqlCompileRunner(config, adapter, node, 1, 1)
|
||||
if allow_introspection:
|
||||
runner = SqlCompileRunner(config, adapter, node, 1, 1)
|
||||
else:
|
||||
runner = SqlCompileRunnerNoIntrospection(config, adapter, node, 1, 1)
|
||||
return runner.safe_run(manifest)
|
||||
|
||||
|
||||
def execute_sql(manifest, project_path, sql):
|
||||
def execute_sql(manifest, project_path, sql, node_name="query"):
|
||||
from dbt.task.sql import SqlExecuteRunner
|
||||
|
||||
config, node, adapter = _get_operation_node(manifest, project_path, sql)
|
||||
config, node, adapter = _get_operation_node(manifest, project_path, sql, node_name)
|
||||
|
||||
runner = SqlExecuteRunner(config, adapter, node, 1, 1)
|
||||
# TODO: use same interface for runner
|
||||
|
||||
return runner.safe_run(manifest)
|
||||
|
||||
|
||||
@@ -128,5 +197,4 @@ def deserialize_manifest(manifest_msgpack):
|
||||
|
||||
|
||||
def serialize_manifest(manifest):
|
||||
# TODO: what should this take as an arg?
|
||||
return manifest.to_msgpack()
|
||||
|
||||
@@ -142,7 +142,6 @@ def main(args=None):
|
||||
exit_code = e.code
|
||||
|
||||
except BaseException as e:
|
||||
traceback.print_exc()
|
||||
fire_event(MainEncounteredError(e=str(e)))
|
||||
fire_event(MainStackTrace(stack_trace=traceback.format_exc()))
|
||||
exit_code = ExitCodes.UnhandledError.value
|
||||
|
||||
@@ -69,6 +69,8 @@ class SchemaYamlRenderer(BaseRenderer):
|
||||
elif self.key == "metrics":
|
||||
if keypath[0] == "sql":
|
||||
return False
|
||||
elif self._is_norender_key(keypath[0:]):
|
||||
return False
|
||||
else: # models, seeds, snapshots, analyses
|
||||
if self._is_norender_key(keypath[0:]):
|
||||
return False
|
||||
|
||||
@@ -115,7 +115,8 @@ def yaml_from_file(source_file: SchemaSourceFile) -> Dict[str, Any]:
|
||||
"""If loading the yaml fails, raise an exception."""
|
||||
path = source_file.path.relative_path
|
||||
try:
|
||||
return load_yaml_text(source_file.contents, source_file.path)
|
||||
# source_file.contents can sometimes be None
|
||||
return load_yaml_text(source_file.contents or "", source_file.path)
|
||||
except ValidationException as e:
|
||||
reason = validator_error_message(e)
|
||||
raise ParsingException(
|
||||
|
||||
@@ -3,6 +3,7 @@ import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import shutil
|
||||
import sys
|
||||
from typing import Optional
|
||||
|
||||
import yaml
|
||||
@@ -302,7 +303,7 @@ class InitTask(BaseTask):
|
||||
available_adapters = list(_get_adapter_plugin_names())
|
||||
if not len(available_adapters):
|
||||
print("No adapters available. Go to https://docs.getdbt.com/docs/available-adapters")
|
||||
exit(1)
|
||||
sys.exit(1)
|
||||
project_name = self.get_valid_project_name()
|
||||
project_path = Path(project_name)
|
||||
if project_path.exists():
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user