Compare commits

..

1 Commits

Author SHA1 Message Date
Michelle Ark
b9f893a291 postgres as service 2024-09-10 15:09:06 -04:00
250 changed files with 1961 additions and 10461 deletions

View File

@@ -1,5 +1,5 @@
[bumpversion] [bumpversion]
current_version = 1.10.0a1 current_version = 1.9.0a1
parse = (?P<major>[\d]+) # major version number parse = (?P<major>[\d]+) # major version number
\.(?P<minor>[\d]+) # minor version number \.(?P<minor>[\d]+) # minor version number
\.(?P<patch>[\d]+) # patch version number \.(?P<patch>[\d]+) # patch version number

View File

@@ -0,0 +1,6 @@
kind: Breaking Changes
body: Fix changing the current working directory when using dpt deps, clean and init.
time: 2023-12-06T19:24:42.575372+09:00
custom:
Author: rariyama
Issue: "8997"

View File

@@ -0,0 +1,6 @@
kind: Dependencies
body: Remove logbook dependency
time: 2024-05-09T09:37:17.745129-05:00
custom:
Author: emmyoop
Issue: "8027"

View File

@@ -0,0 +1,7 @@
kind: Dependencies
body: Increase supported version range for dbt-semantic-interfaces. Needed to support
custom calendar features.
time: 2024-08-20T13:19:09.015225-07:00
custom:
Author: courtneyholcomb
Issue: "9265"

View File

@@ -0,0 +1,6 @@
kind: Docs
body: Enable display of unit tests
time: 2024-03-11T14:03:44.490834-04:00
custom:
Author: gshank
Issue: "501"

View File

@@ -0,0 +1,6 @@
kind: Docs
body: Unit tests not rendering
time: 2024-05-01T02:10:50.987412+02:00
custom:
Author: aranke
Issue: "506"

View File

@@ -0,0 +1,6 @@
kind: Docs
body: Add support for Saved Query node
time: 2024-05-16T22:30:36.206492-07:00
custom:
Author: ChenyuLInx
Issue: "486"

View File

@@ -0,0 +1,6 @@
kind: Docs
body: Fix npm security vulnerabilities as of June 2024
time: 2024-06-13T15:10:48.301989+01:00
custom:
Author: aranke
Issue: "513"

View File

@@ -0,0 +1,6 @@
kind: Features
body: serialize inferred primary key
time: 2024-05-06T17:56:42.757673-05:00
custom:
Author: dave-connors-3
Issue: "9824"

View File

@@ -0,0 +1,6 @@
kind: Features
body: 'Add unit_test: selection method'
time: 2024-05-07T16:27:17.047585-04:00
custom:
Author: michelleark
Issue: "10053"

View File

@@ -0,0 +1,7 @@
kind: Features
body: Maximally parallelize dbt clone
in clone command"
time: 2024-05-22T00:03:09.765977-04:00
custom:
Author: michelleark
Issue: "7914"

View File

@@ -0,0 +1,6 @@
kind: Features
body: Add --host flag to dbt docs serve, defaulting to '127.0.0.1'
time: 2024-05-27T12:44:05.040843-04:00
custom:
Author: michelleark
Issue: "10229"

View File

@@ -0,0 +1,6 @@
kind: Features
body: Update data_test to accept arbitrary config options
time: 2024-05-31T15:08:16.431966-05:00
custom:
Author: McKnight-42
Issue: "10197"

View File

@@ -0,0 +1,6 @@
kind: Features
body: add pre_model and post_model hook calls to data and unit tests to be able to provide extra config options
time: 2024-06-06T11:23:34.758675-05:00
custom:
Author: McKnight-42
Issue: "10198"

View File

@@ -0,0 +1,6 @@
kind: Features
body: add --empty value to jinja context as flags.EMPTY
time: 2024-06-17T10:39:48.275801-04:00
custom:
Author: michelleark
Issue: "10317"

View File

@@ -0,0 +1,6 @@
kind: Features
body: Warning message for snapshot timestamp data types
time: 2024-06-21T14:16:35.717637-04:00
custom:
Author: gshank
Issue: "10234"

View File

@@ -0,0 +1,6 @@
kind: Features
body: Support cumulative_type_params & sub-daily granularities in semantic manifest.
time: 2024-06-25T09:51:07.983248-07:00
custom:
Author: courtneyholcomb
Issue: "10360"

View File

@@ -0,0 +1,6 @@
kind: Features
body: Add time_granularity to metric spec.
time: 2024-06-27T16:29:53.500917-07:00
custom:
Author: courtneyholcomb
Issue: "10376"

View File

@@ -0,0 +1,6 @@
kind: Features
body: Support standard schema/database fields for snapshots
time: 2024-07-12T21:45:46.06011-04:00
custom:
Author: gshank
Issue: "10301"

View File

@@ -0,0 +1,6 @@
kind: Features
body: Support ref and source in foreign key constraint expressions, bump dbt-common minimum to 1.6
time: 2024-07-19T16:18:41.434278-04:00
custom:
Author: michelleark
Issue: "8062"

View File

@@ -0,0 +1,6 @@
kind: Features
body: Support new semantic layer time spine configs to enable sub-daily granularity.
time: 2024-07-22T20:22:38.258249-07:00
custom:
Author: courtneyholcomb
Issue: "10475"

View File

@@ -0,0 +1,6 @@
kind: Features
body: Add support for behavior flags
time: 2024-08-29T13:53:20.16122-04:00
custom:
Author: mikealfare
Issue: "10618"

View File

@@ -1,6 +0,0 @@
kind: Features
body: Add new hard_deletes="new_record" mode for snapshots.
time: 2024-11-04T12:00:53.95191-05:00
custom:
Author: peterallenwebb
Issue: "10235"

View File

@@ -1,6 +0,0 @@
kind: Features
body: Add `batch` context object to model jinja context
time: 2024-11-21T12:56:30.715473-06:00
custom:
Author: QMalcolm
Issue: "11025"

View File

@@ -0,0 +1,6 @@
kind: Fixes
body: Remove unused check_new method
time: 2023-06-01T20:41:57.556342+02:00
custom:
Author: kevinneville
Issue: "7586"

View File

@@ -0,0 +1,6 @@
kind: Fixes
body: Convert "Skipping model due to fail_fast" message to DEBUG level
time: 2024-01-13T07:36:15.836294-00:00
custom:
Author: scottgigante,nevdelap
Issue: "8774"

View File

@@ -0,0 +1,7 @@
kind: Fixes
body: 'Restore previous behavior for --favor-state: only favor defer_relation if not
selected in current command"'
time: 2024-05-08T15:11:27.510912+02:00
custom:
Author: jtcohen6
Issue: "10107"

View File

@@ -0,0 +1,6 @@
kind: Fixes
body: Unit test fixture (csv) returns null for empty value
time: 2024-05-09T09:14:11.772709-04:00
custom:
Author: michelleark
Issue: "9881"

View File

@@ -0,0 +1,7 @@
kind: Fixes
body: Fix json format log and --quiet for ls and jinja print by converting print call
to fire events
time: 2024-05-16T15:39:13.896723-07:00
custom:
Author: ChenyuLInx
Issue: "8756"

View File

@@ -0,0 +1,6 @@
kind: Fixes
body: Add resource type to saved_query
time: 2024-05-16T22:35:10.287514-07:00
custom:
Author: ChenyuLInx
Issue: "10168"

View File

@@ -0,0 +1,7 @@
kind: Fixes
body: 'Fix: Order-insensitive unit test equality assertion for expected/actual with
multiple nulls'
time: 2024-05-22T18:28:55.91733-04:00
custom:
Author: michelleark
Issue: "10167"

View File

@@ -0,0 +1,6 @@
kind: Fixes
body: Renaming or removing a contracted model should raise a BreakingChange warning/error
time: 2024-05-23T20:42:51.033946-04:00
custom:
Author: michelleark
Issue: "10116"

View File

@@ -0,0 +1,6 @@
kind: Fixes
body: prefer disabled project nodes to external node
time: 2024-05-24T13:11:35.440443-04:00
custom:
Author: michelleark
Issue: "10224"

View File

@@ -0,0 +1,6 @@
kind: Fixes
body: Fix issues with selectors and inline nodes
time: 2024-06-05T11:16:52.187667-04:00
custom:
Author: gshank
Issue: 8943 9269

View File

@@ -0,0 +1,6 @@
kind: Fixes
body: Fix snapshot config to work in yaml files
time: 2024-06-07T13:46:48.383215-04:00
custom:
Author: gshank
Issue: "4000"

View File

@@ -0,0 +1,6 @@
kind: Fixes
body: Improve handling of error when loading schema file list
time: 2024-06-10T13:21:30.963371-04:00
custom:
Author: gshank
Issue: "10284"

View File

@@ -0,0 +1,6 @@
kind: Fixes
body: Use model alias for the CTE identifier generated during ephemeral materialization
time: 2024-06-10T20:05:22.510814008Z
custom:
Author: jeancochrane
Issue: "5273"

View File

@@ -0,0 +1,6 @@
kind: Fixes
body: Saved Query node fail during skip
time: 2024-06-12T12:42:56.329073-07:00
custom:
Author: ChenyuLInx
Issue: "10029"

View File

@@ -0,0 +1,6 @@
kind: Fixes
body: Implement state:modified for saved queries
time: 2024-06-12T15:21:39.851426-04:00
custom:
Author: gshank
Issue: "10294"

View File

@@ -0,0 +1,6 @@
kind: Fixes
body: DOn't warn on `unit_test` config paths that are properly used
time: 2024-06-13T18:31:17.486497-07:00
custom:
Author: QMalcolm
Issue: "10311"

View File

@@ -0,0 +1,6 @@
kind: Fixes
body: Fix setting `silence` of `warn_error_options` via `dbt_project.yaml` flags
time: 2024-06-24T17:17:29.464865-07:00
custom:
Author: QMalcolm
Issue: "10160"

View File

@@ -0,0 +1,7 @@
kind: Fixes
body: Attempt to provide test fixture tables with all values to set types correctly
for comparisong with source tables
time: 2024-06-25T17:17:37.514619-07:00
custom:
Author: versusfacit
Issue: "10365"

View File

@@ -0,0 +1,6 @@
kind: Fixes
body: Limit data_tests deprecation to root_project
time: 2024-06-27T15:44:48.579869-04:00
custom:
Author: gshank
Issue: "9835"

View File

@@ -0,0 +1,6 @@
kind: Fixes
body: CLI flags should take precedence over env var flags
time: 2024-07-09T17:24:40.918977-04:00
custom:
Author: gshank
Issue: "10304"

View File

@@ -0,0 +1,6 @@
kind: Fixes
body: Fix typing for artifact schemas
time: 2024-07-14T10:02:54.452099+09:00
custom:
Author: nakamichiworks
Issue: "10442"

View File

@@ -0,0 +1,6 @@
kind: Fixes
body: Fix over deletion of generated_metrics in partial parsing
time: 2024-07-16T13:37:03.49651-04:00
custom:
Author: gshank
Issue: "10450"

View File

@@ -0,0 +1,6 @@
kind: Fixes
body: Fix error constructing warn_error_options
time: 2024-07-16T17:14:27.837171-04:00
custom:
Author: gshank
Issue: "10452"

View File

@@ -0,0 +1,6 @@
kind: Fixes
body: Do not update varchar column definitions if a contract exists
time: 2024-07-28T22:14:21.67712-04:00
custom:
Author: gshank
Issue: "10362"

View File

@@ -0,0 +1,6 @@
kind: Fixes
body: fix all_constraints access, disabled node parsing of non-uniquely named resources
time: 2024-07-31T09:51:52.751135-04:00
custom:
Author: michelleark gshank
Issue: "10509"

View File

@@ -0,0 +1,6 @@
kind: Fixes
body: Propagate measure label when using create_metrics
time: 2024-08-06T17:21:10.265494-07:00
custom:
Author: aliceliu
Issue: "10536"

View File

@@ -0,0 +1,6 @@
kind: Fixes
body: respect --quiet and --warn-error-options for flag deprecations
time: 2024-08-06T19:48:43.399453-04:00
custom:
Author: michelleark
Issue: "10105"

View File

@@ -0,0 +1,6 @@
kind: Fixes
body: Fix state:modified check for exports
time: 2024-08-13T15:42:35.471685-07:00
custom:
Author: aliceliu
Issue: "10138"

View File

@@ -0,0 +1,6 @@
kind: Fixes
body: Filter out empty nodes after graph selection to support consistent selection of nodes that depend on upstream public models
time: 2024-08-16T14:08:07.426235-07:00
custom:
Author: jtcohen6
Issue: "8987"

View File

@@ -1,6 +0,0 @@
kind: Fixes
body: dbt retry does not respect --threads
time: 2024-08-22T12:21:32.358066+05:30
custom:
Author: donjin-master
Issue: "10584"

View File

@@ -0,0 +1,6 @@
kind: Fixes
body: Late render pre- and post-hooks configs in properties / schema YAML files
time: 2024-08-24T21:09:03.252733-06:00
custom:
Author: dbeatty10
Issue: "10603"

View File

@@ -0,0 +1,7 @@
kind: Fixes
body: Allow the use of env_var function in certain macros in which it was previously
unavailable.
time: 2024-08-29T10:57:01.160613-04:00
custom:
Author: peterallenwebb
Issue: "10609"

View File

@@ -0,0 +1,6 @@
kind: Fixes
body: 'Remove deprecation for tests: to data_tests: change'
time: 2024-09-05T18:02:48.086421-04:00
custom:
Author: gshank
Issue: "10564"

View File

@@ -1,6 +0,0 @@
kind: Fixes
body: Catch DbtRuntimeError for hooks
time: 2024-11-21T18:17:39.753235Z
custom:
Author: aranke
Issue: "11012"

View File

@@ -0,0 +1,6 @@
kind: Security
body: Explicitly bind to localhost in docs serve
time: 2024-05-22T09:45:40.748185-04:00
custom:
Author: ChenyuLInx michelleark
Issue: "10209"

View File

@@ -0,0 +1,6 @@
kind: Under the Hood
body: Clear error message for Private package in dbt-core
time: 2024-05-02T15:44:30.713097-07:00
custom:
Author: ChenyuLInx
Issue: "10083"

View File

@@ -0,0 +1,6 @@
kind: Under the Hood
body: Enable use of context in serialization
time: 2024-05-06T14:55:11.1812-04:00
custom:
Author: gshank
Issue: "10093"

View File

@@ -0,0 +1,6 @@
kind: Under the Hood
body: Make RSS high water mark measurement more accurate on Linux
time: 2024-05-19T15:59:46.700842315-04:00
custom:
Author: peterallenwebb
Issue: "10177"

View File

@@ -0,0 +1,6 @@
kind: Under the Hood
body: Enable record filtering by type.
time: 2024-05-29T10:28:14.547624-05:00
custom:
Author: emmyoop
Issue: "10240"

View File

@@ -0,0 +1,6 @@
kind: Under the Hood
body: Remove IntermediateSnapshotNode
time: 2024-06-18T14:06:52.618602-04:00
custom:
Author: gshank
Issue: "10326"

View File

@@ -0,0 +1,6 @@
kind: Under the Hood
body: Additional logging for skipped ephemeral models
time: 2024-07-01T13:17:50.827788-04:00
custom:
Author: gshank
Issue: "10389"

View File

@@ -0,0 +1,6 @@
kind: Under the Hood
body: bump black to 24.3.0
time: 2024-07-16T18:48:59.651834-04:00
custom:
Author: michelleark
Issue: "10454"

View File

@@ -0,0 +1,6 @@
kind: Under the Hood
body: generate protos with protoc version 5.26.1
time: 2024-07-16T20:57:03.332448-04:00
custom:
Author: michelleark
Issue: "10457"

View File

@@ -0,0 +1,6 @@
kind: Under the Hood
body: Move from minimal-snowplow-tracker fork back to snowplow-tracker
time: 2024-08-06T15:54:06.422444-04:00
custom:
Author: peterallenwebb
Issue: "8409"

View File

@@ -0,0 +1,7 @@
kind: Under the Hood
body: Add group info to RunResultError, RunResultFailure, RunResultWarning log lines
time: 2024-08-07T15:56:52.171199-05:00
custom:
Author: aranke
Issue: ""
JiraID: "364"

View File

@@ -0,0 +1,6 @@
kind: Under the Hood
body: Improve speed of tree traversal when finding children, increasing build speed for some selectors
time: 2024-08-09T13:02:34.759905-07:00
custom:
Author: ttusing
Issue: "10434"

View File

@@ -0,0 +1,6 @@
kind: Under the Hood
body: Add test for sources tables with quotes
time: 2024-08-21T09:55:16.038101-04:00
custom:
Author: gshank
Issue: "10582"

View File

@@ -0,0 +1,6 @@
kind: Under the Hood
body: Additional type hints for `core/dbt/version.py`
time: 2024-08-27T10:50:14.047859-05:00
custom:
Author: QMalcolm
Issue: "10612"

View File

@@ -0,0 +1,6 @@
kind: Under the Hood
body: Fix typing issues in core/dbt/contracts/sql.py
time: 2024-08-27T11:31:23.749912-05:00
custom:
Author: QMalcolm
Issue: "10614"

View File

@@ -0,0 +1,6 @@
kind: Under the Hood
body: Fix type errors in `dbt/core/task/clean.py`
time: 2024-08-27T11:48:10.438173-05:00
custom:
Author: QMalcolm
Issue: "10616"

View File

@@ -5,15 +5,6 @@ runs:
steps: steps:
- shell: bash - shell: bash
run: | run: |
sudo apt-get --purge remove postgresql postgresql-* sudo systemctl start postgresql.service
sudo apt update -y
sudo apt install gnupg2 wget vim -y
sudo sh -c 'echo "deb https://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list'
curl -fsSL https://www.postgresql.org/media/keys/ACCC4CF8.asc|sudo gpg --dearmor -o /etc/apt/trusted.gpg.d/postgresql.gpg
sudo apt update -y
sudo apt install postgresql-16
sudo apt-get -y install postgresql postgresql-contrib
sudo systemctl start postgresql
sudo systemctl enable postgresql
pg_isready pg_isready
sudo -u postgres bash ${{ github.action_path }}/setup_db.sh sudo -u postgres bash ${{ github.action_path }}/setup_db.sh

View File

@@ -5,9 +5,7 @@ runs:
steps: steps:
- shell: bash - shell: bash
run: | run: |
brew install postgresql@16 brew services start postgresql
brew link postgresql@16 --force
brew services start postgresql@16
echo "Check PostgreSQL service is running" echo "Check PostgreSQL service is running"
i=10 i=10
COMMAND='pg_isready' COMMAND='pg_isready'

View File

@@ -5,22 +5,8 @@ runs:
steps: steps:
- shell: pwsh - shell: pwsh
run: | run: |
Write-Host -Object "Installing PostgreSQL 16 as windows service..." $pgService = Get-Service -Name postgresql*
$installerArgs = @("--install_runtimes 0", "--superpassword root", "--enable_acledit 1", "--unattendedmodeui none", "--mode unattended")
$filePath = Invoke-DownloadWithRetry -Url "https://get.enterprisedb.com/postgresql/postgresql-16.1-1-windows-x64.exe" -Path "$env:PGROOT/postgresql-16.1-1-windows-x64.exe"
Start-Process -FilePath $filePath -ArgumentList $installerArgs -Wait -PassThru
Write-Host -Object "Validating PostgreSQL 16 Install..."
Get-Service -Name postgresql*
$pgReady = Start-Process -FilePath "$env:PGBIN\pg_isready" -Wait -PassThru
$exitCode = $pgReady.ExitCode
if ($exitCode -ne 0) {
Write-Host -Object "PostgreSQL is not ready. Exitcode: $exitCode"
exit $exitCode
}
Write-Host -Object "Starting PostgreSQL 16 Service..."
$pgService = Get-Service -Name postgresql-x64-16
Set-Service -InputObject $pgService -Status running -StartupType automatic Set-Service -InputObject $pgService -Status running -StartupType automatic
Start-Process -FilePath "$env:PGBIN\pg_isready" -Wait -PassThru
$env:Path += ";$env:PGBIN" $env:Path += ";$env:PGBIN"
bash ${{ github.action_path }}/setup_db.sh bash ${{ github.action_path }}/setup_db.sh

View File

@@ -36,6 +36,6 @@ jobs:
uses: dbt-labs/actions/.github/workflows/open-issue-in-repo.yml@main uses: dbt-labs/actions/.github/workflows/open-issue-in-repo.yml@main
with: with:
issue_repository: "dbt-labs/docs.getdbt.com" issue_repository: "dbt-labs/docs.getdbt.com"
issue_title: "[Core] Docs Changes Needed from ${{ github.event.repository.name }} Issue #${{ github.event.issue.number }}" issue_title: "Docs Changes Needed from ${{ github.event.repository.name }} Issue #${{ github.event.issue.number }}"
issue_body: "At a minimum, update body to include a link to the page on docs.getdbt.com requiring updates and what part(s) of the page you would like to see updated.\n Originating from this issue: https://github.com/dbt-labs/dbt-core/issues/${{ github.event.issue.number }}" issue_body: "At a minimum, update body to include a link to the page on docs.getdbt.com requiring updates and what part(s) of the page you would like to see updated."
secrets: inherit secrets: inherit

View File

@@ -52,14 +52,13 @@ jobs:
- name: Set up Python - name: Set up Python
uses: actions/setup-python@v5 uses: actions/setup-python@v5
with: with:
python-version: '3.9' python-version: '3.8'
- name: Install python dependencies - name: Install python dependencies
run: | run: |
python -m pip install --user --upgrade pip python -m pip install --user --upgrade pip
python -m pip --version python -m pip --version
make dev make dev
make dev_req
mypy --version mypy --version
dbt --version dbt --version
@@ -75,7 +74,7 @@ jobs:
strategy: strategy:
fail-fast: false fail-fast: false
matrix: matrix:
python-version: [ "3.9", "3.10", "3.11", "3.12" ] python-version: [ "3.8", "3.9", "3.10", "3.11", "3.12" ]
env: env:
TOXENV: "unit" TOXENV: "unit"
@@ -140,7 +139,7 @@ jobs:
- name: generate include - name: generate include
id: generate-include id: generate-include
run: | run: |
INCLUDE=('"python-version":"3.9","os":"windows-latest"' '"python-version":"3.9","os":"macos-14"' ) INCLUDE=('"python-version":"3.8","os":"windows-latest"' '"python-version":"3.8","os":"macos-12"' )
INCLUDE_GROUPS="[" INCLUDE_GROUPS="["
for include in ${INCLUDE[@]}; do for include in ${INCLUDE[@]}; do
for group in $(seq 1 ${{ env.PYTHON_INTEGRATION_TEST_WORKERS }}); do for group in $(seq 1 ${{ env.PYTHON_INTEGRATION_TEST_WORKERS }}); do
@@ -162,10 +161,22 @@ jobs:
strategy: strategy:
fail-fast: false fail-fast: false
matrix: matrix:
python-version: [ "3.9", "3.10", "3.11", "3.12" ] python-version: [ "3.8", "3.9", "3.10", "3.11", "3.12" ]
os: [ubuntu-20.04] os: [ubuntu-20.04]
split-group: ${{ fromJson(needs.integration-metadata.outputs.split-groups) }} split-group: ${{ fromJson(needs.integration-metadata.outputs.split-groups) }}
include: ${{ fromJson(needs.integration-metadata.outputs.include) }} include: ${{ fromJson(needs.integration-metadata.outputs.include) }}
services:
postgres:
image: postgres
env:
POSTGRES_PASSWORD: postgres
options: >-
--health-cmd pg_isready
--health-interval 10s
--health-timeout 5s
--health-retries 5
ports:
- 5432:5432
env: env:
TOXENV: integration TOXENV: integration
DBT_INVOCATION_ENV: github-actions DBT_INVOCATION_ENV: github-actions
@@ -190,14 +201,32 @@ jobs:
- name: Set up postgres (linux) - name: Set up postgres (linux)
if: runner.os == 'Linux' if: runner.os == 'Linux'
uses: ./.github/actions/setup-postgres-linux uses: ./.github/actions/setup-postgres-linux
env:
PGHOST: localhost
PGPORT: 5432
PGUSER: postgres
PGPASSWORD: postgres
PGDATABASE: postgres
- name: Set up postgres (macos) - name: Set up postgres (macos)
if: runner.os == 'macOS' if: runner.os == 'macOS'
uses: ./.github/actions/setup-postgres-macos uses: ./.github/actions/setup-postgres-macos
env:
PGHOST: localhost
PGPORT: 5432
PGUSER: postgres
PGPASSWORD: postgres
PGDATABASE: postgres
- name: Set up postgres (windows) - name: Set up postgres (windows)
if: runner.os == 'Windows' if: runner.os == 'Windows'
uses: ./.github/actions/setup-postgres-windows uses: ./.github/actions/setup-postgres-windows
env:
PGHOST: localhost
PGPORT: 5432
PGUSER: postgres
PGPASSWORD: postgres
PGDATABASE: postgres
- name: Install python tools - name: Install python tools
run: | run: |
@@ -264,7 +293,7 @@ jobs:
- name: Set up Python - name: Set up Python
uses: actions/setup-python@v5 uses: actions/setup-python@v5
with: with:
python-version: '3.9' python-version: '3.8'
- name: Install python dependencies - name: Install python dependencies
run: | run: |

View File

@@ -150,7 +150,7 @@ jobs:
- name: Setup Python - name: Setup Python
uses: actions/setup-python@v5 uses: actions/setup-python@v5
with: with:
python-version: "3.9" python-version: "3.8"
- name: Install dbt - name: Install dbt
run: pip install dbt-postgres==${{ needs.set-variables.outputs.release_id }} run: pip install dbt-postgres==${{ needs.set-variables.outputs.release_id }}

View File

@@ -30,14 +30,14 @@ env:
jobs: jobs:
checking-schemas: checking-schemas:
name: "Post-merge schema changes required" name: "Checking schemas"
runs-on: ubuntu-latest runs-on: ubuntu-latest
steps: steps:
- name: Set up Python - name: Set up Python
uses: actions/setup-python@v5 uses: actions/setup-python@v5
with: with:
python-version: 3.9 python-version: 3.8
- name: Checkout dbt repo - name: Checkout dbt repo
uses: actions/checkout@v4 uses: actions/checkout@v4

View File

@@ -76,7 +76,7 @@ jobs:
- name: Setup Python - name: Setup Python
uses: actions/setup-python@v5 uses: actions/setup-python@v5
with: with:
python-version: "3.9" python-version: "3.8"
- name: Install python dependencies - name: Install python dependencies
run: | run: |

View File

@@ -27,6 +27,7 @@ on:
description: 'Version of Python to Test Against' description: 'Version of Python to Test Against'
type: choice type: choice
options: options:
- '3.8'
- '3.9' - '3.9'
- '3.10' - '3.10'
- '3.11' - '3.11'
@@ -35,7 +36,7 @@ on:
type: choice type: choice
options: options:
- 'ubuntu-latest' - 'ubuntu-latest'
- 'macos-14' - 'macos-12'
- 'windows-latest' - 'windows-latest'
num_runs_per_batch: num_runs_per_batch:
description: 'Max number of times to run the test per batch. We always run 10 batches.' description: 'Max number of times to run the test per batch. We always run 10 batches.'
@@ -100,7 +101,7 @@ jobs:
# mac and windows don't use make due to limitations with docker with those runners in GitHub # mac and windows don't use make due to limitations with docker with those runners in GitHub
- name: "Set up postgres (macos)" - name: "Set up postgres (macos)"
if: inputs.os == 'macos-14' if: inputs.os == 'macos-12'
uses: ./.github/actions/setup-postgres-macos uses: ./.github/actions/setup-postgres-macos
- name: "Set up postgres (windows)" - name: "Set up postgres (windows)"

6
.gitignore vendored
View File

@@ -57,9 +57,6 @@ test.env
makefile.test.env makefile.test.env
*.pytest_cache/ *.pytest_cache/
# Unit test artifacts
index.html
# Translations # Translations
*.mo *.mo
@@ -108,6 +105,3 @@ venv/
# poetry # poetry
poetry.lock poetry.lock
# asdf
.tool-versions

View File

@@ -3,7 +3,7 @@
exclude: ^(core/dbt/docs/build/|core/dbt/common/events/types_pb2.py|core/dbt/events/core_types_pb2.py|core/dbt/adapters/events/adapter_types_pb2.py) exclude: ^(core/dbt/docs/build/|core/dbt/common/events/types_pb2.py|core/dbt/events/core_types_pb2.py|core/dbt/adapters/events/adapter_types_pb2.py)
# Force all unspecified python hooks to run python 3.9 # Force all unspecified python hooks to run python 3.8
default_language_version: default_language_version:
python: python3 python: python3

View File

@@ -10,7 +10,6 @@
For information on prior major and minor releases, see their changelogs: For information on prior major and minor releases, see their changelogs:
* [1.9](https://github.com/dbt-labs/dbt-core/blob/1.9.latest/CHANGELOG.md)
* [1.8](https://github.com/dbt-labs/dbt-core/blob/1.8.latest/CHANGELOG.md) * [1.8](https://github.com/dbt-labs/dbt-core/blob/1.8.latest/CHANGELOG.md)
* [1.7](https://github.com/dbt-labs/dbt-core/blob/1.7.latest/CHANGELOG.md) * [1.7](https://github.com/dbt-labs/dbt-core/blob/1.7.latest/CHANGELOG.md)
* [1.6](https://github.com/dbt-labs/dbt-core/blob/1.6.latest/CHANGELOG.md) * [1.6](https://github.com/dbt-labs/dbt-core/blob/1.6.latest/CHANGELOG.md)

View File

@@ -33,6 +33,9 @@ RUN apt-get update \
python-is-python3 \ python-is-python3 \
python-dev-is-python3 \ python-dev-is-python3 \
python3-pip \ python3-pip \
python3.8 \
python3.8-dev \
python3.8-venv \
python3.9 \ python3.9 \
python3.9-dev \ python3.9-dev \
python3.9-venv \ python3.9-venv \

View File

@@ -46,9 +46,9 @@ These types of minor, non-breaking changes are tested by [tests/unit/artifacts/t
#### Updating [schemas.getdbt.com](https://schemas.getdbt.com) #### Updating [schemas.getdbt.com](https://schemas.getdbt.com)
Non-breaking changes to artifact schemas require an update to the corresponding jsonschemas published to [schemas.getdbt.com](https://schemas.getdbt.com), which are defined in https://github.com/dbt-labs/schemas.getdbt.com. To do so: Non-breaking changes to artifact schemas require an update to the corresponding jsonschemas published to [schemas.getdbt.com](https://schemas.getdbt.com), which are defined in https://github.com/dbt-labs/schemas.getdbt.com. To do so:
Note this must be done AFTER the core pull request is merged, otherwise we may end up with unresolvable conflicts and schemas that are invalid prior to base pull request merge. You may create the schemas.getdbt.com pull request prior to merging the base pull request, but do not merge until afterward.
1. Create a PR in https://github.com/dbt-labs/schemas.getdbt.com which reflects the schema changes to the artifact. The schema can be updated in-place for non-breaking changes. Example PR: https://github.com/dbt-labs/schemas.getdbt.com/pull/39 1. Create a PR in https://github.com/dbt-labs/schemas.getdbt.com which reflects the schema changes to the artifact. The schema can be updated in-place for non-breaking changes. Example PR: https://github.com/dbt-labs/schemas.getdbt.com/pull/39
2. Merge the https://github.com/dbt-labs/schemas.getdbt.com PR 2. Merge the https://github.com/dbt-labs/schemas.getdbt.com PR
3. Observe the `Artifact Schema Check` CI check pass on the `dbt-core` PR that updates the artifact schemas, and merge the `dbt-core` PR!
Note: Although `jsonschema` validation using the schemas in [schemas.getdbt.com](https://schemas.getdbt.com) is not encouraged or formally supported, `jsonschema` validation should still continue to work once the schemas are updated because they are forward-compatible and can therefore be used to validate previous minor versions of the schema. Note: Although `jsonschema` validation using the schemas in [schemas.getdbt.com](https://schemas.getdbt.com) is not encouraged or formally supported, `jsonschema` validation should still continue to work once the schemas are updated because they are forward-compatible and can therefore be used to validate previous minor versions of the schema.

View File

@@ -68,10 +68,3 @@ class TimePeriod(StrEnum):
def plural(self) -> str: def plural(self) -> str:
return str(self) + "s" return str(self) + "s"
class BatchSize(StrEnum):
hour = "hour"
day = "day"
month = "month"
year = "year"

View File

@@ -194,7 +194,6 @@ class ParsedResource(ParsedResourceMandatory):
unrendered_config: Dict[str, Any] = field(default_factory=dict) unrendered_config: Dict[str, Any] = field(default_factory=dict)
created_at: float = field(default_factory=lambda: time.time()) created_at: float = field(default_factory=lambda: time.time())
config_call_dict: Dict[str, Any] = field(default_factory=dict) config_call_dict: Dict[str, Any] = field(default_factory=dict)
unrendered_config_call_dict: Dict[str, Any] = field(default_factory=dict)
relation_name: Optional[str] = None relation_name: Optional[str] = None
raw_code: str = "" raw_code: str = ""
@@ -202,8 +201,6 @@ class ParsedResource(ParsedResourceMandatory):
dct = super().__post_serialize__(dct, context) dct = super().__post_serialize__(dct, context)
if context and context.get("artifact") and "config_call_dict" in dct: if context and context.get("artifact") and "config_call_dict" in dct:
del dct["config_call_dict"] del dct["config_call_dict"]
if context and context.get("artifact") and "unrendered_config_call_dict" in dct:
del dct["unrendered_config_call_dict"]
return dct return dct

View File

@@ -80,9 +80,6 @@ class NodeConfig(NodeAndTestConfig):
# 'mergebehavior' dictionary # 'mergebehavior' dictionary
materialized: str = "view" materialized: str = "view"
incremental_strategy: Optional[str] = None incremental_strategy: Optional[str] = None
batch_size: Any = None
lookback: Any = 1
begin: Any = None
persist_docs: Dict[str, Any] = field(default_factory=dict) persist_docs: Dict[str, Any] = field(default_factory=dict)
post_hook: List[Hook] = field( post_hook: List[Hook] = field(
default_factory=list, default_factory=list,
@@ -125,8 +122,6 @@ class NodeConfig(NodeAndTestConfig):
default_factory=ContractConfig, default_factory=ContractConfig,
metadata=MergeBehavior.Update.meta(), metadata=MergeBehavior.Update.meta(),
) )
event_time: Any = None
concurrent_batches: Any = None
def __post_init__(self): def __post_init__(self):
# we validate that node_color has a suitable value to prevent dbt-docs from crashing # we validate that node_color has a suitable value to prevent dbt-docs from crashing

View File

@@ -22,16 +22,9 @@ class ModelConfig(NodeConfig):
) )
@dataclass
class CustomGranularity(dbtClassMixin):
name: str
column_name: Optional[str] = None
@dataclass @dataclass
class TimeSpine(dbtClassMixin): class TimeSpine(dbtClassMixin):
standard_granularity_column: str standard_granularity_column: str
custom_granularities: List[CustomGranularity] = field(default_factory=list)
@dataclass @dataclass

View File

@@ -44,8 +44,6 @@ class QueryParams(dbtClassMixin):
metrics: List[str] metrics: List[str]
group_by: List[str] group_by: List[str]
where: Optional[WhereFilterIntersection] where: Optional[WhereFilterIntersection]
order_by: List[str] = field(default_factory=list)
limit: Optional[int] = None
@dataclass @dataclass

View File

@@ -1,45 +1,22 @@
from dataclasses import dataclass, field from dataclasses import dataclass
from typing import Dict, List, Literal, Optional, Union from typing import Dict, List, Literal, Optional, Union
from dbt.artifacts.resources.types import NodeType from dbt.artifacts.resources.types import NodeType
from dbt.artifacts.resources.v1.components import CompiledResource, DeferRelation from dbt.artifacts.resources.v1.components import CompiledResource, DeferRelation
from dbt.artifacts.resources.v1.config import NodeConfig from dbt.artifacts.resources.v1.config import NodeConfig
from dbt_common.dataclass_schema import ValidationError, dbtClassMixin from dbt_common.dataclass_schema import ValidationError
@dataclass
class SnapshotMetaColumnNames(dbtClassMixin):
dbt_valid_to: Optional[str] = None
dbt_valid_from: Optional[str] = None
dbt_scd_id: Optional[str] = None
dbt_updated_at: Optional[str] = None
dbt_is_deleted: Optional[str] = None
@dataclass @dataclass
class SnapshotConfig(NodeConfig): class SnapshotConfig(NodeConfig):
materialized: str = "snapshot" materialized: str = "snapshot"
strategy: Optional[str] = None strategy: Optional[str] = None
unique_key: Optional[Union[str, List[str]]] = None unique_key: Optional[str] = None
target_schema: Optional[str] = None target_schema: Optional[str] = None
target_database: Optional[str] = None target_database: Optional[str] = None
updated_at: Optional[str] = None updated_at: Optional[str] = None
# Not using Optional because of serialization issues with a Union of str and List[str] # Not using Optional because of serialization issues with a Union of str and List[str]
check_cols: Union[str, List[str], None] = None check_cols: Union[str, List[str], None] = None
snapshot_meta_column_names: SnapshotMetaColumnNames = field(
default_factory=SnapshotMetaColumnNames
)
dbt_valid_to_current: Optional[str] = None
@property
def snapshot_table_column_names(self):
return {
"dbt_valid_from": self.snapshot_meta_column_names.dbt_valid_from or "dbt_valid_from",
"dbt_valid_to": self.snapshot_meta_column_names.dbt_valid_to or "dbt_valid_to",
"dbt_scd_id": self.snapshot_meta_column_names.dbt_scd_id or "dbt_scd_id",
"dbt_updated_at": self.snapshot_meta_column_names.dbt_updated_at or "dbt_updated_at",
"dbt_is_deleted": self.snapshot_meta_column_names.dbt_is_deleted or "dbt_is_deleted",
}
def final_validate(self): def final_validate(self):
if not self.strategy or not self.unique_key: if not self.strategy or not self.unique_key:

View File

@@ -19,7 +19,6 @@ from dbt_common.exceptions import CompilationError
@dataclass @dataclass
class SourceConfig(BaseConfig): class SourceConfig(BaseConfig):
enabled: bool = True enabled: bool = True
event_time: Any = None
@dataclass @dataclass
@@ -71,5 +70,3 @@ class SourceDefinition(ParsedSourceMandatory):
unrendered_config: Dict[str, Any] = field(default_factory=dict) unrendered_config: Dict[str, Any] = field(default_factory=dict)
relation_name: Optional[str] = None relation_name: Optional[str] = None
created_at: float = field(default_factory=lambda: time.time()) created_at: float = field(default_factory=lambda: time.time())
unrendered_database: Optional[str] = None
unrendered_schema: Optional[str] = None

View File

@@ -20,7 +20,6 @@ class UnitTestConfig(BaseConfig):
default_factory=dict, default_factory=dict,
metadata=MergeBehavior.Update.meta(), metadata=MergeBehavior.Update.meta(),
) )
enabled: bool = True
class UnitTestFormat(StrEnum): class UnitTestFormat(StrEnum):

View File

@@ -1,24 +0,0 @@
from __future__ import annotations
from dataclasses import dataclass, field
from datetime import datetime
from typing import List, Tuple
from dbt_common.dataclass_schema import dbtClassMixin
BatchType = Tuple[datetime, datetime]
@dataclass
class BatchResults(dbtClassMixin):
successful: List[BatchType] = field(default_factory=list)
failed: List[BatchType] = field(default_factory=list)
def __add__(self, other: BatchResults) -> BatchResults:
return BatchResults(
successful=self.successful + other.successful,
failed=self.failed + other.failed,
)
def __len__(self):
return len(self.successful) + len(self.failed)

View File

@@ -10,12 +10,6 @@ from dbt_common.utils import cast_to_int, cast_to_str
@dataclass @dataclass
class TimingInfo(dbtClassMixin): class TimingInfo(dbtClassMixin):
"""
Represents a step in the execution of a node.
`name` should be one of: compile, execute, or other
Do not call directly, use `collect_timing_info` instead.
"""
name: str name: str
started_at: Optional[datetime] = None started_at: Optional[datetime] = None
completed_at: Optional[datetime] = None completed_at: Optional[datetime] = None
@@ -27,7 +21,7 @@ class TimingInfo(dbtClassMixin):
self.completed_at = datetime.utcnow() self.completed_at = datetime.utcnow()
def to_msg_dict(self): def to_msg_dict(self):
msg_dict = {"name": str(self.name)} msg_dict = {"name": self.name}
if self.started_at: if self.started_at:
msg_dict["started_at"] = datetime_to_json_string(self.started_at) msg_dict["started_at"] = datetime_to_json_string(self.started_at)
if self.completed_at: if self.completed_at:
@@ -61,7 +55,6 @@ class NodeStatus(StrEnum):
Fail = "fail" Fail = "fail"
Warn = "warn" Warn = "warn"
Skipped = "skipped" Skipped = "skipped"
PartialSuccess = "partial success"
Pass = "pass" Pass = "pass"
RuntimeErr = "runtime error" RuntimeErr = "runtime error"
@@ -70,7 +63,6 @@ class RunStatus(StrEnum):
Success = NodeStatus.Success Success = NodeStatus.Success
Error = NodeStatus.Error Error = NodeStatus.Error
Skipped = NodeStatus.Skipped Skipped = NodeStatus.Skipped
PartialSuccess = NodeStatus.PartialSuccess
class TestStatus(StrEnum): class TestStatus(StrEnum):

View File

@@ -1,5 +1,3 @@
from __future__ import annotations
import copy import copy
import threading import threading
from dataclasses import dataclass, field from dataclasses import dataclass, field
@@ -19,7 +17,6 @@ from dbt.artifacts.schemas.base import (
get_artifact_schema_version, get_artifact_schema_version,
schema_version, schema_version,
) )
from dbt.artifacts.schemas.batch_results import BatchResults
from dbt.artifacts.schemas.results import ( from dbt.artifacts.schemas.results import (
BaseResult, BaseResult,
ExecutionResult, ExecutionResult,
@@ -37,7 +34,6 @@ class RunResult(NodeResult):
agate_table: Optional["agate.Table"] = field( agate_table: Optional["agate.Table"] = field(
default=None, metadata={"serialize": lambda x: None, "deserialize": lambda x: None} default=None, metadata={"serialize": lambda x: None, "deserialize": lambda x: None}
) )
batch_results: Optional[BatchResults] = None
@property @property
def skipped(self): def skipped(self):
@@ -55,7 +51,6 @@ class RunResult(NodeResult):
node=node, node=node,
adapter_response={}, adapter_response={},
failures=None, failures=None,
batch_results=None,
) )
@@ -72,7 +67,6 @@ class RunResultOutput(BaseResult):
compiled: Optional[bool] compiled: Optional[bool]
compiled_code: Optional[str] compiled_code: Optional[str]
relation_name: Optional[str] relation_name: Optional[str]
batch_results: Optional[BatchResults] = None
def process_run_result(result: RunResult) -> RunResultOutput: def process_run_result(result: RunResult) -> RunResultOutput:
@@ -88,7 +82,6 @@ def process_run_result(result: RunResult) -> RunResultOutput:
message=result.message, message=result.message,
adapter_response=result.adapter_response, adapter_response=result.adapter_response,
failures=result.failures, failures=result.failures,
batch_results=result.batch_results,
compiled=result.node.compiled if compiled else None, # type:ignore compiled=result.node.compiled if compiled else None, # type:ignore
compiled_code=result.node.compiled_code if compiled else None, # type:ignore compiled_code=result.node.compiled_code if compiled else None, # type:ignore
relation_name=result.node.relation_name if compiled else None, # type:ignore relation_name=result.node.relation_name if compiled else None, # type:ignore

View File

@@ -1,7 +1,6 @@
import os import os
import sys import sys
from dataclasses import dataclass from dataclasses import dataclass
from datetime import datetime
from importlib import import_module from importlib import import_module
from pathlib import Path from pathlib import Path
from pprint import pformat as pf from pprint import pformat as pf
@@ -38,7 +37,6 @@ FLAGS_DEFAULTS = {
"STRICT_MODE": False, "STRICT_MODE": False,
"STORE_FAILURES": False, "STORE_FAILURES": False,
"INTROSPECT": True, "INTROSPECT": True,
"STATE_MODIFIED_COMPARE_VARS": False,
} }
DEPRECATED_PARAMS = { DEPRECATED_PARAMS = {
@@ -314,9 +312,6 @@ class Flags:
self._assert_mutually_exclusive(params_assigned_from_default, ["SELECT", "INLINE"]) self._assert_mutually_exclusive(params_assigned_from_default, ["SELECT", "INLINE"])
self._assert_mutually_exclusive(params_assigned_from_default, ["SELECTOR", "INLINE"]) self._assert_mutually_exclusive(params_assigned_from_default, ["SELECTOR", "INLINE"])
# Check event_time configs for validity
self._validate_event_time_configs()
# Support lower cased access for legacy code. # Support lower cased access for legacy code.
params = set( params = set(
x for x in dir(self) if not callable(getattr(self, x)) and not x.startswith("__") x for x in dir(self) if not callable(getattr(self, x)) and not x.startswith("__")
@@ -353,36 +348,6 @@ class Flags:
elif flag_set_by_user: elif flag_set_by_user:
set_flag = flag set_flag = flag
def _validate_event_time_configs(self) -> None:
event_time_start: datetime = (
getattr(self, "EVENT_TIME_START") if hasattr(self, "EVENT_TIME_START") else None
)
event_time_end: datetime = (
getattr(self, "EVENT_TIME_END") if hasattr(self, "EVENT_TIME_END") else None
)
# only do validations if at least one of `event_time_start` or `event_time_end` are specified
if event_time_start is not None or event_time_end is not None:
# These `ifs`, combined with the parent `if` make it so that `event_time_start` and
# `event_time_end` are mutually required
if event_time_start is None:
raise DbtUsageException(
"The flag `--event-time-end` was specified, but `--event-time-start` was not. "
"When specifying `--event-time-end`, `--event-time-start` must also be present."
)
if event_time_end is None:
raise DbtUsageException(
"The flag `--event-time-start` was specified, but `--event-time-end` was not. "
"When specifying `--event-time-start`, `--event-time-end` must also be present."
)
# This `if` just is a sanity check that `event_time_start` is before `event_time_end`
if event_time_start >= event_time_end:
raise DbtUsageException(
"Value for `--event-time-start` must be less than `--event-time-end`"
)
def fire_deprecations(self): def fire_deprecations(self):
"""Fires events for deprecated env_var usage.""" """Fires events for deprecated env_var usage."""
[dep_fn() for dep_fn in self.deprecated_env_var_warnings] [dep_fn() for dep_fn in self.deprecated_env_var_warnings]

Some files were not shown because too many files have changed in this diff Show More