mirror of
https://github.com/dbt-labs/dbt-core
synced 2025-12-19 22:01:28 +00:00
Compare commits
9 Commits
enable-pos
...
empty-seed
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
21574bb867 | ||
|
|
4df6b3ffa4 | ||
|
|
4fb7ec0b88 | ||
|
|
728e5ba9b0 | ||
|
|
3f297cb4e3 | ||
|
|
3420afdd93 | ||
|
|
e34b881a5f | ||
|
|
1a8f190124 | ||
|
|
446d2671e0 |
6
.changes/unreleased/Features-20251215-174308.yaml
Normal file
6
.changes/unreleased/Features-20251215-174308.yaml
Normal file
@@ -0,0 +1,6 @@
|
||||
kind: Features
|
||||
body: Support --empty flag for dbt seed
|
||||
time: 2025-12-15T17:43:08.807815-05:00
|
||||
custom:
|
||||
Author: michelleark
|
||||
Issue: "8981"
|
||||
6
.changes/unreleased/Under the Hood-20251215-155046.yaml
Normal file
6
.changes/unreleased/Under the Hood-20251215-155046.yaml
Normal file
@@ -0,0 +1,6 @@
|
||||
kind: Under the Hood
|
||||
body: Bump lower bound for dbt-common to 1.37.2
|
||||
time: 2025-12-15T15:50:46.857793-05:00
|
||||
custom:
|
||||
Author: michelleark
|
||||
Issue: "12284"
|
||||
108
.github/dbt-postgres-testing.yml
vendored
108
.github/dbt-postgres-testing.yml
vendored
@@ -108,62 +108,62 @@ jobs:
|
||||
echo "dbt-postgres-ref=${{ steps.core-ref.outputs.ref }}"
|
||||
echo "dbt-core-ref=${{ steps.common-ref.outputs.ref }}"
|
||||
|
||||
integration-tests-postgres:
|
||||
name: "dbt-postgres integration tests"
|
||||
needs: [job-prep]
|
||||
runs-on: ubuntu-latest
|
||||
defaults:
|
||||
run:
|
||||
working-directory: "./dbt-postgres"
|
||||
environment:
|
||||
name: "dbt-postgres"
|
||||
env:
|
||||
POSTGRES_TEST_HOST: ${{ vars.POSTGRES_TEST_HOST }}
|
||||
POSTGRES_TEST_PORT: ${{ vars.POSTGRES_TEST_PORT }}
|
||||
POSTGRES_TEST_USER: ${{ vars.POSTGRES_TEST_USER }}
|
||||
POSTGRES_TEST_PASS: ${{ secrets.POSTGRES_TEST_PASS }}
|
||||
POSTGRES_TEST_DATABASE: ${{ vars.POSTGRES_TEST_DATABASE }}
|
||||
POSTGRES_TEST_THREADS: ${{ vars.POSTGRES_TEST_THREADS }}
|
||||
services:
|
||||
postgres:
|
||||
image: postgres
|
||||
env:
|
||||
POSTGRES_PASSWORD: postgres
|
||||
options: >-
|
||||
--health-cmd pg_isready
|
||||
--health-interval 10s
|
||||
--health-timeout 5s
|
||||
--health-retries 5
|
||||
ports:
|
||||
- ${{ vars.POSTGRES_TEST_PORT }}:5432
|
||||
steps:
|
||||
- name: "Check out dbt-adapters@${{ needs.job-prep.outputs.dbt-postgres-ref }}"
|
||||
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # actions/checkout@v4
|
||||
with:
|
||||
repository: dbt-labs/dbt-adapters
|
||||
ref: ${{ needs.job-prep.outputs.dbt-postgres-ref }}
|
||||
# integration-tests-postgres:
|
||||
# name: "dbt-postgres integration tests"
|
||||
# needs: [job-prep]
|
||||
# runs-on: ubuntu-latest
|
||||
# defaults:
|
||||
# run:
|
||||
# working-directory: "./dbt-postgres"
|
||||
# environment:
|
||||
# name: "dbt-postgres"
|
||||
# env:
|
||||
# POSTGRES_TEST_HOST: ${{ vars.POSTGRES_TEST_HOST }}
|
||||
# POSTGRES_TEST_PORT: ${{ vars.POSTGRES_TEST_PORT }}
|
||||
# POSTGRES_TEST_USER: ${{ vars.POSTGRES_TEST_USER }}
|
||||
# POSTGRES_TEST_PASS: ${{ secrets.POSTGRES_TEST_PASS }}
|
||||
# POSTGRES_TEST_DATABASE: ${{ vars.POSTGRES_TEST_DATABASE }}
|
||||
# POSTGRES_TEST_THREADS: ${{ vars.POSTGRES_TEST_THREADS }}
|
||||
# services:
|
||||
# postgres:
|
||||
# image: postgres
|
||||
# env:
|
||||
# POSTGRES_PASSWORD: postgres
|
||||
# options: >-
|
||||
# --health-cmd pg_isready
|
||||
# --health-interval 10s
|
||||
# --health-timeout 5s
|
||||
# --health-retries 5
|
||||
# ports:
|
||||
# - ${{ vars.POSTGRES_TEST_PORT }}:5432
|
||||
# steps:
|
||||
# - name: "Check out dbt-adapters@${{ needs.job-prep.outputs.dbt-postgres-ref }}"
|
||||
# uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # actions/checkout@v4
|
||||
# with:
|
||||
# repository: dbt-labs/dbt-adapters
|
||||
# ref: ${{ needs.job-prep.outputs.dbt-postgres-ref }}
|
||||
|
||||
- name: "Set up Python"
|
||||
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # actions/setup-python@v5
|
||||
with:
|
||||
python-version: ${{ inputs.python-version }}
|
||||
# - name: "Set up Python"
|
||||
# uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # actions/setup-python@v5
|
||||
# with:
|
||||
# python-version: ${{ inputs.python-version }}
|
||||
|
||||
- name: "Set environment variables"
|
||||
run: |
|
||||
echo "HATCH_PYTHON=${{ inputs.python-version }}" >> $GITHUB_ENV
|
||||
echo "PIP_ONLY_BINARY=psycopg2-binary" >> $GITHUB_ENV
|
||||
# - name: "Set environment variables"
|
||||
# run: |
|
||||
# echo "HATCH_PYTHON=${{ inputs.python-version }}" >> $GITHUB_ENV
|
||||
# echo "PIP_ONLY_BINARY=psycopg2-binary" >> $GITHUB_ENV
|
||||
|
||||
- name: "Setup test database"
|
||||
run: psql -f ./scripts/setup_test_database.sql
|
||||
env:
|
||||
PGHOST: ${{ vars.POSTGRES_TEST_HOST }}
|
||||
PGPORT: ${{ vars.POSTGRES_TEST_PORT }}
|
||||
PGUSER: postgres
|
||||
PGPASSWORD: postgres
|
||||
PGDATABASE: postgres
|
||||
# - name: "Setup test database"
|
||||
# run: psql -f ./scripts/setup_test_database.sql
|
||||
# env:
|
||||
# PGHOST: ${{ vars.POSTGRES_TEST_HOST }}
|
||||
# PGPORT: ${{ vars.POSTGRES_TEST_PORT }}
|
||||
# PGUSER: postgres
|
||||
# PGPASSWORD: postgres
|
||||
# PGDATABASE: postgres
|
||||
|
||||
- name: "Install hatch"
|
||||
uses: pypa/hatch@257e27e51a6a5616ed08a39a408a21c35c9931bc # pypa/hatch@install
|
||||
# - name: "Install hatch"
|
||||
# uses: pypa/hatch@257e27e51a6a5616ed08a39a408a21c35c9931bc # pypa/hatch@install
|
||||
|
||||
- name: "Run integration tests"
|
||||
run: hatch run ${{ inputs.hatch-env }}:integration-tests
|
||||
# - name: "Run integration tests"
|
||||
# run: hatch run ${{ inputs.hatch-env }}:integration-tests
|
||||
|
||||
@@ -691,6 +691,7 @@ def run_operation(ctx, **kwargs):
|
||||
@cli.command("seed")
|
||||
@click.pass_context
|
||||
@global_flags
|
||||
@p.empty
|
||||
@p.exclude
|
||||
@p.full_refresh
|
||||
@p.profiles_dir
|
||||
|
||||
@@ -6,6 +6,9 @@ SECRET_PLACEHOLDER = "$$$DBT_SECRET_START$$${}$$$DBT_SECRET_END$$$"
|
||||
|
||||
MAXIMUM_SEED_SIZE = 1 * 1024 * 1024
|
||||
MAXIMUM_SEED_SIZE_NAME = "1MB"
|
||||
# Number of rows to load as agate table to obtain column types for empty seed table creation
|
||||
# Seed materializations themselves avoid loading the data to the warehouse
|
||||
EMPTY_SEED_SIZE = 5
|
||||
|
||||
PIN_PACKAGE_URL = (
|
||||
"https://docs.getdbt.com/docs/package-management#section-specifying-package-versions"
|
||||
|
||||
@@ -43,7 +43,7 @@ from dbt.clients.jinja import (
|
||||
)
|
||||
from dbt.clients.jinja_static import statically_parse_unrendered_config
|
||||
from dbt.config import IsFQNResource, Project, RuntimeConfig
|
||||
from dbt.constants import DEFAULT_ENV_PLACEHOLDER
|
||||
from dbt.constants import DEFAULT_ENV_PLACEHOLDER, EMPTY_SEED_SIZE
|
||||
from dbt.context.base import Var, contextmember, contextproperty
|
||||
from dbt.context.configured import FQNLookup
|
||||
from dbt.context.context_config import ContextConfig
|
||||
@@ -1269,6 +1269,8 @@ class ProviderContext(ManifestContext):
|
||||
delimiter = self.model.config.delimiter
|
||||
try:
|
||||
table = agate_helper.from_csv(path, text_columns=column_types, delimiter=delimiter)
|
||||
if getattr(self.config.args, "EMPTY", False):
|
||||
table = table.limit(EMPTY_SEED_SIZE) # type: ignore
|
||||
except ValueError as e:
|
||||
raise LoadAgateTableValueError(e, node=self.model)
|
||||
# this is used by some adapters
|
||||
|
||||
@@ -54,7 +54,7 @@ dependencies = [
|
||||
"dbt-extractor>=0.5.0,<=0.6",
|
||||
"dbt-semantic-interfaces>=0.9.0,<0.10",
|
||||
# Minor versions for these are expected to be backwards-compatible
|
||||
"dbt-common>=1.37.0,<2.0",
|
||||
"dbt-common>=1.37.2,<2.0",
|
||||
"dbt-adapters>=1.15.5,<2.0",
|
||||
"dbt-protos>=1.0.405,<2.0",
|
||||
"pydantic<3",
|
||||
|
||||
@@ -33,7 +33,7 @@ select {{ config.require('meta_key') }} as col_value
|
||||
|
||||
meta_model_meta_require_sql = """
|
||||
-- models/meta_model.sql
|
||||
select {{ config.require('meta_key') }} as col_value
|
||||
select {{ config.meta_require('meta_key') }} as col_value
|
||||
"""
|
||||
|
||||
|
||||
@@ -66,11 +66,11 @@ class TestConfigGetMeta:
|
||||
self,
|
||||
project,
|
||||
):
|
||||
# This test runs a model with a config.get(key, default)
|
||||
# This test runs a model with a config.get(key, default) -> default value returned
|
||||
results = run_dbt(["run"], expect_pass=False)
|
||||
assert len(results) == 1
|
||||
assert str(results[0].status) == "error"
|
||||
assert 'column "my_meta_value" does not exist' in results[0].message
|
||||
assert 'column "meta_default_value" does not exist' in results[0].message
|
||||
|
||||
write_file(meta_model_meta_get_sql, "models", "meta_model.sql")
|
||||
results = run_dbt(["run"], expect_pass=False)
|
||||
@@ -95,10 +95,10 @@ class TestConfigGetMetaRequire:
|
||||
results = run_dbt(["run"], expect_pass=False)
|
||||
assert len(results) == 1
|
||||
assert str(results[0].status) == "error"
|
||||
assert 'column "my_meta_value" does not exist' in results[0].message
|
||||
assert "does not define a required config parameter 'meta_key'" in results[0].message
|
||||
|
||||
write_file(meta_model_meta_require_sql, "models", "meta_model.sql")
|
||||
results = run_dbt(["run"], expect_pass=False)
|
||||
assert len(results) == 1
|
||||
assert str(results[0].status) == "error"
|
||||
assert 'column "my_meta_value" does not exist' in results[0].message
|
||||
assert 'column "none" does not exist' in results[0].message
|
||||
|
||||
@@ -15,6 +15,10 @@ raw_source_csv = """id
|
||||
3
|
||||
"""
|
||||
|
||||
raw_seed_csv = """a,b,c,d,e,f
|
||||
3.2,3,US,2025-01-01,none,false
|
||||
4.5,4,UK,2025-01-02,2,true
|
||||
"""
|
||||
|
||||
model_sql = """
|
||||
select *
|
||||
@@ -67,8 +71,29 @@ unit_tests:
|
||||
2
|
||||
"""
|
||||
|
||||
unit_tests_seed_yml = """
|
||||
unit_tests:
|
||||
- name: test_my_seed
|
||||
model: model
|
||||
given:
|
||||
- input: ref('raw_seed')
|
||||
expect:
|
||||
format: csv
|
||||
rows: |
|
||||
a,b,c,d,e,f
|
||||
3.2,3,US,2025-01-01,none,false
|
||||
4.5,4,UK,2025-01-02,2,true
|
||||
"""
|
||||
|
||||
class TestEmptyFlag:
|
||||
|
||||
class BaseTestEmptyFlag:
|
||||
def assert_row_count(self, project, relation_name: str, expected_row_count: int):
|
||||
relation = relation_from_name(project.adapter, relation_name)
|
||||
result = project.run_sql(f"select count(*) as num_rows from {relation}", fetch="one")
|
||||
assert result[0] == expected_row_count
|
||||
|
||||
|
||||
class TestEmptyFlag(BaseTestEmptyFlag):
|
||||
@pytest.fixture(scope="class")
|
||||
def seeds(self):
|
||||
return {
|
||||
@@ -86,14 +111,10 @@ class TestEmptyFlag:
|
||||
"unit_tests.yml": unit_tests_yml,
|
||||
}
|
||||
|
||||
def assert_row_count(self, project, relation_name: str, expected_row_count: int):
|
||||
relation = relation_from_name(project.adapter, relation_name)
|
||||
result = project.run_sql(f"select count(*) as num_rows from {relation}", fetch="one")
|
||||
assert result[0] == expected_row_count
|
||||
|
||||
def test_run_with_empty(self, project):
|
||||
# create source from seed
|
||||
# Create source from seed for run and build command testing
|
||||
run_dbt(["seed"])
|
||||
self.assert_row_count(project, "raw_source", 1)
|
||||
|
||||
# run without empty - 3 expected rows in output - 1 from each input
|
||||
run_dbt(["run"])
|
||||
@@ -113,3 +134,27 @@ class TestEmptyFlag:
|
||||
|
||||
# ensure dbt compile supports --empty flag
|
||||
run_dbt(["compile", "--empty"])
|
||||
|
||||
|
||||
class TestEmptyFlagSeed(BaseTestEmptyFlag):
|
||||
@pytest.fixture(scope="class")
|
||||
def seeds(self):
|
||||
return {
|
||||
"raw_seed.csv": raw_seed_csv,
|
||||
}
|
||||
|
||||
@pytest.fixture(scope="class")
|
||||
def models(self):
|
||||
return {
|
||||
"model.sql": "select * from {{ ref('raw_seed') }}",
|
||||
"unit_tests.yml": unit_tests_seed_yml,
|
||||
}
|
||||
|
||||
def test_run_with_empty(self, project):
|
||||
run_dbt(["seed", "--empty"])
|
||||
self.assert_row_count(project, "raw_seed", 0)
|
||||
|
||||
results = run_dbt(["build", "--empty"])
|
||||
self.assert_row_count(project, "raw_seed", 0)
|
||||
|
||||
assert len(results) == 3
|
||||
|
||||
Reference in New Issue
Block a user