Compare commits

...

2 Commits

Author SHA1 Message Date
Michelle Ark
1cd5e00486 test updates 2025-06-09 22:24:10 -04:00
Quigley Malcolm
1bc391c16f Ensure source node .freshness is equal to node's .config.freshness 2025-06-09 17:53:44 -05:00
4 changed files with 44 additions and 12 deletions

View File

@@ -0,0 +1,6 @@
kind: Fixes
body: Ensure source node `.freshness` is equal to node's `.config.freshness`
time: 2025-06-09T17:52:39.978403-05:00
custom:
Author: QMalcolm
Issue: "11717"

View File

@@ -206,8 +206,7 @@ class SourcePatcher:
loader=source.loader,
loaded_at_field=loaded_at_field,
loaded_at_query=loaded_at_query,
# The setting to an empty freshness object is to maintain what we were previously doing if no freshenss was specified
freshness=config.freshness or FreshnessThreshold(),
freshness=config.freshness,
quoting=quoting,
resource_type=NodeType.Source,
fqn=target.fqn,

View File

@@ -829,11 +829,7 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False):
"database": project.database,
"description": "My table",
"external": None,
"freshness": {
"error_after": {"count": None, "period": None},
"warn_after": {"count": None, "period": None},
"filter": None,
},
"freshness": None,
"identifier": "seed",
"loaded_at_field": None,
"loaded_at_query": None,
@@ -1370,11 +1366,7 @@ def expected_references_manifest(project):
"database": project.database,
"description": "My table",
"external": None,
"freshness": {
"error_after": {"count": None, "period": None},
"warn_after": {"count": None, "period": None},
"filter": None,
},
"freshness": None,
"identifier": "seed",
"loaded_at_field": None,
"loaded_at_query": None,

View File

@@ -453,6 +453,22 @@ sources:
- name: my_table
loaded_at_query: "select 1 as id"
"""
SOURCE_FRESHNESS_AT_TABLE_AND_CONFIG = """
sources:
- name: my_source
loaded_at_field: test
tables:
- name: my_table
freshness:
warn_after: {count: 1, period: hour}
error_after: {count: 1, period: day}
config:
freshness:
warn_after: {count: 2, period: hour}
error_after: {count: 2, period: day}
"""
SOURCE_FIELD_AT_CUSTOM_FRESHNESS_BOTH_AT_TABLE = """
sources:
- name: my_source
@@ -476,6 +492,12 @@ sources:
class SchemaParserTest(BaseParserTest):
def setUp(self):
super().setUp()
# Reset `warn_error` to False so we don't raise warnigns about top level freshness as errors
set_from_args(
Namespace(warn_error=False, state_modified_compare_more_unrendered_values=False),
None,
)
self.parser = SchemaParser(
project=self.snowplow_project_config,
manifest=self.manifest,
@@ -557,6 +579,19 @@ class SchemaParserSourceTest(SchemaParserTest):
with self.assertRaises(ParsingError):
self.source_patcher.parse_source(unpatched_src_default)
@mock.patch("dbt.parser.sources.get_adapter")
def test_parse_source_resulting_node_freshness_matches_config_freshness(self, _):
block = self.file_block_for(SOURCE_FRESHNESS_AT_TABLE_AND_CONFIG, "test_one.yml")
dct = yaml_from_file(block.file, validate=True)
self.parser.parse_file(block, dct)
unpatched_src_default = self.parser.manifest.sources["source.snowplow.my_source.my_table"]
src_default = self.source_patcher.parse_source(unpatched_src_default)
assert src_default.freshness == src_default.config.freshness
assert src_default.freshness.warn_after.count == 2
assert src_default.freshness.warn_after.period == "hour"
assert src_default.freshness.error_after.count == 2
assert src_default.freshness.error_after.period == "day"
@mock.patch("dbt.parser.sources.get_adapter")
def test_parse_source_field_at_custom_freshness_both_at_source_fails(self, _):
block = self.file_block_for(