Dikootje/dbt-core
1
0
Fork 0
forked from repo-mirrors/dbt-core
Code Pull Requests Activity

Support time spine configs for sub-daily granularity (#10483)

Browse Source
This commit is contained in:
Courtney Holcomb
2024-07-29 10:39:39 -07:00
committed by GitHub
parent c598741262
commit 0a160fc27a
22 changed files with 1358 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
.changes/unreleased/Features-20240722-202238.yaml Normal file
View File

@@ -0,0 +1,6 @@
kind: Features
body: Support new semantic layer time spine configs to enable sub-daily granularity.
time: 2024-07-22T20:22:38.258249-07:00
custom:
Author: courtneyholcomb
Issue: "10475"

4
Makefile
View File

@@ -144,3 +144,7 @@ help: ## Show this help message.
@echo
@echo 'options:'
@echo 'use USE_DOCKER=true to run target in a docker container'
.PHONY: json_schema
json_schema: ## Update generated JSON schema using code changes.
scripts/collect-artifact-schema.py --path schemas

2
core/dbt/artifacts/resources/__init__.py
View File

@@ -46,7 +46,7 @@ from dbt.artifacts.resources.v1.metric import (
MetricTimeWindow,
MetricTypeParams,
)
from dbt.artifacts.resources.v1.model import Model, ModelConfig
from dbt.artifacts.resources.v1.model import Model, ModelConfig, TimeSpine
from dbt.artifacts.resources.v1.owner import Owner
from dbt.artifacts.resources.v1.saved_query import (
Export,

2
core/dbt/artifacts/resources/v1/components.py
View File

@@ -10,6 +10,7 @@ from dbt_common.contracts.config.properties import AdditionalPropertiesMixin
from dbt_common.contracts.constraints import ColumnLevelConstraint
from dbt_common.contracts.util import Mergeable
from dbt_common.dataclass_schema import ExtensibleDbtClassMixin, dbtClassMixin
from dbt_semantic_interfaces.type_enums import TimeGranularity
NodeVersion = Union[str, float]
@@ -66,6 +67,7 @@ class ColumnInfo(AdditionalPropertiesMixin, ExtensibleDbtClassMixin):
quote: Optional[bool] = None
tags: List[str] = field(default_factory=list)
_extra: Dict[str, Any] = field(default_factory=dict)
granularity: Optional[TimeGranularity] = None
@dataclass

7
core/dbt/artifacts/resources/v1/model.py
View File

@@ -11,6 +11,7 @@ from dbt.artifacts.resources.v1.components import (
from dbt.artifacts.resources.v1.config import NodeConfig
from dbt_common.contracts.config.base import MergeBehavior
from dbt_common.contracts.constraints import ModelLevelConstraint
from dbt_common.dataclass_schema import dbtClassMixin
@dataclass
@@ -21,6 +22,11 @@ class ModelConfig(NodeConfig):
)
@dataclass
class TimeSpine(dbtClassMixin):
standard_granularity_column: str
@dataclass
class Model(CompiledResource):
resource_type: Literal[NodeType.Model]
@@ -32,6 +38,7 @@ class Model(CompiledResource):
deprecation_date: Optional[datetime] = None
defer_relation: Optional[DeferRelation] = None
primary_key: List[str] = field(default_factory=list)
time_spine: Optional[TimeSpine] = None
def __post_serialize__(self, dct: Dict, context: Optional[Dict] = None):
dct = super().__post_serialize__(dct, context)

6
core/dbt/constants.py
View File

@@ -1,3 +1,5 @@
from dbt_semantic_interfaces.type_enums import TimeGranularity
DEFAULT_ENV_PLACEHOLDER = "DBT_DEFAULT_PLACEHOLDER"
SECRET_PLACEHOLDER = "$$$DBT_SECRET_START$$${}$$$DBT_SECRET_END$$$"
@@ -15,6 +17,8 @@ DEPENDENCIES_FILE_NAME = "dependencies.yml"
PACKAGE_LOCK_FILE_NAME = "package-lock.yml"
MANIFEST_FILE_NAME = "manifest.json"
SEMANTIC_MANIFEST_FILE_NAME = "semantic_manifest.json"
TIME_SPINE_MODEL_NAME = "metricflow_time_spine"
LEGACY_TIME_SPINE_MODEL_NAME = "metricflow_time_spine"
LEGACY_TIME_SPINE_GRANULARITY = TimeGranularity.DAY
MINIMUM_REQUIRED_TIME_SPINE_GRANULARITY = TimeGranularity.DAY
PARTIAL_PARSE_FILE_NAME = "partial_parse.msgpack"
PACKAGE_LOCK_HASH_KEY = "sha1_hash"

2
core/dbt/contracts/graph/nodes.py
View File

@@ -58,6 +58,7 @@ from dbt.artifacts.resources import SingularTest as SingularTestResource
from dbt.artifacts.resources import Snapshot as SnapshotResource
from dbt.artifacts.resources import SourceDefinition as SourceDefinitionResource
from dbt.artifacts.resources import SqlOperation as SqlOperationResource
from dbt.artifacts.resources import TimeSpine
from dbt.artifacts.resources import UnitTestDefinition as UnitTestDefinitionResource
from dbt.contracts.graph.model_config import UnitTestNodeConfig
from dbt.contracts.graph.node_args import ModelNodeArgs
@@ -1625,6 +1626,7 @@ class ParsedNodePatch(ParsedPatch):
latest_version: Optional[NodeVersion]
constraints: List[Dict[str, Any]]
deprecation_date: Optional[datetime]
time_spine: Optional[TimeSpine] = None
@dataclass

112
core/dbt/contracts/graph/semantic_manifest.py
View File

@@ -1,10 +1,19 @@
from dbt.constants import TIME_SPINE_MODEL_NAME
from typing import List, Optional
from dbt.constants import (
LEGACY_TIME_SPINE_GRANULARITY,
LEGACY_TIME_SPINE_MODEL_NAME,
MINIMUM_REQUIRED_TIME_SPINE_GRANULARITY,
)
from dbt.contracts.graph.manifest import Manifest
from dbt.contracts.graph.nodes import ModelNode
from dbt.events.types import SemanticValidationFailure
from dbt.exceptions import ParsingError
from dbt_common.clients.system import write_file
from dbt_common.events.base_types import EventLevel
from dbt_common.events.functions import fire_event
from dbt_semantic_interfaces.implementations.metric import PydanticMetric
from dbt_semantic_interfaces.implementations.node_relation import PydanticNodeRelation
from dbt_semantic_interfaces.implementations.project_configuration import (
PydanticProjectConfiguration,
)
@@ -13,8 +22,12 @@ from dbt_semantic_interfaces.implementations.semantic_manifest import (
PydanticSemanticManifest,
)
from dbt_semantic_interfaces.implementations.semantic_model import PydanticSemanticModel
from dbt_semantic_interfaces.implementations.time_spine import (
PydanticTimeSpine,
PydanticTimeSpinePrimaryColumn,
)
from dbt_semantic_interfaces.implementations.time_spine_table_configuration import (
PydanticTimeSpineTableConfiguration,
PydanticTimeSpineTableConfiguration as LegacyTimeSpine,
)
from dbt_semantic_interfaces.type_enums import TimeGranularity
from dbt_semantic_interfaces.validations.semantic_manifest_validator import (
@@ -23,7 +36,7 @@ from dbt_semantic_interfaces.validations.semantic_manifest_validator import (
class SemanticManifest:
def __init__(self, manifest) -> None:
def __init__(self, manifest: Manifest) -> None:
self.manifest = manifest
def validate(self) -> bool:
@@ -59,8 +72,50 @@ class SemanticManifest:
write_file(file_path, json)
def _get_pydantic_semantic_manifest(self) -> PydanticSemanticManifest:
pydantic_time_spines: List[PydanticTimeSpine] = []
minimum_time_spine_granularity: Optional[TimeGranularity] = None
for node in self.manifest.nodes.values():
if not (isinstance(node, ModelNode) and node.time_spine):
continue
time_spine = node.time_spine
standard_granularity_column = None
for column in node.columns.values():
if column.name == time_spine.standard_granularity_column:
standard_granularity_column = column
break
# Assertions needed for type checking
if not standard_granularity_column:
raise ParsingError(
"Expected to find time spine standard granularity column in model columns, but did not. "
"This should have been caught in YAML parsing."
)
if not standard_granularity_column.granularity:
raise ParsingError(
"Expected to find granularity set for time spine standard granularity column, but did not. "
"This should have been caught in YAML parsing."
)
pydantic_time_spine = PydanticTimeSpine(
node_relation=PydanticNodeRelation(
alias=node.alias,
schema_name=node.schema,
database=node.database,
relation_name=node.relation_name,
),
primary_column=PydanticTimeSpinePrimaryColumn(
name=time_spine.standard_granularity_column,
time_granularity=standard_granularity_column.granularity,
),
)
pydantic_time_spines.append(pydantic_time_spine)
if (
not minimum_time_spine_granularity
or standard_granularity_column.granularity.to_int()
< minimum_time_spine_granularity.to_int()
):
minimum_time_spine_granularity = standard_granularity_column.granularity
project_config = PydanticProjectConfiguration(
time_spine_table_configurations=[],
time_spine_table_configurations=[], time_spines=pydantic_time_spines
)
pydantic_semantic_manifest = PydanticSemanticManifest(
metrics=[], semantic_models=[], project_configuration=project_config
@@ -79,24 +134,39 @@ class SemanticManifest:
PydanticSavedQuery.parse_obj(saved_query.to_dict())
)
# Look for time-spine table model and create time spine table configuration
if self.manifest.semantic_models:
# Get model for time_spine_table
model = self.manifest.ref_lookup.find(TIME_SPINE_MODEL_NAME, None, None, self.manifest)
if not model:
raise ParsingError(
"The semantic layer requires a 'metricflow_time_spine' model in the project, but none was found. "
"Guidance on creating this model can be found on our docs site ("
"https://docs.getdbt.com/docs/build/metricflow-time-spine) "
)
# Create time_spine_table_config, set it in project_config, and add to semantic manifest
time_spine_table_config = PydanticTimeSpineTableConfiguration(
location=model.relation_name,
column_name="date_day",
grain=TimeGranularity.DAY,
legacy_time_spine_model = self.manifest.ref_lookup.find(
LEGACY_TIME_SPINE_MODEL_NAME, None, None, self.manifest
)
pydantic_semantic_manifest.project_configuration.time_spine_table_configurations = [
time_spine_table_config
]
if legacy_time_spine_model:
if (
not minimum_time_spine_granularity
or LEGACY_TIME_SPINE_GRANULARITY.to_int()
< minimum_time_spine_granularity.to_int()
):
minimum_time_spine_granularity = LEGACY_TIME_SPINE_GRANULARITY
# If no time spines have been configured at DAY or smaller AND legacy time spine model does not exist, error.
if (
not minimum_time_spine_granularity
or minimum_time_spine_granularity.to_int()
> MINIMUM_REQUIRED_TIME_SPINE_GRANULARITY.to_int()
):
raise ParsingError(
"The semantic layer requires a time spine model with granularity DAY or smaller in the project, "
"but none was found. Guidance on creating this model can be found on our docs site "
"(https://docs.getdbt.com/docs/build/metricflow-time-spine)." # TODO: update docs link when available!
)
# For backward compatibility: if legacy time spine exists, include it in the manifest.
if legacy_time_spine_model:
legacy_time_spine = LegacyTimeSpine(
location=legacy_time_spine_model.relation_name,
column_name="date_day",
grain=LEGACY_TIME_SPINE_GRANULARITY,
)
pydantic_semantic_manifest.project_configuration.time_spine_table_configurations = [
legacy_time_spine
]
return pydantic_semantic_manifest

27
core/dbt/contracts/graph/unparsed.py
View File

@@ -116,6 +116,7 @@ class HasColumnAndTestProps(HasColumnProps):
class UnparsedColumn(HasColumnAndTestProps):
quote: Optional[bool] = None
tags: List[str] = field(default_factory=list)
granularity: Optional[str] = None # str is really a TimeGranularity Enum
@dataclass
@@ -206,6 +207,11 @@ class UnparsedNodeUpdate(HasConfig, HasColumnTests, HasColumnAndTestProps, HasYa
access: Optional[str] = None
@dataclass
class UnparsedTimeSpine(dbtClassMixin):
standard_granularity_column: str
@dataclass
class UnparsedModelUpdate(UnparsedNodeUpdate):
quote_columns: Optional[bool] = None
@@ -213,6 +219,7 @@ class UnparsedModelUpdate(UnparsedNodeUpdate):
latest_version: Optional[NodeVersion] = None
versions: Sequence[UnparsedVersion] = field(default_factory=list)
deprecation_date: Optional[datetime.datetime] = None
time_spine: Optional[UnparsedTimeSpine] = None
def __post_init__(self) -> None:
if self.latest_version:
@@ -234,6 +241,26 @@ class UnparsedModelUpdate(UnparsedNodeUpdate):
self.deprecation_date = normalize_date(self.deprecation_date)
if self.time_spine:
columns = (
self.get_columns_for_version(self.latest_version)
if self.latest_version
else self.columns
)
column_names_to_columns = {column.name: column for column in columns}
if self.time_spine.standard_granularity_column not in column_names_to_columns:
raise ParsingError(
f"Time spine standard granularity column must be defined on the model. Got invalid "
f"column name '{self.time_spine.standard_granularity_column}' for model '{self.name}'. Valid names"
f"{' for latest version' if self.latest_version else ''}: {list(column_names_to_columns.keys())}."
)
column = column_names_to_columns[self.time_spine.standard_granularity_column]
if not column.granularity:
raise ParsingError(
f"Time spine standard granularity column must have a granularity defined. Please add one for "
f"column '{self.time_spine.standard_granularity_column}' in model '{self.name}'."
)
def get_columns_for_version(self, version: NodeVersion) -> List[UnparsedColumn]:
if version not in self._version_map:
raise DbtInternalError(

11
core/dbt/parser/common.py
View File

@@ -18,6 +18,7 @@ from dbt.exceptions import ParsingError
from dbt.parser.search import FileBlock
from dbt_common.contracts.constraints import ColumnLevelConstraint, ConstraintType
from dbt_common.exceptions import DbtInternalError
from dbt_semantic_interfaces.type_enums import TimeGranularity
def trimmed(inp: str) -> str:
@@ -185,13 +186,12 @@ class ParserRef:
self.column_info: Dict[str, ColumnInfo] = {}
def _add(self, column: HasColumnProps) -> None:
tags: List[str] = []
tags.extend(getattr(column, "tags", ()))
quote: Optional[bool]
tags: List[str] = getattr(column, "tags", [])
quote: Optional[bool] = None
granularity: Optional[TimeGranularity] = None
if isinstance(column, UnparsedColumn):
quote = column.quote
else:
quote = None
granularity = TimeGranularity(column.granularity) if column.granularity else None
if any(
c
@@ -209,6 +209,7 @@ class ParserRef:
tags=tags,
quote=quote,
_extra=column.extra,
granularity=granularity,
)
@classmethod

15
core/dbt/parser/schemas.py
View File

@@ -6,6 +6,7 @@ from typing import Any, Callable, Dict, Generic, Iterable, List, Optional, Type,
from dbt import deprecations
from dbt.artifacts.resources import RefArgs
from dbt.artifacts.resources.v1.model import TimeSpine
from dbt.clients.jinja_static import statically_parse_ref_or_source
from dbt.clients.yaml_helper import load_yaml_text
from dbt.config import RuntimeConfig
@@ -619,9 +620,16 @@ class NodePatchParser(PatchParser[NodeTarget, ParsedNodePatch], Generic[NodeTarg
# could possibly skip creating one. Leaving here for now for
# code consistency.
deprecation_date: Optional[datetime.datetime] = None
time_spine: Optional[TimeSpine] = None
if isinstance(block.target, UnparsedModelUpdate):
deprecation_date = block.target.deprecation_date
time_spine = (
TimeSpine(
standard_granularity_column=block.target.time_spine.standard_granularity_column
)
if block.target.time_spine
else None
)
patch = ParsedNodePatch(
name=block.target.name,
original_file_path=block.target.original_file_path,
@@ -637,6 +645,7 @@ class NodePatchParser(PatchParser[NodeTarget, ParsedNodePatch], Generic[NodeTarg
latest_version=None,
constraints=block.target.constraints,
deprecation_date=deprecation_date,
time_spine=time_spine,
)
assert isinstance(self.yaml.file, SchemaSourceFile)
source_file: SchemaSourceFile = self.yaml.file
@@ -915,6 +924,7 @@ class ModelPatchParser(NodePatchParser[UnparsedModelUpdate]):
)
# These two will have to be reapplied after config is built for versioned models
self.patch_constraints(node, patch.constraints)
self.patch_time_spine(node, patch.time_spine)
node.build_contract_checksum()
def patch_constraints(self, node, constraints: List[Dict[str, Any]]) -> None:
@@ -953,6 +963,9 @@ class ModelPatchParser(NodePatchParser[UnparsedModelUpdate]):
else:
model_node.sources.append(ref_or_source)
def patch_time_spine(self, node, time_spine: Optional[TimeSpine]) -> None:
node.time_spine = time_spine
def _validate_pk_constraints(
self, model_node: ModelNode, constraints: List[Dict[str, Any]]
) -> None:

2
core/setup.py
View File

@@ -69,7 +69,7 @@ setup(
# Accept patches but avoid automatically updating past a set minor version range.
"dbt-extractor>=0.5.0,<=0.6",
"minimal-snowplow-tracker>=0.0.2,<0.1",
"dbt-semantic-interfaces>=0.6.8,<0.7",
"dbt-semantic-interfaces>=0.6.10,<0.7",
# Minor versions for these are expected to be backwards-compatible
"dbt-common>=1.6.0,<2.0",
"dbt-adapters>=1.1.1,<2.0",

796
schemas/dbt/manifest/v12.json
View File

File diff suppressed because it is too large Load Diff

38
tests/functional/artifacts/expected_manifest.py
View File

@@ -292,6 +292,7 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False):
"quote": None,
"tags": [],
"constraints": [],
"granularity": None,
},
"first_name": {
"name": "first_name",
@@ -301,6 +302,7 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False):
"quote": None,
"tags": [],
"constraints": [],
"granularity": None,
},
"email": {
"name": "email",
@@ -310,6 +312,7 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False):
"quote": None,
"tags": [],
"constraints": [],
"granularity": None,
},
"ip_address": {
"name": "ip_address",
@@ -319,6 +322,7 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False):
"quote": None,
"tags": [],
"constraints": [],
"granularity": None,
},
"updated_at": {
"name": "updated_at",
@@ -328,6 +332,7 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False):
"quote": None,
"tags": [],
"constraints": [],
"granularity": None,
},
},
"contract": {"checksum": None, "enforced": False, "alias_types": True},
@@ -343,6 +348,7 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False):
"access": "protected",
"version": None,
"latest_version": None,
"time_spine": None,
},
"model.test.second_model": {
"compiled_path": os.path.join(compiled_model_path, "second_model.sql"),
@@ -385,6 +391,7 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False):
"quote": None,
"tags": [],
"constraints": [],
"granularity": None,
},
"first_name": {
"name": "first_name",
@@ -394,6 +401,7 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False):
"quote": None,
"tags": [],
"constraints": [],
"granularity": None,
},
"email": {
"name": "email",
@@ -403,6 +411,7 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False):
"quote": None,
"tags": [],
"constraints": [],
"granularity": None,
},
"ip_address": {
"name": "ip_address",
@@ -412,6 +421,7 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False):
"quote": None,
"tags": [],
"constraints": [],
"granularity": None,
},
"updated_at": {
"name": "updated_at",
@@ -421,6 +431,7 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False):
"quote": None,
"tags": [],
"constraints": [],
"granularity": None,
},
},
"contract": {"checksum": None, "enforced": False, "alias_types": True},
@@ -436,6 +447,7 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False):
"access": "protected",
"version": None,
"latest_version": None,
"time_spine": None,
},
"seed.test.seed": {
"build_path": None,
@@ -468,6 +480,7 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False):
"quote": None,
"tags": [],
"constraints": [],
"granularity": None,
},
"first_name": {
"name": "first_name",
@@ -477,6 +490,7 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False):
"quote": None,
"tags": [],
"constraints": [],
"granularity": None,
},
"email": {
"name": "email",
@@ -486,6 +500,7 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False):
"quote": None,
"tags": [],
"constraints": [],
"granularity": None,
},
"ip_address": {
"name": "ip_address",
@@ -495,6 +510,7 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False):
"quote": None,
"tags": [],
"constraints": [],
"granularity": None,
},
"updated_at": {
"name": "updated_at",
@@ -504,6 +520,7 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False):
"quote": None,
"tags": [],
"constraints": [],
"granularity": None,
},
},
"docs": {"node_color": None, "show": True},
@@ -730,6 +747,7 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False):
"quote": None,
"tags": [],
"constraints": [],
"granularity": None,
}
},
"config": {
@@ -957,6 +975,7 @@ def expected_references_manifest(project):
"version": None,
"latest_version": None,
"constraints": [],
"time_spine": None,
},
"model.test.ephemeral_summary": {
"alias": "ephemeral_summary",
@@ -972,6 +991,7 @@ def expected_references_manifest(project):
"quote": None,
"tags": [],
"constraints": [],
"granularity": None,
},
"ct": {
"description": "The number of instances of the first name",
@@ -981,6 +1001,7 @@ def expected_references_manifest(project):
"quote": None,
"tags": [],
"constraints": [],
"granularity": None,
},
},
"config": get_rendered_model_config(materialized="table", group="test_group"),
@@ -1026,6 +1047,7 @@ def expected_references_manifest(project):
"version": None,
"latest_version": None,
"constraints": [],
"time_spine": None,
},
"model.test.view_summary": {
"alias": "view_summary",
@@ -1041,6 +1063,7 @@ def expected_references_manifest(project):
"quote": None,
"tags": [],
"constraints": [],
"granularity": None,
},
"ct": {
"description": "The number of instances of the first name",
@@ -1050,6 +1073,7 @@ def expected_references_manifest(project):
"quote": None,
"tags": [],
"constraints": [],
"granularity": None,
},
},
"config": get_rendered_model_config(),
@@ -1091,6 +1115,7 @@ def expected_references_manifest(project):
"version": None,
"latest_version": None,
"constraints": [],
"time_spine": None,
},
"seed.test.seed": {
"alias": "seed",
@@ -1105,6 +1130,7 @@ def expected_references_manifest(project):
"quote": None,
"tags": [],
"constraints": [],
"granularity": None,
},
"first_name": {
"name": "first_name",
@@ -1114,6 +1140,7 @@ def expected_references_manifest(project):
"quote": None,
"tags": [],
"constraints": [],
"granularity": None,
},
"email": {
"name": "email",
@@ -1123,6 +1150,7 @@ def expected_references_manifest(project):
"quote": None,
"tags": [],
"constraints": [],
"granularity": None,
},
"ip_address": {
"name": "ip_address",
@@ -1132,6 +1160,7 @@ def expected_references_manifest(project):
"quote": None,
"tags": [],
"constraints": [],
"granularity": None,
},
"updated_at": {
"name": "updated_at",
@@ -1141,6 +1170,7 @@ def expected_references_manifest(project):
"quote": None,
"tags": [],
"constraints": [],
"granularity": None,
},
},
"config": get_rendered_seed_config(),
@@ -1219,6 +1249,7 @@ def expected_references_manifest(project):
"quote": None,
"tags": [],
"constraints": [],
"granularity": None,
}
},
"config": {
@@ -1487,6 +1518,7 @@ def expected_versions_manifest(project):
"quote": None,
"tags": [],
"constraints": [],
"granularity": None,
},
"ct": {
"description": "The number of instances of the first name",
@@ -1496,6 +1528,7 @@ def expected_versions_manifest(project):
"quote": None,
"tags": [],
"constraints": [],
"granularity": None,
},
},
"config": get_rendered_model_config(
@@ -1544,6 +1577,7 @@ def expected_versions_manifest(project):
"access": "protected",
"version": 1,
"latest_version": 2,
"time_spine": None,
},
"model.test.versioned_model.v2": {
"alias": "versioned_model_v2",
@@ -1559,6 +1593,7 @@ def expected_versions_manifest(project):
"quote": None,
"tags": [],
"constraints": [],
"granularity": None,
},
"extra": {
"description": "",
@@ -1568,6 +1603,7 @@ def expected_versions_manifest(project):
"quote": None,
"tags": [],
"constraints": [],
"granularity": None,
},
},
"config": get_rendered_model_config(
@@ -1612,6 +1648,7 @@ def expected_versions_manifest(project):
"access": "protected",
"version": 2,
"latest_version": 2,
"time_spine": None,
},
"model.test.ref_versioned_model": {
"alias": "ref_versioned_model",
@@ -1669,6 +1706,7 @@ def expected_versions_manifest(project):
"access": "protected",
"version": None,
"latest_version": None,
"time_spine": None,
},
"test.test.unique_versioned_model_v1_first_name.6138195dec": {
"alias": "unique_versioned_model_v1_first_name",

2
tests/functional/configs/test_contract_configs.py
View File

@@ -331,7 +331,7 @@ class TestModelLevelContractEnabledConfigs:
assert contract_actual_config.enforced is True
expected_columns = "{'id': ColumnInfo(name='id', description='hello', meta={}, data_type='integer', constraints=[ColumnLevelConstraint(type=<ConstraintType.not_null: 'not_null'>, name=None, expression=None, warn_unenforced=True, warn_unsupported=True, to=None, to_columns=[]), ColumnLevelConstraint(type=<ConstraintType.primary_key: 'primary_key'>, name=None, expression=None, warn_unenforced=True, warn_unsupported=True, to=None, to_columns=[]), ColumnLevelConstraint(type=<ConstraintType.check: 'check'>, name=None, expression='(id > 0)', warn_unenforced=True, warn_unsupported=True, to=None, to_columns=[])], quote=True, tags=[], _extra={}), 'color': ColumnInfo(name='color', description='', meta={}, data_type='string', constraints=[], quote=None, tags=[], _extra={}), 'date_day': ColumnInfo(name='date_day', description='', meta={}, data_type='date', constraints=[], quote=None, tags=[], _extra={})}"
expected_columns = "{'id': ColumnInfo(name='id', description='hello', meta={}, data_type='integer', constraints=[ColumnLevelConstraint(type=<ConstraintType.not_null: 'not_null'>, name=None, expression=None, warn_unenforced=True, warn_unsupported=True, to=None, to_columns=[]), ColumnLevelConstraint(type=<ConstraintType.primary_key: 'primary_key'>, name=None, expression=None, warn_unenforced=True, warn_unsupported=True, to=None, to_columns=[]), ColumnLevelConstraint(type=<ConstraintType.check: 'check'>, name=None, expression='(id > 0)', warn_unenforced=True, warn_unsupported=True, to=None, to_columns=[])], quote=True, tags=[], _extra={}, granularity=None), 'color': ColumnInfo(name='color', description='', meta={}, data_type='string', constraints=[], quote=None, tags=[], _extra={}, granularity=None), 'date_day': ColumnInfo(name='date_day', description='', meta={}, data_type='date', constraints=[], quote=None, tags=[], _extra={}, granularity=None)}"
assert expected_columns == str(my_model_columns)

6
tests/functional/docs/test_good_docs_blocks.py
View File

@@ -91,6 +91,7 @@ class TestGoodDocsBlocks:
"meta": {},
"quote": None,
"tags": [],
"granularity": None,
} == model_data["columns"]["id"]
assert {
@@ -101,6 +102,7 @@ class TestGoodDocsBlocks:
"meta": {},
"quote": None,
"tags": [],
"granularity": None,
} == model_data["columns"]["first_name"]
assert {
@@ -111,6 +113,7 @@ class TestGoodDocsBlocks:
"meta": {},
"quote": None,
"tags": [],
"granularity": None,
} == model_data["columns"]["last_name"]
assert len(model_data["columns"]) == 3
@@ -152,6 +155,7 @@ class TestGoodDocsBlocksAltPath:
"meta": {},
"quote": None,
"tags": [],
"granularity": None,
} == model_data["columns"]["id"]
assert {
@@ -162,6 +166,7 @@ class TestGoodDocsBlocksAltPath:
"meta": {},
"quote": None,
"tags": [],
"granularity": None,
} == model_data["columns"]["first_name"]
assert {
@@ -172,6 +177,7 @@ class TestGoodDocsBlocksAltPath:
"meta": {},
"quote": None,
"tags": [],
"granularity": None,
} == model_data["columns"]["last_name"]
assert len(model_data["columns"]) == 3

2
tests/functional/fixtures/happy_path_project/models/metricflow_time_spine_second.sql Normal file
View File

@@ -0,0 +1,2 @@
select
{{ dbt.date_trunc('second', dbt.current_timestamp()) }} as ts_second

10
tests/functional/fixtures/happy_path_project/models/schema.yml
View File

@@ -8,6 +8,16 @@ models:
data_tests:
- unique
- not_null
- name: metricflow_time_spine
description: Day time spine
columns:
- name: date_day
granularity: day
- name: metricflow_time_spine_second
description: Second time spine
columns:
- name: ts_second
granularity: second
sources:
- name: my_source

55
tests/functional/list/test_list.py
View File

@@ -133,12 +133,20 @@ class TestList:
def expect_model_output(self):
expectations = {
"name": ("ephemeral", "incremental", "inner", "metricflow_time_spine", "outer"),
"name": (
"ephemeral",
"incremental",
"inner",
"metricflow_time_spine",
"metricflow_time_spine_second",
"outer",
),
"selector": (
"test.ephemeral",
"test.incremental",
"test.sub.inner",
"test.metricflow_time_spine",
"test.metricflow_time_spine_second",
"test.outer",
),
"json": (
@@ -294,6 +302,44 @@ class TestList:
"alias": "metricflow_time_spine",
"resource_type": "model",
},
{
"name": "metricflow_time_spine_second",
"package_name": "test",
"depends_on": {
"nodes": [],
"macros": ["macro.dbt.current_timestamp", "macro.dbt.date_trunc"],
},
"tags": [],
"config": {
"enabled": True,
"group": None,
"materialized": "view",
"post-hook": [],
"tags": [],
"pre-hook": [],
"quoting": {},
"column_types": {},
"persist_docs": {},
"full_refresh": None,
"unique_key": None,
"on_schema_change": "ignore",
"on_configuration_change": "apply",
"database": None,
"schema": None,
"alias": None,
"meta": {},
"grants": {},
"packages": [],
"incremental_strategy": None,
"docs": {"node_color": None, "show": True},
"contract": {"enforced": False, "alias_types": True},
"access": "protected",
},
"original_file_path": normalize("models/metricflow_time_spine_second.sql"),
"unique_id": "model.test.metricflow_time_spine_second",
"alias": "metricflow_time_spine_second",
"resource_type": "model",
},
{
"name": "outer",
"package_name": "test",
@@ -338,6 +384,7 @@ class TestList:
self.dir("models/incremental.sql"),
self.dir("models/sub/inner.sql"),
self.dir("models/metricflow_time_spine.sql"),
self.dir("models/metricflow_time_spine_second.sql"),
self.dir("models/outer.sql"),
),
}
@@ -573,6 +620,7 @@ class TestList:
"test.not_null_outer_id",
"test.unique_outer_id",
"test.metricflow_time_spine",
"test.metricflow_time_spine_second",
"test.t",
"semantic_model:test.my_sm",
"metric:test.total_outer",
@@ -618,6 +666,7 @@ class TestList:
"test.ephemeral",
"test.outer",
"test.metricflow_time_spine",
"test.metricflow_time_spine_second",
"test.incremental",
}
@@ -638,6 +687,7 @@ class TestList:
"test.outer",
"test.sub.inner",
"test.metricflow_time_spine",
"test.metricflow_time_spine_second",
"test.t",
"test.unique_outer_id",
}
@@ -658,6 +708,7 @@ class TestList:
"test.not_null_outer_id",
"test.outer",
"test.metricflow_time_spine",
"test.metricflow_time_spine_second",
"test.sub.inner",
"test.t",
}
@@ -693,6 +744,7 @@ class TestList:
"test.outer",
"test.sub.inner",
"test.metricflow_time_spine",
"test.metricflow_time_spine_second",
"test.t",
"test.unique_outer_id",
}
@@ -707,6 +759,7 @@ class TestList:
"test.outer",
"test.sub.inner",
"test.metricflow_time_spine",
"test.metricflow_time_spine_second",
}
del os.environ["DBT_EXCLUDE_RESOURCE_TYPES"]

86
tests/functional/time_spines/fixtures.py Normal file
View File

@@ -0,0 +1,86 @@
models_people_sql = """
select 1 as id, 'Drew' as first_name, 'Banin' as last_name, 'yellow' as favorite_color, true as loves_dbt, 5 as tenure, current_timestamp as created_at
union all
select 2 as id, 'Jeremy' as first_name, 'Cohen' as last_name, 'indigo' as favorite_color, true as loves_dbt, 4 as tenure, current_timestamp as created_at
union all
select 3 as id, 'Callum' as first_name, 'McCann' as last_name, 'emerald' as favorite_color, true as loves_dbt, 0 as tenure, current_timestamp as created_at
"""
semantic_model_people_yml = """
version: 2
semantic_models:
- name: semantic_people
model: ref('people')
dimensions:
- name: favorite_color
type: categorical
- name: created_at
type: TIME
type_params:
time_granularity: day
measures:
- name: years_tenure
agg: SUM
expr: tenure
- name: people
agg: count
expr: id
entities:
- name: id
type: primary
defaults:
agg_time_dimension: created_at
"""
metricflow_time_spine_sql = """
SELECT to_date('02/20/2023, 'mm/dd/yyyy') as date_day
"""
metricflow_time_spine_second_sql = """
SELECT to_datetime('02/20/2023, 'mm/dd/yyyy hh:mm:ss') as ts_second
"""
valid_time_spines_yml = """
version: 2
models:
- name: metricflow_time_spine_second
time_spine:
standard_granularity_column: ts_second
columns:
- name: ts_second
granularity: second
- name: metricflow_time_spine
time_spine:
standard_granularity_column: date_day
columns:
- name: date_day
granularity: day
"""
missing_time_spine_yml = """
models:
- name: metricflow_time_spine
columns:
- name: ts_second
granularity: second
"""
time_spine_missing_granularity_yml = """
models:
- name: metricflow_time_spine_second
time_spine:
standard_granularity_column: ts_second
columns:
- name: ts_second
"""
time_spine_missing_column_yml = """
models:
- name: metricflow_time_spine_second
time_spine:
standard_granularity_column: ts_second
columns:
- name: date_day
"""

198
tests/functional/time_spines/test_time_spines.py Normal file
View File

@@ -0,0 +1,198 @@
from typing import Set
import pytest
from dbt.cli.main import dbtRunner
from dbt.contracts.graph.manifest import Manifest
from dbt.contracts.graph.semantic_manifest import SemanticManifest
from dbt.exceptions import ParsingError
from dbt.tests.util import get_manifest
from dbt_semantic_interfaces.type_enums import TimeGranularity
from tests.functional.time_spines.fixtures import (
metricflow_time_spine_second_sql,
metricflow_time_spine_sql,
models_people_sql,
semantic_model_people_yml,
time_spine_missing_column_yml,
time_spine_missing_granularity_yml,
valid_time_spines_yml,
)
class TestValidTimeSpines:
"""Tests that YAML using current time spine configs parses as expected."""
@pytest.fixture(scope="class")
def models(self):
return {
"metricflow_time_spine.sql": metricflow_time_spine_sql,
"metricflow_time_spine_second.sql": metricflow_time_spine_second_sql,
"time_spines.yml": valid_time_spines_yml,
"semantic_model_people.yml": semantic_model_people_yml,
"people.sql": models_people_sql,
}
def test_time_spines(self, project):
runner = dbtRunner()
result = runner.invoke(["parse"])
assert result.success
assert isinstance(result.result, Manifest)
manifest = get_manifest(project.project_root)
assert manifest
# Test that models and columns are set as expected
time_spine_models = {
id.split(".")[-1]: node for id, node in manifest.nodes.items() if node.time_spine
}
day_model_name = "metricflow_time_spine"
second_model_name = "metricflow_time_spine_second"
day_column_name = "date_day"
second_column_name = "ts_second"
model_names_to_col_names = {
day_model_name: day_column_name,
second_model_name: second_column_name,
}
model_names_to_granularities = {
day_model_name: TimeGranularity.DAY,
second_model_name: TimeGranularity.SECOND,
}
assert len(time_spine_models) == 2
expected_time_spine_aliases = {second_model_name, day_model_name}
assert set(time_spine_models.keys()) == expected_time_spine_aliases
for model in time_spine_models.values():
assert (
model.time_spine.standard_granularity_column
== model_names_to_col_names[model.name]
)
assert len(model.columns) == 1
assert (
list(model.columns.values())[0].granularity
== model_names_to_granularities[model.name]
)
# Test that project configs are set as expected in semantic manifest
semantic_manifest = SemanticManifest(manifest)
assert semantic_manifest.validate()
project_config = semantic_manifest._get_pydantic_semantic_manifest().project_configuration
# Legacy config
assert len(project_config.time_spine_table_configurations) == 1
legacy_time_spine_config = project_config.time_spine_table_configurations[0]
assert legacy_time_spine_config.column_name == day_column_name
assert legacy_time_spine_config.location.replace('"', "").split(".")[-1] == day_model_name
assert legacy_time_spine_config.grain == TimeGranularity.DAY
# Current configs
assert len(project_config.time_spines) == 2
sl_time_spine_aliases: Set[str] = set()
for sl_time_spine in project_config.time_spines:
alias = sl_time_spine.node_relation.alias
sl_time_spine_aliases.add(alias)
assert sl_time_spine.primary_column.name == model_names_to_col_names[alias]
assert (
sl_time_spine.primary_column.time_granularity
== model_names_to_granularities[alias]
)
assert sl_time_spine_aliases == expected_time_spine_aliases
class TestValidLegacyTimeSpine:
"""Tests that YAML using only legacy time spine config parses as expected."""
@pytest.fixture(scope="class")
def models(self):
return {
"metricflow_time_spine.sql": metricflow_time_spine_sql,
"semantic_model_people.yml": semantic_model_people_yml,
"people.sql": models_people_sql,
}
def test_time_spines(self, project):
runner = dbtRunner()
result = runner.invoke(["parse"])
assert result.success
assert isinstance(result.result, Manifest)
manifest = get_manifest(project.project_root)
assert manifest
# Test that project configs are set as expected in semantic manifest
semantic_manifest = SemanticManifest(manifest)
assert semantic_manifest.validate()
project_config = semantic_manifest._get_pydantic_semantic_manifest().project_configuration
# Legacy config
assert len(project_config.time_spine_table_configurations) == 1
legacy_time_spine_config = project_config.time_spine_table_configurations[0]
assert legacy_time_spine_config.column_name == "date_day"
assert (
legacy_time_spine_config.location.replace('"', "").split(".")[-1]
== "metricflow_time_spine"
)
assert legacy_time_spine_config.grain == TimeGranularity.DAY
# Current configs
assert len(project_config.time_spines) == 0
class TestMissingTimeSpine:
"""Tests that YAML with semantic models but no time spines errors."""
@pytest.fixture(scope="class")
def models(self):
return {
"semantic_model_people.yml": semantic_model_people_yml,
"people.sql": models_people_sql,
}
def test_time_spines(self, project):
runner = dbtRunner()
result = runner.invoke(["parse"])
assert isinstance(result.exception, ParsingError)
assert (
"The semantic layer requires a time spine model with granularity DAY or smaller"
in result.exception.msg
)
class TestTimeSpineColumnMissing:
"""Tests that YAML with time spine column not in model errors."""
@pytest.fixture(scope="class")
def models(self):
return {
"semantic_model_people.yml": semantic_model_people_yml,
"people.sql": models_people_sql,
"metricflow_time_spine.sql": metricflow_time_spine_sql,
"metricflow_time_spine_second.sql": metricflow_time_spine_second_sql,
"time_spines.yml": time_spine_missing_column_yml,
}
def test_time_spines(self, project):
runner = dbtRunner()
result = runner.invoke(["parse"])
assert isinstance(result.exception, ParsingError)
assert (
"Time spine standard granularity column must be defined on the model."
in result.exception.msg
)
class TestTimeSpineGranularityMissing:
"""Tests that YAML with time spine column without granularity errors."""
@pytest.fixture(scope="class")
def models(self):
return {
"semantic_model_people.yml": semantic_model_people_yml,
"people.sql": models_people_sql,
"metricflow_time_spine.sql": metricflow_time_spine_sql,
"metricflow_time_spine_second.sql": metricflow_time_spine_second_sql,
"time_spines.yml": time_spine_missing_granularity_yml,
}
def test_time_spines(self, project):
runner = dbtRunner()
result = runner.invoke(["parse"])
assert isinstance(result.exception, ParsingError)
assert (
"Time spine standard granularity column must have a granularity defined."
in result.exception.msg
)

1
tests/unit/contracts/graph/test_manifest.py
View File

@@ -94,6 +94,7 @@ REQUIRED_PARSED_NODE_KEYS = frozenset(
"constraints",
"deprecation_date",
"defer_relation",
"time_spine",
}
)