Compare commits

...

7 Commits

Author SHA1 Message Date
Michelle Ark
3d12e456f8 update version node alias 2023-03-28 10:03:57 -04:00
Michelle Ark
cc3b6afa28 ParserRef.from_version 2023-03-27 21:37:40 -04:00
Michelle Ark
96b2a6f1a2 pass more integration tests 2023-03-27 19:46:47 -04:00
Michelle Ark
810cec1b03 fix unit tests, most integration tests 2023-03-27 18:49:27 -04:00
Michelle Ark
9b5acff969 Merge branch 'main' into 6866/model-versions 2023-03-27 14:01:31 -04:00
Michelle Ark
9b2f8fe4df versions - ref resolution 2023-03-27 13:48:53 -04:00
Michelle Ark
9b9d02d96e versions parsing - first pass. columns + refs don't work 2023-03-21 18:31:11 -04:00
15 changed files with 8650 additions and 92 deletions

View File

@@ -212,16 +212,19 @@ class BaseResolver(metaclass=abc.ABCMeta):
class BaseRefResolver(BaseResolver):
@abc.abstractmethod
def resolve(self, name: str, package: Optional[str] = None) -> RelationProxy:
def resolve(
self, name: str, package: Optional[str] = None, version: Optional[str] = None
) -> RelationProxy:
...
def _repack_args(self, name: str, package: Optional[str]) -> List[str]:
if package is None:
return [name]
else:
return [package, name]
def _repack_args(self, name: str, package: Optional[str], version: Optional[str]) -> List[str]:
repacked_args = [name] if package is None else [package, name]
if version:
repacked_args.append(f"version:{version}")
def validate_args(self, name: str, package: Optional[str]):
return repacked_args
def validate_args(self, name: str, package: Optional[str], version: Optional[str]):
if not isinstance(name, str):
raise CompilationError(
f"The name argument to ref() must be a string, got {type(name)}"
@@ -232,9 +235,15 @@ class BaseRefResolver(BaseResolver):
f"The package argument to ref() must be a string or None, got {type(package)}"
)
def __call__(self, *args: str) -> RelationProxy:
if version is not None and not isinstance(version, str):
raise CompilationError(
f"The version argument to ref() must be a string or None, got {type(version)}"
)
def __call__(self, *args: str, **kwargs) -> RelationProxy:
name: str
package: Optional[str] = None
version: Optional[str] = None
if len(args) == 1:
name = args[0]
@@ -242,8 +251,10 @@ class BaseRefResolver(BaseResolver):
package, name = args
else:
raise RefArgsError(node=self.model, args=args)
self.validate_args(name, package)
return self.resolve(name, package)
version = kwargs.get("version")
self.validate_args(name, package, version)
return self.resolve(name, package, version)
class BaseSourceResolver(BaseResolver):
@@ -448,8 +459,10 @@ class RuntimeDatabaseWrapper(BaseDatabaseWrapper):
# `ref` implementations
class ParseRefResolver(BaseRefResolver):
def resolve(self, name: str, package: Optional[str] = None) -> RelationProxy:
self.model.refs.append(self._repack_args(name, package))
def resolve(
self, name: str, package: Optional[str] = None, version: Optional[str] = None
) -> RelationProxy:
self.model.refs.append(self._repack_args(name, package, version))
return self.Relation.create_from(self.config, self.model)
@@ -458,15 +471,22 @@ ResolveRef = Union[Disabled, ManifestNode]
class RuntimeRefResolver(BaseRefResolver):
def resolve(self, target_name: str, target_package: Optional[str] = None) -> RelationProxy:
def resolve(
self,
target_name: str,
target_package: Optional[str] = None,
target_version: Optional[str] = None,
) -> RelationProxy:
target_model = self.manifest.resolve_ref(
target_name,
target_package,
target_version,
self.current_project,
self.model.package_name,
)
if target_model is None or isinstance(target_model, Disabled):
# TODO: add version to error
raise TargetNotFoundError(
node=self.model,
target_name=target_name,
@@ -474,7 +494,7 @@ class RuntimeRefResolver(BaseRefResolver):
target_package=target_package,
disabled=isinstance(target_model, Disabled),
)
self.validate(target_model, target_name, target_package)
self.validate(target_model, target_name, target_package, target_version)
return self.create_relation(target_model, target_name)
def create_relation(self, target_model: ManifestNode, name: str) -> RelationProxy:
@@ -485,10 +505,14 @@ class RuntimeRefResolver(BaseRefResolver):
return self.Relation.create_from(self.config, target_model)
def validate(
self, resolved: ManifestNode, target_name: str, target_package: Optional[str]
self,
resolved: ManifestNode,
target_name: str,
target_package: Optional[str],
target_version: Optional[str],
) -> None:
if resolved.unique_id not in self.model.depends_on.nodes:
args = self._repack_args(target_name, target_package)
args = self._repack_args(target_name, target_package, target_version)
raise RefBadContextError(node=self.model, args=args)
@@ -498,6 +522,7 @@ class OperationRefResolver(RuntimeRefResolver):
resolved: ManifestNode,
target_name: str,
target_package: Optional[str],
target_version: Optional[str],
) -> None:
pass

View File

@@ -145,19 +145,22 @@ class SourceLookup(dbtClassMixin):
class RefableLookup(dbtClassMixin):
# model, seed, snapshot
_lookup_types: ClassVar[set] = set(NodeType.refable())
_versioned_types: ClassVar[set] = set(NodeType.versioned())
# refables are actually unique, so the Dict[PackageName, UniqueID] will
# only ever have exactly one value, but doing 3 dict lookups instead of 1
# is not a big deal at all and retains consistency
def __init__(self, manifest: "Manifest"):
self.storage: Dict[str, Dict[PackageName, UniqueID]] = {}
self.storage: Dict[str, Dict[PackageName, List[UniqueID]]] = {}
self.populate(manifest)
def get_unique_id(self, key, package: Optional[PackageName]):
return find_unique_id_for_package(self.storage, key, package)
def get_unique_id(self, key, package: Optional[PackageName], version: Optional[str]):
return self.find_unique_id_for_package_and_version(key, package, version)
def find(self, key, package: Optional[PackageName], manifest: "Manifest"):
unique_id = self.get_unique_id(key, package)
def find(
self, key, package: Optional[PackageName], version: Optional[str], manifest: "Manifest"
):
unique_id = self.get_unique_id(key, package, version)
if unique_id is not None:
return self.perform_lookup(unique_id, manifest)
return None
@@ -166,7 +169,14 @@ class RefableLookup(dbtClassMixin):
if node.resource_type in self._lookup_types:
if node.name not in self.storage:
self.storage[node.name] = {}
self.storage[node.name][node.package_name] = node.unique_id
if node.package_name not in self.storage[node.name]:
self.storage[node.name][node.package_name] = []
if node.resource_type in self._versioned_types and node.is_latest_version:
# keep latest version at the front of unique id list
self.storage[node.name][node.package_name].insert(0, node.unique_id)
else:
self.storage[node.name][node.package_name].append(node.unique_id)
def populate(self, manifest):
for node in manifest.nodes.values():
@@ -179,6 +189,37 @@ class RefableLookup(dbtClassMixin):
)
return manifest.nodes[unique_id]
def find_unique_id_for_package_and_version(
self, key: str, package: Optional[PackageName], version: Optional[str]
):
if key not in self.storage:
return None
pkg_dct: Mapping[PackageName, List[UniqueID]] = self.storage[key]
unique_ids: Optional[List[UniqueID]] = None
if package is None:
if not pkg_dct:
return None
else:
unique_ids = next(iter(pkg_dct.values()))
elif package in pkg_dct:
unique_ids = pkg_dct[package]
else:
return None
if len(unique_ids) >= 1:
if version:
for unique_id in unique_ids:
if unique_id.endswith(f"v{version}"):
return unique_id
return None
else:
return unique_ids[0]
else:
return None
class MetricLookup(dbtClassMixin):
def __init__(self, manifest: "Manifest"):
@@ -899,6 +940,7 @@ class Manifest(MacroMethods, DataClassMessagePackMixin, dbtClassMixin):
self,
target_model_name: str,
target_model_package: Optional[str],
target_model_version: Optional[str],
current_project: str,
node_package: str,
) -> MaybeNonSource:
@@ -908,7 +950,7 @@ class Manifest(MacroMethods, DataClassMessagePackMixin, dbtClassMixin):
candidates = _search_packages(current_project, node_package, target_model_package)
for pkg in candidates:
node = self.ref_lookup.find(target_model_name, pkg, self)
node = self.ref_lookup.find(target_model_name, pkg, target_model_version, self)
if node is not None and node.config.enabled:
return node

View File

@@ -17,6 +17,7 @@ from typing import (
from dbt.dataclass_schema import dbtClassMixin, ExtensibleDbtClassMixin
# from dbt.helper_types import IncludeExclude
from dbt.clients.system import write_file
from dbt.contracts.files import FileHash
from dbt.contracts.graph.unparsed import (
@@ -392,6 +393,10 @@ class ParsedNode(NodeInfoMixin, ParsedNodeMandatory, SerializableType):
self.created_at = time.time()
self.description = patch.description
self.columns = patch.columns
# TODO - these are model specific, so is access
self.version = patch.version
self.is_latest_version = patch.is_latest_version
self.name = patch.name
# This might not be the ideal place to validate the "access" field,
# but at this point we have the information we need to properly
# validate and we don't before this.
@@ -513,6 +518,8 @@ class HookNode(CompiledNode):
class ModelNode(CompiledNode):
resource_type: NodeType = field(metadata={"restrict": [NodeType.Model]})
access: AccessType = AccessType.Protected
version: Optional[str] = None
is_latest_version: Optional[bool] = None
# TODO: rm?
@@ -1190,6 +1197,8 @@ class ParsedPatch(HasYamlMetadata, Replaceable):
class ParsedNodePatch(ParsedPatch):
columns: Dict[str, ColumnInfo]
access: Optional[str]
version: Optional[str]
is_latest_version: Optional[bool]
@dataclass

View File

@@ -142,15 +142,43 @@ class HasConfig:
config: Dict[str, Any] = field(default_factory=dict)
@dataclass
class UnparsedVersion(HasConfig, HasColumnProps):
defined_in: Optional[str] = None
columns: Sequence[Union[dbt.helper_types.IncludeExclude, UnparsedColumn]] = field(
default_factory=list
)
# TODO: all other model configs? Not sure HasColumnProps is the right thing to use here
@property
def include_exclude(self) -> dbt.helper_types.IncludeExclude:
return self._include_exclude
def __post_init__(self):
has_include_exclude = False
self._include_exclude = dbt.helper_types.IncludeExclude(include=[])
for column in self.columns:
if isinstance(column, dbt.helper_types.IncludeExclude):
if not has_include_exclude:
self._include_exclude = column
has_include_exclude = True
else:
raise ParsingError("version can have at most one include/exclude element")
@dataclass
class UnparsedAnalysisUpdate(HasConfig, HasColumnDocs, HasColumnProps, HasYamlMetadata):
access: Optional[str] = None
latest_version: Optional[str] = None
versions: Sequence[UnparsedVersion] = field(default_factory=list)
@dataclass
class UnparsedNodeUpdate(HasConfig, HasColumnTests, HasColumnAndTestProps, HasYamlMetadata):
quote_columns: Optional[bool] = None
access: Optional[str] = None
latest_version: Optional[str] = None
versions: Sequence[UnparsedVersion] = field(default_factory=list)
@dataclass

View File

@@ -101,7 +101,6 @@ class SelectorMethod(metaclass=abc.ABCMeta):
def parsed_nodes(
self, included_nodes: Set[UniqueId]
) -> Iterator[Tuple[UniqueId, ManifestNode]]:
for key, node in self.manifest.nodes.items():
unique_id = UniqueId(key)
if unique_id not in included_nodes:

View File

@@ -19,13 +19,17 @@
{% macro default__generate_alias_name(custom_alias_name=none, node=none) -%}
{%- if custom_alias_name is none -%}
{%- if custom_alias_name -%}
{{ node.name }}
{{ custom_alias_name | trim }}
{%- elif node.version -%}
{{ return(node.name ~ "_v" ~ node.version) }}
{%- else -%}
{{ custom_alias_name | trim }}
{{ node.name }}
{%- endif -%}

View File

@@ -56,6 +56,10 @@ class NodeType(StrEnum):
cls.Snapshot,
]
@classmethod
def versioned(cls) -> List["NodeType"]:
return [cls.Model]
@classmethod
def documentable(cls) -> List["NodeType"]:
return [

View File

@@ -1103,7 +1103,7 @@ def _check_resource_uniqueness(
full_node_name = str(relation)
existing_node = names_resources.get(name)
if existing_node is not None:
if existing_node is not None and existing_node.version is None:
raise dbt.exceptions.DuplicateResourceNameError(existing_node, node)
existing_alias = alias_resources.get(full_node_name)
@@ -1182,6 +1182,12 @@ def _process_refs_for_exposure(manifest: Manifest, current_project: str, exposur
target_model: Optional[Union[Disabled, ManifestNode]] = None
target_model_name: str
target_model_package: Optional[str] = None
target_model_version: Optional[str] = None
target_model_version_packed = next((r for r in ref if r.startswith("version:")), None)
if target_model_version_packed:
ref.remove(target_model_version_packed)
target_model_version = target_model_version_packed.split(":")[-1]
if len(ref) == 1:
target_model_name = ref[0]
@@ -1195,6 +1201,7 @@ def _process_refs_for_exposure(manifest: Manifest, current_project: str, exposur
target_model = manifest.resolve_ref(
target_model_name,
target_model_package,
target_model_version,
current_project,
exposure.package_name,
)
@@ -1236,6 +1243,12 @@ def _process_refs_for_metric(manifest: Manifest, current_project: str, metric: M
target_model: Optional[Union[Disabled, ManifestNode]] = None
target_model_name: str
target_model_package: Optional[str] = None
target_model_version: Optional[str] = None
target_model_version_packed = next((r for r in ref if r.startswith("version:")), None)
if target_model_version_packed:
ref.remove(target_model_version_packed)
target_model_version = target_model_version_packed.split(":")[-1]
if len(ref) == 1:
target_model_name = ref[0]
@@ -1249,6 +1262,7 @@ def _process_refs_for_metric(manifest: Manifest, current_project: str, metric: M
target_model = manifest.resolve_ref(
target_model_name,
target_model_package,
target_model_version,
current_project,
metric.package_name,
)
@@ -1342,6 +1356,12 @@ def _process_refs_for_node(manifest: Manifest, current_project: str, node: Manif
target_model: Optional[Union[Disabled, ManifestNode]] = None
target_model_name: str
target_model_package: Optional[str] = None
target_model_version: Optional[str] = None
target_model_version_raw = next((r for r in ref if r.startswith("version:")), None)
if target_model_version_raw:
ref.remove(target_model_version_raw)
target_model_version = target_model_version_raw.split(":")[-1]
if len(ref) == 1:
target_model_name = ref[0]
@@ -1352,9 +1372,14 @@ def _process_refs_for_node(manifest: Manifest, current_project: str, node: Manif
f"Refs should always be 1 or 2 arguments - got {len(ref)}"
)
if target_model_version_raw:
# add 'version:' argument back to ref TODO - is there a nicer way to do this..
ref.append(target_model_version_raw)
target_model = manifest.resolve_ref(
target_model_name,
target_model_package,
target_model_version,
current_project,
node.package_name,
)

View File

@@ -3,7 +3,7 @@ import os
import pathlib
from abc import ABCMeta, abstractmethod
from typing import Iterable, Dict, Any, Union, List, Optional, Generic, TypeVar, Type
from typing import Iterable, Dict, Any, Union, List, Optional, Generic, TypeVar, Type, Sequence
from dbt.dataclass_schema import ValidationError, dbtClassMixin
@@ -53,6 +53,7 @@ from dbt.contracts.graph.unparsed import (
UnparsedMetric,
UnparsedSourceDefinition,
UnparsedGroup,
UnparsedVersion,
)
from dbt.exceptions import (
CompilationError,
@@ -150,6 +151,21 @@ class ParserRef:
refs._add(column)
return refs
@classmethod
def from_version(
cls, base_columns: Sequence[HasColumnProps], version: UnparsedVersion
) -> "ParserRef":
refs = cls()
for base_column in base_columns:
if version.include_exclude.includes(base_column.name):
refs._add(base_column)
for column in version.columns:
if isinstance(column, UnparsedColumn):
refs._add(column)
return refs
def _trimmed(inp: str) -> str:
if len(inp) < 50:
@@ -347,7 +363,9 @@ class SchemaParser(SimpleParser[GenericTestBlock, GenericTestNode]):
"""Look up attached node for Testable target nodes other than sources. Can be None if generic test attached to SQL node with no corresponding .sql file."""
attached_node = None # type: Optional[Union[ManifestNode, GraphMemberNode]]
if not isinstance(target, UnpatchedSourceDefinition):
attached_node_unique_id = self.manifest.ref_lookup.get_unique_id(target.name, None)
attached_node_unique_id = self.manifest.ref_lookup.get_unique_id(
target.name, None, None
) # TODO: is it possible to get the version at this point?
if attached_node_unique_id:
attached_node = self.manifest.nodes[attached_node_unique_id]
else:
@@ -858,11 +876,13 @@ class NodePatchParser(NonSourceParser[NodeTarget, ParsedNodePatch], Generic[Node
docs=block.target.docs,
config=block.target.config,
access=block.target.access,
version=None, # version set from versioned model patch
is_latest_version=None, # is_latest_version set from versioned_model_patch
)
assert isinstance(self.yaml.file, SchemaSourceFile)
source_file: SchemaSourceFile = self.yaml.file
if patch.yaml_key in ["models", "seeds", "snapshots"]:
unique_id = self.manifest.ref_lookup.get_unique_id(patch.name, None)
unique_id = self.manifest.ref_lookup.get_unique_id(patch.name, None, None)
if unique_id:
resource_type = NodeType(unique_id.split(".")[0])
if resource_type.pluralize() != patch.yaml_key:
@@ -878,12 +898,68 @@ class NodePatchParser(NonSourceParser[NodeTarget, ParsedNodePatch], Generic[Node
return
elif patch.yaml_key == "analyses":
unique_id = self.manifest.analysis_lookup.get_unique_id(patch.name, None)
unique_id = self.manifest.analysis_lookup.get_unique_id(patch.name, None, None)
else:
raise DbtInternalError(
f"Unexpected yaml_key {patch.yaml_key} for patch in "
f"file {source_file.path.original_file_path}"
)
# patch versioned nodes
versions = block.target.versions
if versions:
latest_version = block.target.latest_version or max(
[version.name for version in versions]
)
for unparsed_version in versions:
versioned_model_name = (
unparsed_version.defined_in or f"{patch.name}_v{unparsed_version.name}"
)
# ref lookup without version - version is not set yet
versioned_model_unique_id = self.manifest.ref_lookup.get_unique_id(
versioned_model_name, None, None
)
if versioned_model_unique_id is None:
warn_or_error(
NoNodeForYamlKey(
patch_name=versioned_model_name,
yaml_key=patch.yaml_key,
file_path=source_file.path.original_file_path,
)
)
continue
versioned_model_node = self.manifest.nodes.pop(versioned_model_unique_id)
versioned_model_node.unique_id = (
f"model.{patch.package_name}.{patch.name}.v{unparsed_version.name}"
)
versioned_model_node.fqn[-1] = patch.name # bleugh
versioned_model_node.fqn.append(f"v{unparsed_version.name}")
self.manifest.nodes[versioned_model_node.unique_id] = versioned_model_node
# flatten columns based on include/exclude
version_refs: ParserRef = ParserRef.from_version(
block.target.columns, unparsed_version
)
versioned_model_patch = ParsedNodePatch(
name=patch.name,
original_file_path=versioned_model_node.original_file_path,
yaml_key=patch.yaml_key,
package_name=versioned_model_node.package_name,
description=unparsed_version.description or versioned_model_node.description,
columns=version_refs.column_info,
meta=unparsed_version.meta or versioned_model_node.meta,
docs=unparsed_version.docs or versioned_model_node.docs,
config=unparsed_version.config,
access=versioned_model_node.access,
version=unparsed_version.name,
is_latest_version=latest_version == unparsed_version.name,
)
versioned_model_node.patch(versioned_model_patch)
self.patch_node_config(versioned_model_node, versioned_model_patch)
self.manifest.rebuild_ref_lookup() # TODO: is this necessary?
return
# handle disabled nodes
if unique_id is None:
# Node might be disabled. Following call returns list of matching disabled nodes

View File

@@ -216,12 +216,12 @@
},
"dbt_version": {
"type": "string",
"default": "1.5.0b1"
"default": "1.5.0b4"
},
"generated_at": {
"type": "string",
"format": "date-time",
"default": "2023-02-27T20:52:53.570061Z"
"default": "2023-03-27T23:35:23.488098Z"
},
"invocation_id": {
"oneOf": [
@@ -232,7 +232,7 @@
"type": "null"
}
],
"default": "f578f819-6554-4f6f-8af4-635e35249cb2"
"default": "80b7b864-e029-4265-8c39-b478ec9fa492"
},
"env": {
"type": "object",
@@ -449,7 +449,7 @@
},
"created_at": {
"type": "number",
"default": 1677531173.5741172
"default": 1679960123.4897149
},
"config_call_dict": {
"type": "object",
@@ -816,35 +816,11 @@
]
},
"constraints": {
"oneOf": [
{
"type": "array",
"items": {
"type": "object",
"required": ["type"],
"properties": {
"type": {
"type": "string"
},
"name": {
"type": "string"
},
"expression": {
"type": "string"
},
"warn_unenforced": {
"type": "boolean"
},
"warn_unsupported": {
"type": "boolean"
}
}
}
},
{
"type": "null"
}
]
"type": "array",
"items": {
"$ref": "#/definitions/ColumnLevelConstraint"
},
"default": []
},
"quote": {
"oneOf": [
@@ -867,6 +843,55 @@
"additionalProperties": true,
"description": "Used in all ManifestNodes and SourceDefinition"
},
"ColumnLevelConstraint": {
"type": "object",
"required": [
"type"
],
"properties": {
"type": {
"type": "string",
"enum": [
"check",
"not_null",
"unique",
"primary_key",
"foreign_key",
"custom"
]
},
"name": {
"oneOf": [
{
"type": "string"
},
{
"type": "null"
}
]
},
"expression": {
"oneOf": [
{
"type": "string"
},
{
"type": "null"
}
]
},
"warn_unenforced": {
"type": "boolean",
"default": true
},
"warn_unsupported": {
"type": "boolean",
"default": true
}
},
"additionalProperties": false,
"description": "ColumnLevelConstraint(type: dbt.contracts.graph.nodes.ConstraintType, name: Optional[str] = None, expression: Optional[str] = None, warn_unenforced: bool = True, warn_unsupported: bool = True)"
},
"DependsOn": {
"type": "object",
"required": [],
@@ -1056,7 +1081,7 @@
},
"created_at": {
"type": "number",
"default": 1677531173.577135
"default": 1679960123.490774
},
"config_call_dict": {
"type": "object",
@@ -1441,7 +1466,7 @@
},
"created_at": {
"type": "number",
"default": 1677531173.578816
"default": 1679960123.491379
},
"config_call_dict": {
"type": "object",
@@ -1714,7 +1739,7 @@
},
"created_at": {
"type": "number",
"default": 1677531173.580534
"default": 1679960123.491997
},
"config_call_dict": {
"type": "object",
@@ -1822,10 +1847,30 @@
"public"
],
"default": "protected"
},
"version": {
"oneOf": [
{
"type": "string"
},
{
"type": "null"
}
]
},
"is_latest_version": {
"oneOf": [
{
"type": "boolean"
},
{
"type": "null"
}
]
}
},
"additionalProperties": false,
"description": "ModelNode(database: Optional[str], schema: str, name: str, resource_type: dbt.node_types.NodeType, package_name: str, path: str, original_file_path: str, unique_id: str, fqn: List[str], alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.NodeConfig = <factory>, _event_status: Dict[str, Any] = <factory>, tags: List[str] = <factory>, description: str = '', columns: Dict[str, dbt.contracts.graph.nodes.ColumnInfo] = <factory>, meta: Dict[str, Any] = <factory>, group: Optional[str] = None, docs: dbt.contracts.graph.unparsed.Docs = <factory>, patch_path: Optional[str] = None, build_path: Optional[str] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = <factory>, created_at: float = <factory>, config_call_dict: Dict[str, Any] = <factory>, relation_name: Optional[str] = None, raw_code: str = '', language: str = 'sql', refs: List[List[str]] = <factory>, sources: List[List[str]] = <factory>, metrics: List[List[str]] = <factory>, depends_on: dbt.contracts.graph.nodes.DependsOn = <factory>, compiled_path: Optional[str] = None, compiled: bool = False, compiled_code: Optional[str] = None, extra_ctes_injected: bool = False, extra_ctes: List[dbt.contracts.graph.nodes.InjectedCTE] = <factory>, _pre_injected_sql: Optional[str] = None, contract: bool = False, access: dbt.node_types.AccessType = <AccessType.Protected: 'protected'>)"
"description": "ModelNode(database: Optional[str], schema: str, name: str, resource_type: dbt.node_types.NodeType, package_name: str, path: str, original_file_path: str, unique_id: str, fqn: List[str], alias: str, checksum: dbt.contracts.files.FileHash, config: dbt.contracts.graph.model_config.NodeConfig = <factory>, _event_status: Dict[str, Any] = <factory>, tags: List[str] = <factory>, description: str = '', columns: Dict[str, dbt.contracts.graph.nodes.ColumnInfo] = <factory>, meta: Dict[str, Any] = <factory>, group: Optional[str] = None, docs: dbt.contracts.graph.unparsed.Docs = <factory>, patch_path: Optional[str] = None, build_path: Optional[str] = None, deferred: bool = False, unrendered_config: Dict[str, Any] = <factory>, created_at: float = <factory>, config_call_dict: Dict[str, Any] = <factory>, relation_name: Optional[str] = None, raw_code: str = '', language: str = 'sql', refs: List[List[str]] = <factory>, sources: List[List[str]] = <factory>, metrics: List[List[str]] = <factory>, depends_on: dbt.contracts.graph.nodes.DependsOn = <factory>, compiled_path: Optional[str] = None, compiled: bool = False, compiled_code: Optional[str] = None, extra_ctes_injected: bool = False, extra_ctes: List[dbt.contracts.graph.nodes.InjectedCTE] = <factory>, _pre_injected_sql: Optional[str] = None, contract: bool = False, access: dbt.node_types.AccessType = <AccessType.Protected: 'protected'>, version: Optional[str] = None, is_latest_version: Optional[bool] = None)"
},
"RPCNode": {
"type": "object",
@@ -1986,7 +2031,7 @@
},
"created_at": {
"type": "number",
"default": 1677531173.5821729
"default": 1679960123.492646
},
"config_call_dict": {
"type": "object",
@@ -2249,7 +2294,7 @@
},
"created_at": {
"type": "number",
"default": 1677531173.5838
"default": 1679960123.4932292
},
"config_call_dict": {
"type": "object",
@@ -2507,7 +2552,7 @@
},
"created_at": {
"type": "number",
"default": 1677531173.5856462
"default": 1679960123.493912
},
"config_call_dict": {
"type": "object",
@@ -2802,7 +2847,7 @@
},
"created_at": {
"type": "number",
"default": 1677531173.588902
"default": 1679960123.495002
},
"config_call_dict": {
"type": "object",
@@ -3285,7 +3330,7 @@
},
"created_at": {
"type": "number",
"default": 1677531173.591784
"default": 1679960123.495994
},
"config_call_dict": {
"type": "object",
@@ -3685,7 +3730,7 @@
},
"created_at": {
"type": "number",
"default": 1677531173.594429
"default": 1679960123.497026
}
},
"additionalProperties": false,
@@ -3795,12 +3840,12 @@
},
"dbt_version": {
"type": "string",
"default": "1.5.0b1"
"default": "1.5.0b4"
},
"generated_at": {
"type": "string",
"format": "date-time",
"default": "2023-02-27T20:52:53.564664Z"
"default": "2023-03-27T23:35:23.486009Z"
},
"invocation_id": {
"oneOf": [
@@ -3811,7 +3856,7 @@
"type": "null"
}
],
"default": "f578f819-6554-4f6f-8af4-635e35249cb2"
"default": "80b7b864-e029-4265-8c39-b478ec9fa492"
},
"env": {
"type": "object",
@@ -3822,7 +3867,7 @@
}
},
"additionalProperties": false,
"description": "FreshnessMetadata(dbt_schema_version: str = <factory>, dbt_version: str = '1.5.0b1', generated_at: datetime.datetime = <factory>, invocation_id: Optional[str] = <factory>, env: Dict[str, str] = <factory>)"
"description": "FreshnessMetadata(dbt_schema_version: str = <factory>, dbt_version: str = '1.5.0b4', generated_at: datetime.datetime = <factory>, invocation_id: Optional[str] = <factory>, env: Dict[str, str] = <factory>)"
},
"SourceFreshnessRuntimeError": {
"type": "object",
@@ -4164,7 +4209,7 @@
},
"created_at": {
"type": "number",
"default": 1677531173.595732
"default": 1679960123.497319
},
"supported_languages": {
"oneOf": [
@@ -4407,7 +4452,7 @@
},
"created_at": {
"type": "number",
"default": 1677531173.597582
"default": 1679960123.4979782
}
},
"additionalProperties": false,
@@ -4630,7 +4675,7 @@
},
"created_at": {
"type": "number",
"default": 1677531173.599355
"default": 1679960123.498676
},
"group": {
"oneOf": [

View File

@@ -781,6 +781,8 @@ def basic_parsed_model_patch_dict():
},
"config": {},
"access": "public",
"version": "1",
"is_latest_version": True,
}
@@ -797,6 +799,8 @@ def basic_parsed_model_patch_object():
meta={},
config={},
access="public",
version="1",
is_latest_version=True,
)
@@ -829,6 +833,8 @@ def patched_model_object():
checksum=FileHash.from_contents(""),
unrendered_config={},
access=AccessType.Public,
version="1",
is_latest_version=True,
)
@@ -1911,6 +1917,8 @@ def populated_parsed_node_patch_dict():
"package_name": "test",
"config": {},
"access": "public",
"version": "1",
"is_latest_version": True,
}
@@ -1927,6 +1935,8 @@ def populated_parsed_node_patch_object():
docs=Docs(show=False),
config={},
access="public",
version="1",
is_latest_version=True,
)

View File

@@ -418,6 +418,7 @@ class TestUnparsedNodeUpdate(ContractTestCase):
"meta": {},
"config": {},
"constraints": [],
"versions": [],
}
self.assert_from_dict(minimum, from_dict)
self.assert_to_dict(minimum, to_dict)
@@ -456,6 +457,7 @@ class TestUnparsedNodeUpdate(ContractTestCase):
"tests": ["table_test"],
"meta": {"key": ["value1", "value2"]},
"constraints": [],
"versions": [],
"columns": [
{
"name": "x",

View File

@@ -88,6 +88,8 @@ REQUIRED_PARSED_NODE_KEYS = frozenset(
"relation_name",
"contract",
"access",
"version",
"is_latest_version",
}
)
@@ -1279,14 +1281,14 @@ def test_find_materialization_by_name(macros, adapter_type, expected):
assert result.package_name == expected_package
FindNodeSpec = namedtuple("FindNodeSpec", "nodes,sources,package,expected")
FindNodeSpec = namedtuple("FindNodeSpec", "nodes,sources,package,version,expected")
def _refable_parameter_sets():
sets = [
# empties
FindNodeSpec(nodes=[], sources=[], package=None, expected=None),
FindNodeSpec(nodes=[], sources=[], package="root", expected=None),
FindNodeSpec(nodes=[], sources=[], package=None, version=None, expected=None),
FindNodeSpec(nodes=[], sources=[], package="root", version=None, expected=None),
]
sets.extend(
# only one model, no package specified -> find it in any package
@@ -1294,6 +1296,7 @@ def _refable_parameter_sets():
nodes=[MockNode(project, "my_model")],
sources=[],
package=None,
version=None,
expected=(project, "my_model"),
)
for project in ["root", "dep"]
@@ -1305,12 +1308,14 @@ def _refable_parameter_sets():
nodes=[MockNode("root", "my_model")],
sources=[],
package="root",
version=None,
expected=("root", "my_model"),
),
FindNodeSpec(
nodes=[MockNode("dep", "my_model")],
sources=[],
package="root",
version=None,
expected=None,
),
# a source with that name exists, but not a refable
@@ -1318,6 +1323,7 @@ def _refable_parameter_sets():
nodes=[],
sources=[MockSource("root", "my_source", "my_model")],
package=None,
version=None,
expected=None,
),
# a source with that name exists, and a refable
@@ -1325,18 +1331,21 @@ def _refable_parameter_sets():
nodes=[MockNode("root", "my_model")],
sources=[MockSource("root", "my_source", "my_model")],
package=None,
version=None,
expected=("root", "my_model"),
),
FindNodeSpec(
nodes=[MockNode("root", "my_model")],
sources=[MockSource("root", "my_source", "my_model")],
package="root",
version=None,
expected=("root", "my_model"),
),
FindNodeSpec(
nodes=[MockNode("root", "my_model")],
sources=[MockSource("root", "my_source", "my_model")],
package="dep",
version=None,
expected=None,
),
]
@@ -1353,15 +1362,16 @@ def id_nodes(arg):
@pytest.mark.parametrize(
"nodes,sources,package,expected",
"nodes,sources,package,version,expected",
_refable_parameter_sets(),
ids=id_nodes,
)
def test_resolve_ref(nodes, sources, package, expected):
def test_resolve_ref(nodes, sources, package, version, expected):
manifest = make_manifest(nodes=nodes, sources=sources)
result = manifest.resolve_ref(
target_model_name="my_model",
target_model_package=package,
target_model_version=version,
current_project="root",
node_package="root",
)
@@ -1378,8 +1388,8 @@ def test_resolve_ref(nodes, sources, package, expected):
def _source_parameter_sets():
sets = [
# empties
FindNodeSpec(nodes=[], sources=[], package="dep", expected=None),
FindNodeSpec(nodes=[], sources=[], package="root", expected=None),
FindNodeSpec(nodes=[], sources=[], package="dep", version=None, expected=None),
FindNodeSpec(nodes=[], sources=[], package="root", version=None, expected=None),
]
sets.extend(
# models with the name, but not sources
@@ -1387,6 +1397,7 @@ def _source_parameter_sets():
nodes=[MockNode("root", name)],
sources=[],
package=project,
version=None,
expected=None,
)
for project in ("root", "dep")
@@ -1398,6 +1409,7 @@ def _source_parameter_sets():
nodes=[MockNode("root", "my_source"), MockNode("root", "my_table")],
sources=[MockSource("root", "my_source", "my_table")],
package=project,
version=None,
expected=("root", "my_source", "my_table"),
)
for project in ("root", "dep")
@@ -1408,6 +1420,7 @@ def _source_parameter_sets():
nodes=[],
sources=[MockSource("root", "my_other_source", "my_table")],
package=project,
version=None,
expected=None,
)
for project in ("root", "dep")
@@ -1418,6 +1431,7 @@ def _source_parameter_sets():
nodes=[],
sources=[MockSource("root", "my_source", "my_other_table")],
package=project,
version=None,
expected=None,
)
for project in ("root", "dep")
@@ -1428,6 +1442,7 @@ def _source_parameter_sets():
nodes=[],
sources=[MockSource("other", "my_source", "my_table")],
package="root",
version=None,
expected=("other", "my_source", "my_table"),
)
)
@@ -1437,6 +1452,7 @@ def _source_parameter_sets():
nodes=[],
sources=[MockSource("root", "my_source", "my_table")],
package=project,
version=None,
expected=("root", "my_source", "my_table"),
)
for project in ("root", "dep")
@@ -1446,11 +1462,11 @@ def _source_parameter_sets():
@pytest.mark.parametrize(
"nodes,sources,package,expected",
"nodes,sources,package,version,expected",
_source_parameter_sets(),
ids=id_nodes,
)
def test_resolve_source(nodes, sources, package, expected):
def test_resolve_source(nodes, sources, package, version, expected):
manifest = make_manifest(nodes=nodes, sources=sources)
result = manifest.resolve_source(
target_source_name="my_source",

File diff suppressed because one or more lines are too long

View File

@@ -332,6 +332,8 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False):
"checksum": checksum_file(model_sql_path),
"unrendered_config": unrendered_model_config,
"access": "protected",
"version": None,
"is_latest_version": None,
},
"model.test.second_model": {
"compiled_path": os.path.join(compiled_model_path, "second_model.sql"),
@@ -421,6 +423,8 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False):
"checksum": checksum_file(second_model_sql_path),
"unrendered_config": unrendered_second_config,
"access": "protected",
"version": None,
"is_latest_version": None,
},
"seed.test.seed": {
"build_path": None,
@@ -934,6 +938,8 @@ def expected_references_manifest(project):
"checksum": checksum_file(ephemeral_copy_path),
"unrendered_config": get_unrendered_model_config(materialized="ephemeral"),
"access": "protected",
"version": None,
"is_latest_version": None,
},
"model.test.ephemeral_summary": {
"alias": "ephemeral_summary",
@@ -996,6 +1002,8 @@ def expected_references_manifest(project):
materialized="table", group="test_group"
),
"access": "protected",
"version": None,
"is_latest_version": None,
},
"model.test.view_summary": {
"alias": "view_summary",
@@ -1054,6 +1062,8 @@ def expected_references_manifest(project):
"checksum": checksum_file(view_summary_path),
"unrendered_config": get_unrendered_model_config(materialized="view"),
"access": "protected",
"version": None,
"is_latest_version": None,
},
"seed.test.seed": {
"alias": "seed",