forked from repo-mirrors/dbt-core
Compare commits
11 Commits
main
...
depends-on
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
a1466420e2 | ||
|
|
0744fd7aab | ||
|
|
73896cad8c | ||
|
|
f37ec40d7c | ||
|
|
db6a11821a | ||
|
|
b7d7ad2652 | ||
|
|
a1bc80ce0d | ||
|
|
d6adb49737 | ||
|
|
22621aca64 | ||
|
|
8ff33f9ef3 | ||
|
|
0a718add93 |
7
.changes/unreleased/Features-20240729-173203.yaml
Normal file
7
.changes/unreleased/Features-20240729-173203.yaml
Normal file
@@ -0,0 +1,7 @@
|
||||
kind: Features
|
||||
body: Include models that depend on changed vars in state:modified, add state:modified.vars
|
||||
selection method
|
||||
time: 2024-07-29T17:32:03.368508-04:00
|
||||
custom:
|
||||
Author: michelleark
|
||||
Issue: "4304"
|
||||
6
.changes/unreleased/Fixes-20240923-190758.yaml
Normal file
6
.changes/unreleased/Fixes-20240923-190758.yaml
Normal file
@@ -0,0 +1,6 @@
|
||||
kind: Fixes
|
||||
body: Allow singular tests to be documented in properties.yml
|
||||
time: 2024-09-23T19:07:58.151069+01:00
|
||||
custom:
|
||||
Author: aranke
|
||||
Issue: "9005"
|
||||
@@ -197,6 +197,7 @@ class ParsedResource(ParsedResourceMandatory):
|
||||
unrendered_config_call_dict: Dict[str, Any] = field(default_factory=dict)
|
||||
relation_name: Optional[str] = None
|
||||
raw_code: str = ""
|
||||
vars: Dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
def __post_serialize__(self, dct: Dict, context: Optional[Dict] = None):
|
||||
dct = super().__post_serialize__(dct, context)
|
||||
|
||||
@@ -41,6 +41,7 @@ class Exposure(GraphResource):
|
||||
tags: List[str] = field(default_factory=list)
|
||||
config: ExposureConfig = field(default_factory=ExposureConfig)
|
||||
unrendered_config: Dict[str, Any] = field(default_factory=dict)
|
||||
vars: Dict[str, Any] = field(default_factory=dict)
|
||||
url: Optional[str] = None
|
||||
depends_on: DependsOn = field(default_factory=DependsOn)
|
||||
refs: List[RefArgs] = field(default_factory=list)
|
||||
|
||||
@@ -69,5 +69,6 @@ class SourceDefinition(ParsedSourceMandatory):
|
||||
config: SourceConfig = field(default_factory=SourceConfig)
|
||||
patch_path: Optional[str] = None
|
||||
unrendered_config: Dict[str, Any] = field(default_factory=dict)
|
||||
vars: Dict[str, Any] = field(default_factory=dict)
|
||||
relation_name: Optional[str] = None
|
||||
created_at: float = field(default_factory=lambda: time.time())
|
||||
|
||||
@@ -158,14 +158,8 @@ def _parse_versions(versions: Union[List[str], str]) -> List[VersionSpecifier]:
|
||||
return [VersionSpecifier.from_version_string(v) for v in versions]
|
||||
|
||||
|
||||
def _all_source_paths(
|
||||
model_paths: List[str],
|
||||
seed_paths: List[str],
|
||||
snapshot_paths: List[str],
|
||||
analysis_paths: List[str],
|
||||
macro_paths: List[str],
|
||||
) -> List[str]:
|
||||
paths = chain(model_paths, seed_paths, snapshot_paths, analysis_paths, macro_paths)
|
||||
def _all_source_paths(*args: List[str]) -> List[str]:
|
||||
paths = chain(*args)
|
||||
# Strip trailing slashes since the path is the same even though the name is not
|
||||
stripped_paths = map(lambda s: s.rstrip("/"), paths)
|
||||
return list(set(stripped_paths))
|
||||
@@ -409,7 +403,7 @@ class PartialProject(RenderComponents):
|
||||
snapshot_paths: List[str] = value_or(cfg.snapshot_paths, ["snapshots"])
|
||||
|
||||
all_source_paths: List[str] = _all_source_paths(
|
||||
model_paths, seed_paths, snapshot_paths, analysis_paths, macro_paths
|
||||
model_paths, seed_paths, snapshot_paths, analysis_paths, macro_paths, test_paths
|
||||
)
|
||||
|
||||
docs_paths: List[str] = value_or(cfg.docs_paths, all_source_paths)
|
||||
@@ -652,6 +646,7 @@ class Project:
|
||||
self.snapshot_paths,
|
||||
self.analysis_paths,
|
||||
self.macro_paths,
|
||||
self.test_paths,
|
||||
)
|
||||
|
||||
@property
|
||||
|
||||
@@ -31,23 +31,35 @@ class FQNLookup:
|
||||
self.resource_type = NodeType.Model
|
||||
|
||||
|
||||
class SchemaYamlVars:
|
||||
def __init__(self):
|
||||
self.env_vars = {}
|
||||
self.vars = {}
|
||||
|
||||
|
||||
class ConfiguredVar(Var):
|
||||
def __init__(
|
||||
self,
|
||||
context: Dict[str, Any],
|
||||
config: AdapterRequiredConfig,
|
||||
project_name: str,
|
||||
schema_yaml_vars: Optional[SchemaYamlVars] = None,
|
||||
):
|
||||
super().__init__(context, config.cli_vars)
|
||||
self._config = config
|
||||
self._project_name = project_name
|
||||
self.schema_yaml_vars = schema_yaml_vars
|
||||
|
||||
def __call__(self, var_name, default=Var._VAR_NOTSET):
|
||||
my_config = self._config.load_dependencies()[self._project_name]
|
||||
|
||||
var_found = False
|
||||
var_value = None
|
||||
|
||||
# cli vars > active project > local project
|
||||
if var_name in self._config.cli_vars:
|
||||
return self._config.cli_vars[var_name]
|
||||
var_found = True
|
||||
var_value = self._config.cli_vars[var_name]
|
||||
|
||||
adapter_type = self._config.credentials.type
|
||||
lookup = FQNLookup(self._project_name)
|
||||
@@ -58,19 +70,21 @@ class ConfiguredVar(Var):
|
||||
all_vars.add(my_config.vars.vars_for(lookup, adapter_type))
|
||||
all_vars.add(active_vars)
|
||||
|
||||
if var_name in all_vars:
|
||||
return all_vars[var_name]
|
||||
if not var_found and var_name in all_vars:
|
||||
var_found = True
|
||||
var_value = all_vars[var_name]
|
||||
|
||||
if default is not Var._VAR_NOTSET:
|
||||
return default
|
||||
if not var_found and default is not Var._VAR_NOTSET:
|
||||
var_found = True
|
||||
var_value = default
|
||||
|
||||
return self.get_missing_var(var_name)
|
||||
if not var_found:
|
||||
return self.get_missing_var(var_name)
|
||||
else:
|
||||
if self.schema_yaml_vars:
|
||||
self.schema_yaml_vars.vars[var_name] = var_value
|
||||
|
||||
|
||||
class SchemaYamlVars:
|
||||
def __init__(self):
|
||||
self.env_vars = {}
|
||||
self.vars = {}
|
||||
return var_value
|
||||
|
||||
|
||||
class SchemaYamlContext(ConfiguredContext):
|
||||
@@ -82,7 +96,7 @@ class SchemaYamlContext(ConfiguredContext):
|
||||
|
||||
@contextproperty()
|
||||
def var(self) -> ConfiguredVar:
|
||||
return ConfiguredVar(self._ctx, self.config, self._project_name)
|
||||
return ConfiguredVar(self._ctx, self.config, self._project_name, self.schema_yaml_vars)
|
||||
|
||||
@contextmember()
|
||||
def env_var(self, var: str, default: Optional[str] = None) -> str:
|
||||
|
||||
@@ -790,6 +790,14 @@ class ParseVar(ModelConfiguredVar):
|
||||
# in the parser, just always return None.
|
||||
return None
|
||||
|
||||
def __call__(self, var_name: str, default: Any = ModelConfiguredVar._VAR_NOTSET) -> Any:
|
||||
var_value = super().__call__(var_name, default)
|
||||
|
||||
if self._node and hasattr(self._node, "vars"):
|
||||
self._node.vars[var_name] = var_value
|
||||
|
||||
return var_value
|
||||
|
||||
|
||||
class RuntimeVar(ModelConfiguredVar):
|
||||
pass
|
||||
|
||||
@@ -212,6 +212,7 @@ class SchemaSourceFile(BaseSourceFile):
|
||||
# created too, but those are in 'sources'
|
||||
sop: List[SourceKey] = field(default_factory=list)
|
||||
env_vars: Dict[str, Any] = field(default_factory=dict)
|
||||
vars: Dict[str, Any] = field(default_factory=dict)
|
||||
unrendered_configs: Dict[str, Any] = field(default_factory=dict)
|
||||
pp_dict: Optional[Dict[str, Any]] = None
|
||||
pp_test_index: Optional[Dict[str, Any]] = None
|
||||
@@ -318,6 +319,22 @@ class SchemaSourceFile(BaseSourceFile):
|
||||
test_ids.extend(self.data_tests[key][name])
|
||||
return test_ids
|
||||
|
||||
def add_vars(self, vars: Dict[str, Any], yaml_key: str, name: str) -> None:
|
||||
if yaml_key not in self.vars:
|
||||
self.vars[yaml_key] = {}
|
||||
|
||||
if name not in self.vars[yaml_key]:
|
||||
self.vars[yaml_key][name] = vars
|
||||
|
||||
def get_vars(self, yaml_key: str, name: str) -> Dict[str, Any]:
|
||||
if yaml_key not in self.vars:
|
||||
return {}
|
||||
|
||||
if name not in self.vars[yaml_key]:
|
||||
return {}
|
||||
|
||||
return self.vars[yaml_key][name]
|
||||
|
||||
def add_unrendered_config(self, unrendered_config, yaml_key, name, version=None):
|
||||
versioned_name = f"{name}_v{version}" if version is not None else name
|
||||
|
||||
|
||||
@@ -58,6 +58,7 @@ from dbt.contracts.graph.nodes import (
|
||||
SavedQuery,
|
||||
SeedNode,
|
||||
SemanticModel,
|
||||
SingularTestNode,
|
||||
SourceDefinition,
|
||||
UnitTestDefinition,
|
||||
UnitTestFileFixture,
|
||||
@@ -89,7 +90,7 @@ DocName = str
|
||||
RefName = str
|
||||
|
||||
|
||||
def find_unique_id_for_package(storage, key, package: Optional[PackageName]):
|
||||
def find_unique_id_for_package(storage, key, package: Optional[PackageName]) -> Optional[UniqueID]:
|
||||
if key not in storage:
|
||||
return None
|
||||
|
||||
@@ -470,6 +471,43 @@ class AnalysisLookup(RefableLookup):
|
||||
_versioned_types: ClassVar[set] = set()
|
||||
|
||||
|
||||
class SingularTestLookup(dbtClassMixin):
|
||||
def __init__(self, manifest: "Manifest") -> None:
|
||||
self.storage: Dict[str, Dict[PackageName, UniqueID]] = {}
|
||||
self.populate(manifest)
|
||||
|
||||
def get_unique_id(self, search_name, package: Optional[PackageName]) -> Optional[UniqueID]:
|
||||
return find_unique_id_for_package(self.storage, search_name, package)
|
||||
|
||||
def find(
|
||||
self, search_name, package: Optional[PackageName], manifest: "Manifest"
|
||||
) -> Optional[SingularTestNode]:
|
||||
unique_id = self.get_unique_id(search_name, package)
|
||||
if unique_id is not None:
|
||||
return self.perform_lookup(unique_id, manifest)
|
||||
return None
|
||||
|
||||
def add_singular_test(self, source: SingularTestNode) -> None:
|
||||
if source.search_name not in self.storage:
|
||||
self.storage[source.search_name] = {}
|
||||
|
||||
self.storage[source.search_name][source.package_name] = source.unique_id
|
||||
|
||||
def populate(self, manifest: "Manifest") -> None:
|
||||
for node in manifest.nodes.values():
|
||||
if isinstance(node, SingularTestNode):
|
||||
self.add_singular_test(node)
|
||||
|
||||
def perform_lookup(self, unique_id: UniqueID, manifest: "Manifest") -> SingularTestNode:
|
||||
if unique_id not in manifest.nodes:
|
||||
raise dbt_common.exceptions.DbtInternalError(
|
||||
f"Singular test {unique_id} found in cache but not found in manifest"
|
||||
)
|
||||
node = manifest.nodes[unique_id]
|
||||
assert isinstance(node, SingularTestNode)
|
||||
return node
|
||||
|
||||
|
||||
def _packages_to_search(
|
||||
current_project: str,
|
||||
node_package: str,
|
||||
@@ -869,6 +907,9 @@ class Manifest(MacroMethods, dbtClassMixin):
|
||||
_analysis_lookup: Optional[AnalysisLookup] = field(
|
||||
default=None, metadata={"serialize": lambda x: None, "deserialize": lambda x: None}
|
||||
)
|
||||
_singular_test_lookup: Optional[SingularTestLookup] = field(
|
||||
default=None, metadata={"serialize": lambda x: None, "deserialize": lambda x: None}
|
||||
)
|
||||
_parsing_info: ParsingInfo = field(
|
||||
default_factory=ParsingInfo,
|
||||
metadata={"serialize": lambda x: None, "deserialize": lambda x: None},
|
||||
@@ -1264,6 +1305,12 @@ class Manifest(MacroMethods, dbtClassMixin):
|
||||
self._analysis_lookup = AnalysisLookup(self)
|
||||
return self._analysis_lookup
|
||||
|
||||
@property
|
||||
def singular_test_lookup(self) -> SingularTestLookup:
|
||||
if self._singular_test_lookup is None:
|
||||
self._singular_test_lookup = SingularTestLookup(self)
|
||||
return self._singular_test_lookup
|
||||
|
||||
@property
|
||||
def external_node_unique_ids(self):
|
||||
return [node.unique_id for node in self.nodes.values() if node.is_external_node]
|
||||
@@ -1708,6 +1755,7 @@ class Manifest(MacroMethods, dbtClassMixin):
|
||||
self._semantic_model_by_measure_lookup,
|
||||
self._disabled_lookup,
|
||||
self._analysis_lookup,
|
||||
self._singular_test_lookup,
|
||||
)
|
||||
return self.__class__, args
|
||||
|
||||
|
||||
@@ -369,6 +369,12 @@ class ParsedNode(ParsedResource, NodeInfoMixin, ParsedNodeMandatory, Serializabl
|
||||
# This would only apply to seeds
|
||||
return True
|
||||
|
||||
def same_vars(self, old) -> bool:
|
||||
if get_flags().state_modified_compare_vars:
|
||||
return self.vars == old.vars
|
||||
else:
|
||||
return True
|
||||
|
||||
def same_contents(self, old, adapter_type) -> bool:
|
||||
if old is None:
|
||||
return False
|
||||
@@ -382,6 +388,7 @@ class ParsedNode(ParsedResource, NodeInfoMixin, ParsedNodeMandatory, Serializabl
|
||||
and self.same_persisted_description(old)
|
||||
and self.same_fqn(old)
|
||||
and self.same_database_representation(old)
|
||||
and self.same_vars(old)
|
||||
and same_contract
|
||||
and True
|
||||
)
|
||||
@@ -1251,6 +1258,12 @@ class SourceDefinition(
|
||||
old.unrendered_config,
|
||||
)
|
||||
|
||||
def same_vars(self, other: "SourceDefinition") -> bool:
|
||||
if get_flags().state_modified_compare_vars:
|
||||
return self.vars == other.vars
|
||||
else:
|
||||
return True
|
||||
|
||||
def same_contents(self, old: Optional["SourceDefinition"]) -> bool:
|
||||
# existing when it didn't before is a change!
|
||||
if old is None:
|
||||
@@ -1271,6 +1284,7 @@ class SourceDefinition(
|
||||
and self.same_quoting(old)
|
||||
and self.same_freshness(old)
|
||||
and self.same_external(old)
|
||||
and self.same_vars(old)
|
||||
and True
|
||||
)
|
||||
|
||||
@@ -1367,6 +1381,12 @@ class Exposure(GraphNode, ExposureResource):
|
||||
old.unrendered_config,
|
||||
)
|
||||
|
||||
def same_vars(self, old: "Exposure") -> bool:
|
||||
if get_flags().state_modified_compare_vars:
|
||||
return self.vars == old.vars
|
||||
else:
|
||||
return True
|
||||
|
||||
def same_contents(self, old: Optional["Exposure"]) -> bool:
|
||||
# existing when it didn't before is a change!
|
||||
# metadata/tags changes are not "changes"
|
||||
@@ -1383,6 +1403,7 @@ class Exposure(GraphNode, ExposureResource):
|
||||
and self.same_label(old)
|
||||
and self.same_depends_on(old)
|
||||
and self.same_config(old)
|
||||
and self.same_vars(old)
|
||||
and True
|
||||
)
|
||||
|
||||
@@ -1634,6 +1655,7 @@ class ParsedNodePatch(ParsedPatch):
|
||||
latest_version: Optional[NodeVersion]
|
||||
constraints: List[Dict[str, Any]]
|
||||
deprecation_date: Optional[datetime]
|
||||
vars: Dict[str, Any]
|
||||
time_spine: Optional[TimeSpine] = None
|
||||
|
||||
|
||||
@@ -1642,6 +1664,11 @@ class ParsedMacroPatch(ParsedPatch):
|
||||
arguments: List[MacroArgument] = field(default_factory=list)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ParsedSingularTestPatch(ParsedPatch):
|
||||
pass
|
||||
|
||||
|
||||
# ====================================
|
||||
# Node unions/categories
|
||||
# ====================================
|
||||
|
||||
@@ -202,6 +202,11 @@ class UnparsedAnalysisUpdate(HasConfig, HasColumnDocs, HasColumnProps, HasYamlMe
|
||||
access: Optional[str] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class UnparsedSingularTestUpdate(HasConfig, HasColumnProps, HasYamlMetadata):
|
||||
pass
|
||||
|
||||
|
||||
@dataclass
|
||||
class UnparsedNodeUpdate(HasConfig, HasColumnTests, HasColumnAndTestProps, HasYamlMetadata):
|
||||
quote_columns: Optional[bool] = None
|
||||
|
||||
@@ -342,6 +342,7 @@ class ProjectFlags(ExtensibleDbtClassMixin):
|
||||
require_resource_names_without_spaces: bool = False
|
||||
source_freshness_run_project_hooks: bool = False
|
||||
state_modified_compare_more_unrendered_values: bool = False
|
||||
state_modified_compare_vars: bool = False
|
||||
|
||||
@property
|
||||
def project_only_flags(self) -> Dict[str, Any]:
|
||||
@@ -350,6 +351,7 @@ class ProjectFlags(ExtensibleDbtClassMixin):
|
||||
"require_resource_names_without_spaces": self.require_resource_names_without_spaces,
|
||||
"source_freshness_run_project_hooks": self.source_freshness_run_project_hooks,
|
||||
"state_modified_compare_more_unrendered_values": self.state_modified_compare_more_unrendered_values,
|
||||
"state_modified_compare_vars": self.state_modified_compare_vars,
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -752,6 +752,7 @@ class StateSelectorMethod(SelectorMethod):
|
||||
"modified.relation": self.check_modified_factory("same_database_representation"),
|
||||
"modified.macros": self.check_modified_macros,
|
||||
"modified.contract": self.check_modified_contract("same_contract", adapter_type),
|
||||
"modified.vars": self.check_modified_factory("same_vars"),
|
||||
}
|
||||
if selector in state_checks:
|
||||
checker = state_checks[selector]
|
||||
|
||||
@@ -13,6 +13,7 @@ from dbt.contracts.graph.unparsed import (
|
||||
UnparsedMacroUpdate,
|
||||
UnparsedModelUpdate,
|
||||
UnparsedNodeUpdate,
|
||||
UnparsedSingularTestUpdate,
|
||||
)
|
||||
from dbt.exceptions import ParsingError
|
||||
from dbt.node_types import NodeType
|
||||
@@ -58,6 +59,7 @@ Target = TypeVar(
|
||||
UnpatchedSourceDefinition,
|
||||
UnparsedExposure,
|
||||
UnparsedModelUpdate,
|
||||
UnparsedSingularTestUpdate,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -91,6 +91,9 @@ class ExposureParser(YamlReader):
|
||||
unique_id = f"{NodeType.Exposure}.{package_name}.{unparsed.name}"
|
||||
path = self.yaml.path.relative_path
|
||||
|
||||
assert isinstance(self.yaml.file, SchemaSourceFile)
|
||||
exposure_vars = self.yaml.file.get_vars(self.key, unparsed.name)
|
||||
|
||||
fqn = self.schema_parser.get_fqn_prefix(path)
|
||||
fqn.append(unparsed.name)
|
||||
|
||||
@@ -133,6 +136,7 @@ class ExposureParser(YamlReader):
|
||||
maturity=unparsed.maturity,
|
||||
config=config,
|
||||
unrendered_config=unrendered_config,
|
||||
vars=exposure_vars,
|
||||
)
|
||||
ctx = generate_parse_exposure(
|
||||
parsed,
|
||||
@@ -144,7 +148,6 @@ class ExposureParser(YamlReader):
|
||||
get_rendered(depends_on_jinja, ctx, parsed, capture_macros=True)
|
||||
# parsed now has a populated refs/sources/metrics
|
||||
|
||||
assert isinstance(self.yaml.file, SchemaSourceFile)
|
||||
if parsed.config.enabled:
|
||||
self.manifest.add_exposure(self.yaml.file, parsed)
|
||||
else:
|
||||
|
||||
@@ -17,6 +17,7 @@ from dbt.contracts.graph.nodes import (
|
||||
ModelNode,
|
||||
ParsedMacroPatch,
|
||||
ParsedNodePatch,
|
||||
ParsedSingularTestPatch,
|
||||
UnpatchedSourceDefinition,
|
||||
)
|
||||
from dbt.contracts.graph.unparsed import (
|
||||
@@ -27,6 +28,7 @@ from dbt.contracts.graph.unparsed import (
|
||||
UnparsedMacroUpdate,
|
||||
UnparsedModelUpdate,
|
||||
UnparsedNodeUpdate,
|
||||
UnparsedSingularTestUpdate,
|
||||
UnparsedSourceDefinition,
|
||||
)
|
||||
from dbt.events.types import (
|
||||
@@ -207,6 +209,10 @@ class SchemaParser(SimpleParser[YamlBlock, ModelNode]):
|
||||
parser = MacroPatchParser(self, yaml_block, "macros")
|
||||
parser.parse()
|
||||
|
||||
if "data_tests" in dct:
|
||||
parser = SingularTestPatchParser(self, yaml_block, "data_tests")
|
||||
parser.parse()
|
||||
|
||||
# PatchParser.parse() (but never test_blocks)
|
||||
if "analyses" in dct:
|
||||
parser = AnalysisPatchParser(self, yaml_block, "analyses")
|
||||
@@ -301,7 +307,9 @@ class SchemaParser(SimpleParser[YamlBlock, ModelNode]):
|
||||
self.manifest.rebuild_ref_lookup()
|
||||
|
||||
|
||||
Parsed = TypeVar("Parsed", UnpatchedSourceDefinition, ParsedNodePatch, ParsedMacroPatch)
|
||||
Parsed = TypeVar(
|
||||
"Parsed", UnpatchedSourceDefinition, ParsedNodePatch, ParsedMacroPatch, ParsedSingularTestPatch
|
||||
)
|
||||
NodeTarget = TypeVar("NodeTarget", UnparsedNodeUpdate, UnparsedAnalysisUpdate, UnparsedModelUpdate)
|
||||
NonSourceTarget = TypeVar(
|
||||
"NonSourceTarget",
|
||||
@@ -309,6 +317,7 @@ NonSourceTarget = TypeVar(
|
||||
UnparsedAnalysisUpdate,
|
||||
UnparsedMacroUpdate,
|
||||
UnparsedModelUpdate,
|
||||
UnparsedSingularTestUpdate,
|
||||
)
|
||||
|
||||
|
||||
@@ -403,10 +412,14 @@ class YamlReader(metaclass=ABCMeta):
|
||||
|
||||
if self.schema_yaml_vars.env_vars:
|
||||
self.schema_parser.manifest.env_vars.update(self.schema_yaml_vars.env_vars)
|
||||
for var in self.schema_yaml_vars.env_vars.keys():
|
||||
schema_file.add_env_var(var, self.key, entry["name"])
|
||||
for env_var in self.schema_yaml_vars.env_vars.keys():
|
||||
schema_file.add_env_var(env_var, self.key, entry["name"])
|
||||
self.schema_yaml_vars.env_vars = {}
|
||||
|
||||
if self.schema_yaml_vars.vars:
|
||||
schema_file.add_vars(self.schema_yaml_vars.vars, self.key, entry["name"])
|
||||
self.schema_yaml_vars.vars = {}
|
||||
|
||||
yield entry
|
||||
|
||||
def render_entry(self, dct):
|
||||
@@ -677,6 +690,9 @@ class NodePatchParser(PatchParser[NodeTarget, ParsedNodePatch], Generic[NodeTarg
|
||||
# code consistency.
|
||||
deprecation_date: Optional[datetime.datetime] = None
|
||||
time_spine: Optional[TimeSpine] = None
|
||||
assert isinstance(self.yaml.file, SchemaSourceFile)
|
||||
source_file: SchemaSourceFile = self.yaml.file
|
||||
|
||||
if isinstance(block.target, UnparsedModelUpdate):
|
||||
deprecation_date = block.target.deprecation_date
|
||||
time_spine = (
|
||||
@@ -709,9 +725,9 @@ class NodePatchParser(PatchParser[NodeTarget, ParsedNodePatch], Generic[NodeTarg
|
||||
constraints=block.target.constraints,
|
||||
deprecation_date=deprecation_date,
|
||||
time_spine=time_spine,
|
||||
vars=source_file.get_vars(block.target.yaml_key, block.target.name),
|
||||
)
|
||||
assert isinstance(self.yaml.file, SchemaSourceFile)
|
||||
source_file: SchemaSourceFile = self.yaml.file
|
||||
|
||||
if patch.yaml_key in ["models", "seeds", "snapshots"]:
|
||||
unique_id = self.manifest.ref_lookup.get_unique_id(
|
||||
patch.name, self.project.project_name, None
|
||||
@@ -805,6 +821,8 @@ class NodePatchParser(PatchParser[NodeTarget, ParsedNodePatch], Generic[NodeTarg
|
||||
node.description = patch.description
|
||||
node.columns = patch.columns
|
||||
node.name = patch.name
|
||||
# Prefer node-level vars to vars from patch
|
||||
node.vars = {**patch.vars, **node.vars}
|
||||
|
||||
if not isinstance(node, ModelNode):
|
||||
for attr in ["latest_version", "access", "version", "constraints"]:
|
||||
@@ -954,6 +972,7 @@ class ModelPatchParser(NodePatchParser[UnparsedModelUpdate]):
|
||||
latest_version=latest_version,
|
||||
constraints=unparsed_version.constraints or target.constraints,
|
||||
deprecation_date=unparsed_version.deprecation_date,
|
||||
vars=source_file.get_vars(block.target.yaml_key, block.target.name),
|
||||
)
|
||||
# Node patched before config because config patching depends on model name,
|
||||
# which may have been updated in the version patch
|
||||
@@ -1116,6 +1135,55 @@ class AnalysisPatchParser(NodePatchParser[UnparsedAnalysisUpdate]):
|
||||
return UnparsedAnalysisUpdate
|
||||
|
||||
|
||||
class SingularTestPatchParser(PatchParser[UnparsedSingularTestUpdate, ParsedSingularTestPatch]):
|
||||
def get_block(self, node: UnparsedSingularTestUpdate) -> TargetBlock:
|
||||
return TargetBlock.from_yaml_block(self.yaml, node)
|
||||
|
||||
def _target_type(self) -> Type[UnparsedSingularTestUpdate]:
|
||||
return UnparsedSingularTestUpdate
|
||||
|
||||
def parse_patch(self, block: TargetBlock[UnparsedSingularTestUpdate], refs: ParserRef) -> None:
|
||||
patch = ParsedSingularTestPatch(
|
||||
name=block.target.name,
|
||||
description=block.target.description,
|
||||
meta=block.target.meta,
|
||||
docs=block.target.docs,
|
||||
config=block.target.config,
|
||||
original_file_path=block.target.original_file_path,
|
||||
yaml_key=block.target.yaml_key,
|
||||
package_name=block.target.package_name,
|
||||
)
|
||||
|
||||
assert isinstance(self.yaml.file, SchemaSourceFile)
|
||||
source_file: SchemaSourceFile = self.yaml.file
|
||||
|
||||
unique_id = self.manifest.singular_test_lookup.get_unique_id(
|
||||
block.name, block.target.package_name
|
||||
)
|
||||
if not unique_id:
|
||||
warn_or_error(
|
||||
NoNodeForYamlKey(
|
||||
patch_name=patch.name,
|
||||
yaml_key=patch.yaml_key,
|
||||
file_path=source_file.path.original_file_path,
|
||||
)
|
||||
)
|
||||
return
|
||||
|
||||
node = self.manifest.nodes.get(unique_id)
|
||||
assert node is not None
|
||||
|
||||
source_file.append_patch(patch.yaml_key, unique_id)
|
||||
if patch.config:
|
||||
self.patch_node_config(node, patch)
|
||||
|
||||
node.patch_path = patch.file_id
|
||||
node.description = patch.description
|
||||
node.created_at = time.time()
|
||||
node.meta = patch.meta
|
||||
node.docs = patch.docs
|
||||
|
||||
|
||||
class MacroPatchParser(PatchParser[UnparsedMacroUpdate, ParsedMacroPatch]):
|
||||
def get_block(self, node: UnparsedMacroUpdate) -> TargetBlock:
|
||||
return TargetBlock.from_yaml_block(self.yaml, node)
|
||||
|
||||
@@ -12,6 +12,7 @@ from dbt.context.context_config import (
|
||||
ContextConfigGenerator,
|
||||
UnrenderedConfigGenerator,
|
||||
)
|
||||
from dbt.contracts.files import SchemaSourceFile
|
||||
from dbt.contracts.graph.manifest import Manifest, SourceKey
|
||||
from dbt.contracts.graph.nodes import (
|
||||
GenericTestNode,
|
||||
@@ -158,6 +159,10 @@ class SourcePatcher:
|
||||
rendered=False,
|
||||
)
|
||||
|
||||
schema_file = self.manifest.files[target.file_id]
|
||||
assert isinstance(schema_file, SchemaSourceFile)
|
||||
source_vars = schema_file.get_vars("sources", source.name)
|
||||
|
||||
if not isinstance(config, SourceConfig):
|
||||
raise DbtInternalError(
|
||||
f"Calculated a {type(config)} for a source, but expected a SourceConfig"
|
||||
@@ -190,6 +195,7 @@ class SourcePatcher:
|
||||
tags=tags,
|
||||
config=config,
|
||||
unrendered_config=unrendered_config,
|
||||
vars=source_vars,
|
||||
)
|
||||
|
||||
if (
|
||||
|
||||
@@ -721,6 +721,12 @@
|
||||
"type": "string",
|
||||
"default": ""
|
||||
},
|
||||
"vars": {
|
||||
"type": "object",
|
||||
"propertyNames": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"root_path": {
|
||||
"anyOf": [
|
||||
{
|
||||
@@ -1754,6 +1760,12 @@
|
||||
"type": "string",
|
||||
"default": ""
|
||||
},
|
||||
"vars": {
|
||||
"type": "object",
|
||||
"propertyNames": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"language": {
|
||||
"type": "string",
|
||||
"default": "sql"
|
||||
@@ -2402,6 +2414,12 @@
|
||||
"type": "string",
|
||||
"default": ""
|
||||
},
|
||||
"vars": {
|
||||
"type": "object",
|
||||
"propertyNames": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"language": {
|
||||
"type": "string",
|
||||
"default": "sql"
|
||||
@@ -3187,6 +3205,12 @@
|
||||
"type": "string",
|
||||
"default": ""
|
||||
},
|
||||
"vars": {
|
||||
"type": "object",
|
||||
"propertyNames": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"language": {
|
||||
"type": "string",
|
||||
"default": "sql"
|
||||
@@ -3991,6 +4015,12 @@
|
||||
"type": "string",
|
||||
"default": ""
|
||||
},
|
||||
"vars": {
|
||||
"type": "object",
|
||||
"propertyNames": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"language": {
|
||||
"type": "string",
|
||||
"default": "sql"
|
||||
@@ -5346,6 +5376,12 @@
|
||||
"type": "string",
|
||||
"default": ""
|
||||
},
|
||||
"vars": {
|
||||
"type": "object",
|
||||
"propertyNames": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"language": {
|
||||
"type": "string",
|
||||
"default": "sql"
|
||||
@@ -5994,6 +6030,12 @@
|
||||
"type": "string",
|
||||
"default": ""
|
||||
},
|
||||
"vars": {
|
||||
"type": "object",
|
||||
"propertyNames": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"language": {
|
||||
"type": "string",
|
||||
"default": "sql"
|
||||
@@ -6946,6 +6988,12 @@
|
||||
"type": "string",
|
||||
"default": ""
|
||||
},
|
||||
"vars": {
|
||||
"type": "object",
|
||||
"propertyNames": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"language": {
|
||||
"type": "string",
|
||||
"default": "sql"
|
||||
@@ -8083,6 +8131,12 @@
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"vars": {
|
||||
"type": "object",
|
||||
"propertyNames": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"relation_name": {
|
||||
"anyOf": [
|
||||
{
|
||||
@@ -8454,6 +8508,12 @@
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"vars": {
|
||||
"type": "object",
|
||||
"propertyNames": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"url": {
|
||||
"anyOf": [
|
||||
{
|
||||
@@ -10525,6 +10585,12 @@
|
||||
"type": "string",
|
||||
"default": ""
|
||||
},
|
||||
"vars": {
|
||||
"type": "object",
|
||||
"propertyNames": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"root_path": {
|
||||
"anyOf": [
|
||||
{
|
||||
@@ -11558,6 +11624,12 @@
|
||||
"type": "string",
|
||||
"default": ""
|
||||
},
|
||||
"vars": {
|
||||
"type": "object",
|
||||
"propertyNames": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"language": {
|
||||
"type": "string",
|
||||
"default": "sql"
|
||||
@@ -12206,6 +12278,12 @@
|
||||
"type": "string",
|
||||
"default": ""
|
||||
},
|
||||
"vars": {
|
||||
"type": "object",
|
||||
"propertyNames": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"language": {
|
||||
"type": "string",
|
||||
"default": "sql"
|
||||
@@ -12991,6 +13069,12 @@
|
||||
"type": "string",
|
||||
"default": ""
|
||||
},
|
||||
"vars": {
|
||||
"type": "object",
|
||||
"propertyNames": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"language": {
|
||||
"type": "string",
|
||||
"default": "sql"
|
||||
@@ -13795,6 +13879,12 @@
|
||||
"type": "string",
|
||||
"default": ""
|
||||
},
|
||||
"vars": {
|
||||
"type": "object",
|
||||
"propertyNames": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"language": {
|
||||
"type": "string",
|
||||
"default": "sql"
|
||||
@@ -15150,6 +15240,12 @@
|
||||
"type": "string",
|
||||
"default": ""
|
||||
},
|
||||
"vars": {
|
||||
"type": "object",
|
||||
"propertyNames": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"language": {
|
||||
"type": "string",
|
||||
"default": "sql"
|
||||
@@ -15798,6 +15894,12 @@
|
||||
"type": "string",
|
||||
"default": ""
|
||||
},
|
||||
"vars": {
|
||||
"type": "object",
|
||||
"propertyNames": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"language": {
|
||||
"type": "string",
|
||||
"default": "sql"
|
||||
@@ -16750,6 +16852,12 @@
|
||||
"type": "string",
|
||||
"default": ""
|
||||
},
|
||||
"vars": {
|
||||
"type": "object",
|
||||
"propertyNames": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"language": {
|
||||
"type": "string",
|
||||
"default": "sql"
|
||||
@@ -17878,6 +17986,12 @@
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"vars": {
|
||||
"type": "object",
|
||||
"propertyNames": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"relation_name": {
|
||||
"anyOf": [
|
||||
{
|
||||
@@ -18047,6 +18161,12 @@
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"vars": {
|
||||
"type": "object",
|
||||
"propertyNames": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"url": {
|
||||
"anyOf": [
|
||||
{
|
||||
|
||||
@@ -363,6 +363,7 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False):
|
||||
"extra_ctes": [],
|
||||
"checksum": checksum_file(model_sql_path),
|
||||
"unrendered_config": unrendered_model_config,
|
||||
"vars": {},
|
||||
"access": "protected",
|
||||
"version": None,
|
||||
"latest_version": None,
|
||||
@@ -462,6 +463,7 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False):
|
||||
"extra_ctes": [],
|
||||
"checksum": checksum_file(second_model_sql_path),
|
||||
"unrendered_config": unrendered_second_config,
|
||||
"vars": {},
|
||||
"access": "protected",
|
||||
"version": None,
|
||||
"latest_version": None,
|
||||
@@ -544,6 +546,7 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False):
|
||||
"docs": {"node_color": None, "show": True},
|
||||
"checksum": checksum_file(seed_path),
|
||||
"unrendered_config": unrendered_seed_config,
|
||||
"vars": {},
|
||||
"relation_name": relation_name_node_format.format(
|
||||
project.database, my_schema_name, "seed"
|
||||
),
|
||||
@@ -599,6 +602,7 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False):
|
||||
},
|
||||
"checksum": {"name": "none", "checksum": ""},
|
||||
"unrendered_config": unrendered_test_config,
|
||||
"vars": {},
|
||||
"contract": {"checksum": None, "enforced": False, "alias_types": True},
|
||||
},
|
||||
"snapshot.test.snapshot_seed": {
|
||||
@@ -646,6 +650,7 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False):
|
||||
"tags": [],
|
||||
"unique_id": "snapshot.test.snapshot_seed",
|
||||
"unrendered_config": unrendered_snapshot_config,
|
||||
"vars": {"alternate_schema": alternate_schema},
|
||||
},
|
||||
"test.test.test_nothing_model_.5d38568946": {
|
||||
"alias": "test_nothing_model_",
|
||||
@@ -698,6 +703,7 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False):
|
||||
},
|
||||
"checksum": {"name": "none", "checksum": ""},
|
||||
"unrendered_config": unrendered_test_config,
|
||||
"vars": {},
|
||||
},
|
||||
"test.test.unique_model_id.67b76558ff": {
|
||||
"alias": "unique_model_id",
|
||||
@@ -751,6 +757,7 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False):
|
||||
},
|
||||
"checksum": {"name": "none", "checksum": ""},
|
||||
"unrendered_config": unrendered_test_config,
|
||||
"vars": {},
|
||||
},
|
||||
},
|
||||
"sources": {
|
||||
@@ -807,6 +814,7 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False):
|
||||
"unique_id": "source.test.my_source.my_table",
|
||||
"fqn": ["test", "my_source", "my_table"],
|
||||
"unrendered_config": {},
|
||||
"vars": {},
|
||||
},
|
||||
},
|
||||
"exposures": {
|
||||
@@ -841,6 +849,7 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False):
|
||||
"unique_id": "exposure.test.notebook_exposure",
|
||||
"url": "http://example.com/notebook/1",
|
||||
"unrendered_config": {},
|
||||
"vars": {},
|
||||
},
|
||||
"exposure.test.simple_exposure": {
|
||||
"created_at": ANY,
|
||||
@@ -873,6 +882,7 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False):
|
||||
"meta": {},
|
||||
"tags": [],
|
||||
"unrendered_config": {},
|
||||
"vars": {},
|
||||
},
|
||||
},
|
||||
"metrics": {},
|
||||
@@ -990,6 +1000,7 @@ def expected_references_manifest(project):
|
||||
"extra_ctes": [],
|
||||
"checksum": checksum_file(ephemeral_copy_path),
|
||||
"unrendered_config": get_unrendered_model_config(materialized="ephemeral"),
|
||||
"vars": {},
|
||||
"access": "protected",
|
||||
"version": None,
|
||||
"latest_version": None,
|
||||
@@ -1062,6 +1073,7 @@ def expected_references_manifest(project):
|
||||
"unrendered_config": get_unrendered_model_config(
|
||||
materialized="table", group="test_group"
|
||||
),
|
||||
"vars": {},
|
||||
"access": "protected",
|
||||
"version": None,
|
||||
"latest_version": None,
|
||||
@@ -1130,6 +1142,7 @@ def expected_references_manifest(project):
|
||||
"extra_ctes": [],
|
||||
"checksum": checksum_file(view_summary_path),
|
||||
"unrendered_config": get_unrendered_model_config(materialized="view"),
|
||||
"vars": {},
|
||||
"access": "protected",
|
||||
"version": None,
|
||||
"latest_version": None,
|
||||
@@ -1213,6 +1226,7 @@ def expected_references_manifest(project):
|
||||
"unique_id": "seed.test.seed",
|
||||
"checksum": checksum_file(seed_path),
|
||||
"unrendered_config": get_unrendered_seed_config(),
|
||||
"vars": {},
|
||||
"relation_name": '"{0}"."{1}".seed'.format(project.database, my_schema_name),
|
||||
},
|
||||
"snapshot.test.snapshot_seed": {
|
||||
@@ -1227,7 +1241,10 @@ def expected_references_manifest(project):
|
||||
"config": get_rendered_snapshot_config(target_schema=alternate_schema),
|
||||
"contract": {"checksum": None, "enforced": False, "alias_types": True},
|
||||
"database": model_database,
|
||||
"depends_on": {"macros": [], "nodes": ["seed.test.seed"]},
|
||||
"depends_on": {
|
||||
"macros": [],
|
||||
"nodes": ["seed.test.seed"],
|
||||
},
|
||||
"description": "",
|
||||
"docs": {"node_color": None, "show": True},
|
||||
"extra_ctes": [],
|
||||
@@ -1255,6 +1272,7 @@ def expected_references_manifest(project):
|
||||
"unrendered_config": get_unrendered_snapshot_config(
|
||||
target_schema=alternate_schema
|
||||
),
|
||||
"vars": {"alternate_schema": alternate_schema},
|
||||
},
|
||||
},
|
||||
"sources": {
|
||||
@@ -1309,6 +1327,7 @@ def expected_references_manifest(project):
|
||||
"unique_id": "source.test.my_source.my_table",
|
||||
"fqn": ["test", "my_source", "my_table"],
|
||||
"unrendered_config": {},
|
||||
"vars": {},
|
||||
},
|
||||
},
|
||||
"exposures": {
|
||||
@@ -1334,6 +1353,7 @@ def expected_references_manifest(project):
|
||||
"package_name": "test",
|
||||
"path": "schema.yml",
|
||||
"refs": [{"name": "view_summary", "package": None, "version": None}],
|
||||
"vars": {},
|
||||
"resource_type": "exposure",
|
||||
"sources": [],
|
||||
"type": "notebook",
|
||||
@@ -1594,6 +1614,7 @@ def expected_versions_manifest(project):
|
||||
group="test_group",
|
||||
meta={"size": "large", "color": "blue"},
|
||||
),
|
||||
"vars": {},
|
||||
"access": "protected",
|
||||
"version": 1,
|
||||
"latest_version": 2,
|
||||
@@ -1665,6 +1686,7 @@ def expected_versions_manifest(project):
|
||||
"unrendered_config": get_unrendered_model_config(
|
||||
materialized="view", group="test_group", meta={"size": "large", "color": "red"}
|
||||
),
|
||||
"vars": {},
|
||||
"access": "protected",
|
||||
"version": 2,
|
||||
"latest_version": 2,
|
||||
@@ -1723,6 +1745,7 @@ def expected_versions_manifest(project):
|
||||
"extra_ctes": [],
|
||||
"checksum": checksum_file(ref_versioned_model_path),
|
||||
"unrendered_config": get_unrendered_model_config(),
|
||||
"vars": {},
|
||||
"access": "protected",
|
||||
"version": None,
|
||||
"latest_version": None,
|
||||
@@ -1780,6 +1803,7 @@ def expected_versions_manifest(project):
|
||||
},
|
||||
"checksum": {"name": "none", "checksum": ""},
|
||||
"unrendered_config": unrendered_test_config,
|
||||
"vars": {},
|
||||
},
|
||||
"test.test.unique_versioned_model_v1_count.0b4c0b688a": {
|
||||
"alias": "unique_versioned_model_v1_count",
|
||||
@@ -1833,6 +1857,7 @@ def expected_versions_manifest(project):
|
||||
},
|
||||
"checksum": {"name": "none", "checksum": ""},
|
||||
"unrendered_config": unrendered_test_config,
|
||||
"vars": {},
|
||||
},
|
||||
"test.test.unique_versioned_model_v2_first_name.998430d28e": {
|
||||
"alias": "unique_versioned_model_v2_first_name",
|
||||
@@ -1886,6 +1911,7 @@ def expected_versions_manifest(project):
|
||||
},
|
||||
"checksum": {"name": "none", "checksum": ""},
|
||||
"unrendered_config": unrendered_test_config,
|
||||
"vars": {},
|
||||
},
|
||||
},
|
||||
"exposures": {
|
||||
@@ -1917,6 +1943,7 @@ def expected_versions_manifest(project):
|
||||
"unique_id": "exposure.test.notebook_exposure",
|
||||
"url": None,
|
||||
"unrendered_config": {},
|
||||
"vars": {},
|
||||
},
|
||||
},
|
||||
"metrics": {},
|
||||
|
||||
32
tests/functional/data_test_patch/fixtures.py
Normal file
32
tests/functional/data_test_patch/fixtures.py
Normal file
@@ -0,0 +1,32 @@
|
||||
tests__my_singular_test_sql = """
|
||||
with my_cte as (
|
||||
select 1 as id, 'foo' as name
|
||||
union all
|
||||
select 2 as id, 'bar' as name
|
||||
)
|
||||
select * from my_cte
|
||||
"""
|
||||
|
||||
tests__schema_yml = """
|
||||
data_tests:
|
||||
- name: my_singular_test
|
||||
description: "{{ doc('my_singular_test_documentation') }}"
|
||||
config:
|
||||
error_if: ">10"
|
||||
meta:
|
||||
some_key: some_val
|
||||
"""
|
||||
|
||||
tests__doc_block_md = """
|
||||
{% docs my_singular_test_documentation %}
|
||||
|
||||
Some docs from a doc block
|
||||
|
||||
{% enddocs %}
|
||||
"""
|
||||
|
||||
tests__invalid_name_schema_yml = """
|
||||
data_tests:
|
||||
- name: my_double_test
|
||||
description: documentation, but make it double
|
||||
"""
|
||||
53
tests/functional/data_test_patch/test_singular_test_patch.py
Normal file
53
tests/functional/data_test_patch/test_singular_test_patch.py
Normal file
@@ -0,0 +1,53 @@
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
from dbt.tests.util import get_artifact, run_dbt, run_dbt_and_capture
|
||||
from tests.functional.data_test_patch.fixtures import (
|
||||
tests__doc_block_md,
|
||||
tests__invalid_name_schema_yml,
|
||||
tests__my_singular_test_sql,
|
||||
tests__schema_yml,
|
||||
)
|
||||
|
||||
|
||||
class TestPatchSingularTest:
|
||||
@pytest.fixture(scope="class")
|
||||
def tests(self):
|
||||
return {
|
||||
"my_singular_test.sql": tests__my_singular_test_sql,
|
||||
"schema.yml": tests__schema_yml,
|
||||
"doc_block.md": tests__doc_block_md,
|
||||
}
|
||||
|
||||
def test_compile(self, project):
|
||||
run_dbt(["compile"])
|
||||
manifest = get_artifact(project.project_root, "target", "manifest.json")
|
||||
assert len(manifest["nodes"]) == 1
|
||||
|
||||
my_singular_test_node = manifest["nodes"]["test.test.my_singular_test"]
|
||||
assert my_singular_test_node["description"] == "Some docs from a doc block"
|
||||
assert my_singular_test_node["config"]["error_if"] == ">10"
|
||||
assert my_singular_test_node["config"]["meta"] == {"some_key": "some_val"}
|
||||
|
||||
|
||||
class TestPatchSingularTestInvalidName:
|
||||
@pytest.fixture(scope="class")
|
||||
def tests(self):
|
||||
return {
|
||||
"my_singular_test.sql": tests__my_singular_test_sql,
|
||||
"schema_with_invalid_name.yml": tests__invalid_name_schema_yml,
|
||||
}
|
||||
|
||||
def test_compile(self, project):
|
||||
_, log_output = run_dbt_and_capture(["compile"])
|
||||
|
||||
file_path = (
|
||||
"tests\\schema_with_invalid_name.yml"
|
||||
if os.name == "nt"
|
||||
else "tests/schema_with_invalid_name.yml"
|
||||
)
|
||||
assert (
|
||||
f"Did not find matching node for patch with name 'my_double_test' in the 'data_tests' section of file '{file_path}'"
|
||||
in log_output
|
||||
)
|
||||
@@ -1144,3 +1144,163 @@ class TestChangedSemanticModelContents(BaseModifiedState):
|
||||
write_file(modified_semantic_model_schema_yml, "models", "schema.yml")
|
||||
results = run_dbt(["list", "-s", "state:modified", "--state", "./state"])
|
||||
assert len(results) == 1
|
||||
|
||||
|
||||
class TestModifiedVars(BaseModifiedState):
|
||||
@pytest.fixture(scope="class")
|
||||
def project_config_update(self):
|
||||
return {
|
||||
"flags": {
|
||||
"state_modified_compare_vars": True,
|
||||
},
|
||||
"vars": {"my_var": 1},
|
||||
}
|
||||
|
||||
@pytest.fixture(scope="class")
|
||||
def models(self):
|
||||
return {
|
||||
"model_with_var.sql": "select {{ var('my_var') }} as id",
|
||||
}
|
||||
|
||||
def test_changed_vars(self, project):
|
||||
self.run_and_save_state()
|
||||
|
||||
# No var change
|
||||
assert not run_dbt(["list", "-s", "state:modified", "--state", "./state"])
|
||||
assert not run_dbt(["list", "-s", "state:modified.vars", "--state", "./state"])
|
||||
|
||||
# Modify var (my_var: 1 -> 2)
|
||||
update_config_file({"vars": {"my_var": 2}}, "dbt_project.yml")
|
||||
assert run_dbt(["list", "-s", "state:modified", "--state", "./state"]) == [
|
||||
"test.model_with_var"
|
||||
]
|
||||
assert run_dbt(["list", "-s", "state:modified.vars", "--state", "./state"]) == [
|
||||
"test.model_with_var"
|
||||
]
|
||||
|
||||
# Reset dbt_project.yml
|
||||
update_config_file({"vars": {"my_var": 1}}, "dbt_project.yml")
|
||||
|
||||
# Modify var via --var CLI flag
|
||||
assert not run_dbt(
|
||||
["list", "--vars", '{"my_var": 1}', "-s", "state:modified", "--state", "./state"]
|
||||
)
|
||||
assert run_dbt(
|
||||
["list", "--vars", '{"my_var": 2}', "-s", "state:modified", "--state", "./state"]
|
||||
) == ["test.model_with_var"]
|
||||
|
||||
|
||||
macro_with_var_sql = """
|
||||
{% macro macro_with_var() %}
|
||||
{{ var('my_var') }}
|
||||
{% endmacro %}
|
||||
"""
|
||||
|
||||
|
||||
class TestModifiedMacroVars(BaseModifiedState):
|
||||
@pytest.fixture(scope="class")
|
||||
def project_config_update(self):
|
||||
return {
|
||||
"flags": {
|
||||
"state_modified_compare_vars": True,
|
||||
},
|
||||
"vars": {"my_var": 1},
|
||||
}
|
||||
|
||||
@pytest.fixture(scope="class")
|
||||
def models(self):
|
||||
return {
|
||||
"model_with_macro.sql": "select {{ macro_with_var() }} as id",
|
||||
}
|
||||
|
||||
@pytest.fixture(scope="class")
|
||||
def macros(self):
|
||||
return {
|
||||
"macro_with_var.sql": macro_with_var_sql,
|
||||
}
|
||||
|
||||
def test_changed_vars(self, project):
|
||||
self.run_and_save_state()
|
||||
|
||||
# No var change
|
||||
assert not run_dbt(["list", "-s", "state:modified", "--state", "./state"])
|
||||
assert not run_dbt(["list", "-s", "state:modified.vars", "--state", "./state"])
|
||||
|
||||
# Modify var (my_var: 1 -> 2)
|
||||
update_config_file({"vars": {"my_var": 2}}, "dbt_project.yml")
|
||||
assert run_dbt(["list", "-s", "state:modified", "--state", "./state"]) == [
|
||||
"test.model_with_macro"
|
||||
]
|
||||
assert run_dbt(["list", "-s", "state:modified.vars", "--state", "./state"]) == [
|
||||
"test.model_with_macro"
|
||||
]
|
||||
|
||||
# Macros themselves not captured as modified because the var value depends on a node's context
|
||||
assert not run_dbt(["list", "-s", "state:modified.macros", "--state", "./state"])
|
||||
|
||||
|
||||
# TODO: test versioned models, tests
|
||||
model_with_var_schema_yml = """
|
||||
version: 2
|
||||
models:
|
||||
- name: model_with_var
|
||||
config:
|
||||
materialized: "{{ var('my_var') }}"
|
||||
|
||||
exposures:
|
||||
- name: exposure_name
|
||||
type: dashboard
|
||||
owner:
|
||||
name: "{{ var('my_var') }}"
|
||||
|
||||
sources:
|
||||
- name: jaffle_shop
|
||||
database: "{{ var('my_var') }}"
|
||||
schema: jaffle_shop
|
||||
tables:
|
||||
- name: orders
|
||||
- name: customers
|
||||
"""
|
||||
|
||||
|
||||
class TestModifiedVarsSchemaYml(BaseModifiedState):
|
||||
@pytest.fixture(scope="class")
|
||||
def project_config_update(self):
|
||||
return {
|
||||
"flags": {
|
||||
"state_modified_compare_vars": True,
|
||||
},
|
||||
"vars": {"my_var": "table"},
|
||||
}
|
||||
|
||||
@pytest.fixture(scope="class")
|
||||
def models(self):
|
||||
return {"model_with_var.sql": "select 1 as id", "schema.yml": model_with_var_schema_yml}
|
||||
|
||||
def test_changed_vars(self, project):
|
||||
self.run_and_save_state()
|
||||
|
||||
# No var change
|
||||
assert not run_dbt(["list", "-s", "state:modified", "--state", "./state"])
|
||||
assert not run_dbt(["list", "-s", "state:modified.vars", "--state", "./state"])
|
||||
|
||||
# Modify var (my_var: table -> view)
|
||||
update_config_file({"vars": {"my_var": "view"}}, "dbt_project.yml")
|
||||
assert sorted(run_dbt(["list", "-s", "state:modified", "--state", "./state"])) == sorted(
|
||||
[
|
||||
"test.model_with_var",
|
||||
"exposure:test.exposure_name",
|
||||
"source:test.jaffle_shop.customers",
|
||||
"source:test.jaffle_shop.orders",
|
||||
]
|
||||
)
|
||||
assert sorted(
|
||||
run_dbt(["list", "-s", "state:modified.vars", "--state", "./state"])
|
||||
) == sorted(
|
||||
[
|
||||
"test.model_with_var",
|
||||
"exposure:test.exposure_name",
|
||||
"source:test.jaffle_shop.customers",
|
||||
"source:test.jaffle_shop.orders",
|
||||
]
|
||||
)
|
||||
|
||||
@@ -49,7 +49,10 @@ class TestList:
|
||||
"json": {
|
||||
"name": "my_snapshot",
|
||||
"package_name": "test",
|
||||
"depends_on": {"nodes": [], "macros": []},
|
||||
"depends_on": {
|
||||
"nodes": [],
|
||||
"macros": [],
|
||||
},
|
||||
"tags": [],
|
||||
"config": {
|
||||
"enabled": True,
|
||||
|
||||
@@ -31,7 +31,7 @@ class TestProjectMethods:
|
||||
def test_all_source_paths(self, project: Project):
|
||||
assert (
|
||||
project.all_source_paths.sort()
|
||||
== ["models", "seeds", "snapshots", "analyses", "macros"].sort()
|
||||
== ["models", "seeds", "snapshots", "analyses", "macros", "tests"].sort()
|
||||
)
|
||||
|
||||
def test_generic_test_paths(self, project: Project):
|
||||
@@ -99,7 +99,8 @@ class TestProjectInitialization(BaseConfigTest):
|
||||
self.assertEqual(project.test_paths, ["tests"])
|
||||
self.assertEqual(project.analysis_paths, ["analyses"])
|
||||
self.assertEqual(
|
||||
set(project.docs_paths), set(["models", "seeds", "snapshots", "analyses", "macros"])
|
||||
set(project.docs_paths),
|
||||
{"models", "seeds", "snapshots", "analyses", "macros", "tests"},
|
||||
)
|
||||
self.assertEqual(project.asset_paths, [])
|
||||
self.assertEqual(project.target_path, "target")
|
||||
@@ -128,7 +129,7 @@ class TestProjectInitialization(BaseConfigTest):
|
||||
)
|
||||
self.assertEqual(
|
||||
set(project.docs_paths),
|
||||
set(["other-models", "seeds", "snapshots", "analyses", "macros"]),
|
||||
{"other-models", "seeds", "snapshots", "analyses", "macros", "tests"},
|
||||
)
|
||||
|
||||
def test_all_overrides(self):
|
||||
|
||||
@@ -129,7 +129,7 @@ class TestRuntimeConfigFiles(BaseConfigTest):
|
||||
self.assertEqual(config.test_paths, ["tests"])
|
||||
self.assertEqual(config.analysis_paths, ["analyses"])
|
||||
self.assertEqual(
|
||||
set(config.docs_paths), set(["models", "seeds", "snapshots", "analyses", "macros"])
|
||||
set(config.docs_paths), {"models", "seeds", "snapshots", "analyses", "macros", "tests"}
|
||||
)
|
||||
self.assertEqual(config.asset_paths, [])
|
||||
self.assertEqual(config.target_path, "target")
|
||||
|
||||
@@ -96,6 +96,7 @@ REQUIRED_PARSED_NODE_KEYS = frozenset(
|
||||
"deprecation_date",
|
||||
"defer_relation",
|
||||
"time_spine",
|
||||
"vars",
|
||||
"batches",
|
||||
}
|
||||
)
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
from argparse import Namespace
|
||||
import pickle
|
||||
import re
|
||||
from dataclasses import replace
|
||||
@@ -24,6 +25,12 @@ from tests.unit.utils import (
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def args_for_flags() -> Namespace:
|
||||
return Namespace(
|
||||
state_modified_compare_vars=False
|
||||
)
|
||||
|
||||
def norm_whitespace(string):
|
||||
_RE_COMBINE_WHITESPACE = re.compile(r"\s+")
|
||||
string = _RE_COMBINE_WHITESPACE.sub(" ", string).strip()
|
||||
@@ -199,6 +206,7 @@ def basic_compiled_dict():
|
||||
"checksum": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
||||
},
|
||||
"unrendered_config": {},
|
||||
"vars": {},
|
||||
"unrendered_config_call_dict": {},
|
||||
"config_call_dict": {},
|
||||
"access": "protected",
|
||||
@@ -519,6 +527,7 @@ def basic_compiled_schema_test_dict():
|
||||
"unrendered_config": {
|
||||
"severity": "warn",
|
||||
},
|
||||
"vars": {},
|
||||
"unrendered_config_call_dict": {},
|
||||
"config_call_dict": {},
|
||||
}
|
||||
|
||||
@@ -65,8 +65,10 @@ from tests.unit.utils import (
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def flags_for_args() -> Namespace:
|
||||
return Namespace(SEND_ANONYMOUS_USAGE_STATS=False)
|
||||
def args_for_flags() -> Namespace:
|
||||
return Namespace(
|
||||
send_anonymous_usage_stats=False, state_modified_compare_vars=False
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
@@ -199,6 +201,7 @@ def base_parsed_model_dict():
|
||||
"checksum": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
||||
},
|
||||
"unrendered_config": {},
|
||||
"vars": {},
|
||||
"unrendered_config_call_dict": {},
|
||||
"config_call_dict": {},
|
||||
"access": AccessType.Protected.value,
|
||||
@@ -256,6 +259,7 @@ def minimal_parsed_model_dict():
|
||||
"checksum": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
||||
},
|
||||
"unrendered_config": {},
|
||||
"vars": {},
|
||||
}
|
||||
|
||||
|
||||
@@ -322,6 +326,7 @@ def complex_parsed_model_dict():
|
||||
"materialized": "ephemeral",
|
||||
"post_hook": ['insert into blah(a, b) select "1", 1'],
|
||||
},
|
||||
"vars": {},
|
||||
"unrendered_config_call_dict": {},
|
||||
"config_call_dict": {},
|
||||
"access": AccessType.Protected.value,
|
||||
@@ -532,6 +537,7 @@ def basic_parsed_seed_dict():
|
||||
"meta": {},
|
||||
"checksum": {"name": "path", "checksum": "seeds/seed.csv"},
|
||||
"unrendered_config": {},
|
||||
"vars": {},
|
||||
"unrendered_config_call_dict": {},
|
||||
"config_call_dict": {},
|
||||
}
|
||||
@@ -638,6 +644,7 @@ def complex_parsed_seed_dict():
|
||||
"unrendered_config": {
|
||||
"persist_docs": {"relation": True, "columns": True},
|
||||
},
|
||||
"vars": {},
|
||||
"unrendered_config_call_dict": {},
|
||||
"config_call_dict": {},
|
||||
}
|
||||
@@ -838,6 +845,7 @@ def base_parsed_hook_dict():
|
||||
"checksum": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
||||
},
|
||||
"unrendered_config": {},
|
||||
"vars": {},
|
||||
"unrendered_config_call_dict": {},
|
||||
"config_call_dict": {},
|
||||
}
|
||||
@@ -932,6 +940,7 @@ def complex_parsed_hook_dict():
|
||||
"column_types": {"a": "text"},
|
||||
"materialized": "table",
|
||||
},
|
||||
"vars": {},
|
||||
"unrendered_config_call_dict": {},
|
||||
"config_call_dict": {},
|
||||
}
|
||||
@@ -1077,6 +1086,7 @@ def basic_parsed_schema_test_dict():
|
||||
"checksum": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
||||
},
|
||||
"unrendered_config": {},
|
||||
"vars": {},
|
||||
"unrendered_config_call_dict": {},
|
||||
"config_call_dict": {},
|
||||
}
|
||||
@@ -1166,6 +1176,7 @@ def complex_parsed_schema_test_dict():
|
||||
"checksum": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
||||
},
|
||||
"unrendered_config": {"materialized": "table", "severity": "WARN"},
|
||||
"vars": {},
|
||||
"unrendered_config_call_dict": {},
|
||||
"config_call_dict": {},
|
||||
}
|
||||
@@ -1561,6 +1572,7 @@ def basic_timestamp_snapshot_dict():
|
||||
"target_database": "some_snapshot_db",
|
||||
"target_schema": "some_snapshot_schema",
|
||||
},
|
||||
"vars": {},
|
||||
"unrendered_config_call_dict": {},
|
||||
"config_call_dict": {},
|
||||
}
|
||||
@@ -1668,6 +1680,7 @@ def basic_check_snapshot_dict():
|
||||
},
|
||||
"unrendered_config_call_dict": {},
|
||||
"config_call_dict": {},
|
||||
"vars": {},
|
||||
}
|
||||
|
||||
|
||||
@@ -1877,6 +1890,7 @@ def basic_parsed_source_definition_dict():
|
||||
"enabled": True,
|
||||
},
|
||||
"unrendered_config": {},
|
||||
"vars": {},
|
||||
}
|
||||
|
||||
|
||||
@@ -1909,6 +1923,7 @@ def complex_parsed_source_definition_dict():
|
||||
"freshness": {"warn_after": {"period": "hour", "count": 1}, "error_after": {}},
|
||||
"loaded_at_field": "loaded_at",
|
||||
"unrendered_config": {},
|
||||
"vars": {},
|
||||
}
|
||||
|
||||
|
||||
@@ -2080,6 +2095,7 @@ def basic_parsed_exposure_dict():
|
||||
"enabled": True,
|
||||
},
|
||||
"unrendered_config": {},
|
||||
"vars": {},
|
||||
}
|
||||
|
||||
|
||||
@@ -2135,6 +2151,7 @@ def complex_parsed_exposure_dict():
|
||||
"enabled": True,
|
||||
},
|
||||
"unrendered_config": {},
|
||||
"vars": {},
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
from argparse import Namespace
|
||||
import copy
|
||||
from dataclasses import replace
|
||||
from pathlib import Path
|
||||
@@ -643,6 +644,11 @@ def previous_state(manifest):
|
||||
return create_previous_state(manifest)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def args_for_flags():
|
||||
return Namespace(state_modified_compare_vars=False)
|
||||
|
||||
|
||||
def add_node(manifest, node):
|
||||
manifest.nodes[node.unique_id] = node
|
||||
|
||||
|
||||
Reference in New Issue
Block a user