forked from repo-mirrors/dbt-core
Compare commits
11 Commits
main
...
ct-2517-ls
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e0facc0b71 | ||
|
|
f14df6846a | ||
|
|
41128b6bbd | ||
|
|
61d187249b | ||
|
|
7ad85ccb1a | ||
|
|
78f1184910 | ||
|
|
779db753e1 | ||
|
|
483753766e | ||
|
|
7e3168ee21 | ||
|
|
a1423f8459 | ||
|
|
b4c3dc18f2 |
6
.changes/unreleased/Features-20230607-113514.yaml
Normal file
6
.changes/unreleased/Features-20230607-113514.yaml
Normal file
@@ -0,0 +1,6 @@
|
||||
kind: Features
|
||||
body: Implement list command for public models
|
||||
time: 2023-06-07T11:35:14.267845-04:00
|
||||
custom:
|
||||
Author: gshank
|
||||
Issue: "7496"
|
||||
@@ -176,6 +176,11 @@ class Linker:
|
||||
else:
|
||||
raise GraphDependencyNotFoundError(node, dependency)
|
||||
|
||||
# Add public_nodes to graph. TODO: we might want to do this only for the "ls" command
|
||||
for pub_dependency in node.depends_on_public_nodes:
|
||||
if pub_dependency in manifest.public_nodes:
|
||||
self.dependency(node.unique_id, (manifest.public_nodes[pub_dependency].unique_id))
|
||||
|
||||
def link_graph(self, manifest: Manifest):
|
||||
for source in manifest.sources.values():
|
||||
self.add_node(source.unique_id)
|
||||
@@ -256,7 +261,7 @@ class Linker:
|
||||
index_dict = dict()
|
||||
for node_index, node_name in enumerate(self.graph):
|
||||
index_dict[node_name] = node_index
|
||||
data = manifest.expect(node_name).to_dict(omit_none=True)
|
||||
data = manifest.expect(node_name).to_ls_dict(omit_none=True)
|
||||
graph_nodes[node_index] = {"name": node_name, "type": data["resource_type"]}
|
||||
|
||||
for node_index, node in graph_nodes.items():
|
||||
|
||||
@@ -925,7 +925,7 @@ class Manifest(MacroMethods, DataClassMessagePackMixin, dbtClassMixin):
|
||||
|
||||
# Called in dbt.compilation.Linker.write_graph and
|
||||
# dbt.graph.queue.get and ._include_in_cost
|
||||
def expect(self, unique_id: str) -> GraphMemberNode:
|
||||
def expect(self, unique_id: str):
|
||||
if unique_id in self.nodes:
|
||||
return self.nodes[unique_id]
|
||||
elif unique_id in self.sources:
|
||||
@@ -934,6 +934,9 @@ class Manifest(MacroMethods, DataClassMessagePackMixin, dbtClassMixin):
|
||||
return self.exposures[unique_id]
|
||||
elif unique_id in self.metrics:
|
||||
return self.metrics[unique_id]
|
||||
elif unique_id in self.public_nodes:
|
||||
public_node = self.public_nodes[unique_id]
|
||||
return public_node
|
||||
else:
|
||||
# something terrible has happened
|
||||
raise dbt.exceptions.DbtInternalError(
|
||||
|
||||
@@ -130,6 +130,9 @@ class BaseNode(dbtClassMixin, Replaceable):
|
||||
def get_materialization(self):
|
||||
return self.config.materialized
|
||||
|
||||
def to_ls_dict(self, **kwargs):
|
||||
return self.to_dict(**kwargs)
|
||||
|
||||
|
||||
@dataclass
|
||||
class GraphNode(BaseNode):
|
||||
|
||||
@@ -61,7 +61,7 @@ class PublicModel(dbtClassMixin, ManifestOrPublicNode):
|
||||
# Needed for ref resolution code
|
||||
@property
|
||||
def resource_type(self):
|
||||
return NodeType.Model
|
||||
return NodeType.PublicModel
|
||||
|
||||
# Needed for ref resolution code
|
||||
@property
|
||||
@@ -95,6 +95,15 @@ class PublicModel(dbtClassMixin, ManifestOrPublicNode):
|
||||
def alias(self):
|
||||
return self.identifier
|
||||
|
||||
@property
|
||||
def fqn(self):
|
||||
return [self.package_name, self.name]
|
||||
|
||||
def to_ls_dict(self, **kwargs):
|
||||
dct = self.to_dict(**kwargs)
|
||||
dct["resource_type"] = NodeType.PublicModel.value
|
||||
return dct
|
||||
|
||||
|
||||
@dataclass
|
||||
class PublicationMandatory:
|
||||
|
||||
@@ -21,7 +21,7 @@ from .selector_spec import (
|
||||
|
||||
INTERSECTION_DELIMITER = ","
|
||||
|
||||
DEFAULT_INCLUDES: List[str] = ["fqn:*", "source:*", "exposure:*", "metric:*"]
|
||||
DEFAULT_INCLUDES: List[str] = ["fqn:*", "source:*", "exposure:*", "metric:*", "public:*"]
|
||||
DEFAULT_EXCLUDES: List[str] = []
|
||||
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
from typing import Set, List, Optional, Tuple
|
||||
from typing import Set, List, Optional, Tuple, Union
|
||||
|
||||
from .graph import Graph, UniqueId
|
||||
from .queue import GraphQueue
|
||||
@@ -13,6 +13,7 @@ from dbt.exceptions import (
|
||||
InvalidSelectorError,
|
||||
)
|
||||
from dbt.contracts.graph.nodes import GraphMemberNode
|
||||
from dbt.contracts.publication import PublicModel
|
||||
from dbt.contracts.graph.manifest import Manifest
|
||||
from dbt.contracts.state import PreviousState
|
||||
|
||||
@@ -166,17 +167,20 @@ class NodeSelector(MethodManager):
|
||||
elif unique_id in self.manifest.metrics:
|
||||
metric = self.manifest.metrics[unique_id]
|
||||
return metric.config.enabled
|
||||
elif unique_id in self.manifest.public_nodes:
|
||||
# There are no disabled public nodes
|
||||
return True
|
||||
node = self.manifest.nodes[unique_id]
|
||||
return not node.empty and node.config.enabled
|
||||
|
||||
def node_is_match(self, node: GraphMemberNode) -> bool:
|
||||
def node_is_match(self, node) -> bool:
|
||||
"""Determine if a node is a match for the selector. Non-match nodes
|
||||
will be excluded from results during filtering.
|
||||
"""
|
||||
return True
|
||||
|
||||
def _is_match(self, unique_id: UniqueId) -> bool:
|
||||
node: GraphMemberNode
|
||||
node: Union[GraphMemberNode, PublicModel]
|
||||
if unique_id in self.manifest.nodes:
|
||||
node = self.manifest.nodes[unique_id]
|
||||
elif unique_id in self.manifest.sources:
|
||||
@@ -185,6 +189,8 @@ class NodeSelector(MethodManager):
|
||||
node = self.manifest.exposures[unique_id]
|
||||
elif unique_id in self.manifest.metrics:
|
||||
node = self.manifest.metrics[unique_id]
|
||||
elif unique_id in self.manifest.public_nodes:
|
||||
node = self.manifest.public_nodes[unique_id]
|
||||
else:
|
||||
raise DbtInternalError(f"Node {unique_id} not found in the manifest!")
|
||||
return self.node_is_match(node)
|
||||
|
||||
@@ -19,6 +19,7 @@ from dbt.contracts.graph.nodes import (
|
||||
ManifestNode,
|
||||
ModelNode,
|
||||
)
|
||||
from dbt.contracts.publication import PublicModel
|
||||
from dbt.contracts.graph.unparsed import UnparsedVersion
|
||||
from dbt.contracts.state import PreviousState
|
||||
from dbt.exceptions import (
|
||||
@@ -53,6 +54,7 @@ class MethodName(StrEnum):
|
||||
SourceStatus = "source_status"
|
||||
Wildcard = "wildcard"
|
||||
Version = "version"
|
||||
PublicModel = "public"
|
||||
|
||||
|
||||
def is_selected_node(fqn: List[str], node_selector: str, is_versioned: bool) -> bool:
|
||||
@@ -144,6 +146,15 @@ class SelectorMethod(metaclass=abc.ABCMeta):
|
||||
continue
|
||||
yield unique_id, metric
|
||||
|
||||
def public_model_nodes(
|
||||
self, included_nodes: Set[UniqueId]
|
||||
) -> Iterator[Tuple[UniqueId, PublicModel]]:
|
||||
for key, public_model in self.manifest.public_nodes.items():
|
||||
unique_id = UniqueId(key)
|
||||
if unique_id not in included_nodes:
|
||||
continue
|
||||
yield unique_id, public_model
|
||||
|
||||
def all_nodes(
|
||||
self, included_nodes: Set[UniqueId]
|
||||
) -> Iterator[Tuple[UniqueId, SelectorTarget]]:
|
||||
@@ -152,6 +163,7 @@ class SelectorMethod(metaclass=abc.ABCMeta):
|
||||
self.source_nodes(included_nodes),
|
||||
self.exposure_nodes(included_nodes),
|
||||
self.metric_nodes(included_nodes),
|
||||
self.public_model_nodes(included_nodes),
|
||||
)
|
||||
|
||||
def configurable_nodes(
|
||||
@@ -159,6 +171,13 @@ class SelectorMethod(metaclass=abc.ABCMeta):
|
||||
) -> Iterator[Tuple[UniqueId, ResultNode]]:
|
||||
yield from chain(self.parsed_nodes(included_nodes), self.source_nodes(included_nodes))
|
||||
|
||||
def parsed_and_public_nodes(
|
||||
self, included_nodes: Set[UniqueId]
|
||||
) -> Iterator[Tuple[UniqueId, Union[ResultNode, PublicModel]]]:
|
||||
yield from chain(
|
||||
self.parsed_nodes(included_nodes), self.public_model_nodes(included_nodes)
|
||||
)
|
||||
|
||||
def non_source_nodes(
|
||||
self,
|
||||
included_nodes: Set[UniqueId],
|
||||
@@ -167,6 +186,7 @@ class SelectorMethod(metaclass=abc.ABCMeta):
|
||||
self.parsed_nodes(included_nodes),
|
||||
self.exposure_nodes(included_nodes),
|
||||
self.metric_nodes(included_nodes),
|
||||
self.public_model_nodes(included_nodes),
|
||||
)
|
||||
|
||||
def groupable_nodes(
|
||||
@@ -210,36 +230,36 @@ class QualifiedNameSelectorMethod(SelectorMethod):
|
||||
|
||||
:param str selector: The selector or node name
|
||||
"""
|
||||
parsed_nodes = list(self.parsed_nodes(included_nodes))
|
||||
for node, real_node in parsed_nodes:
|
||||
if self.node_is_match(selector, real_node.fqn, real_node.is_versioned):
|
||||
yield node
|
||||
parsed_nodes = list(self.parsed_and_public_nodes(included_nodes))
|
||||
for unique_id, node in parsed_nodes:
|
||||
if self.node_is_match(selector, node.fqn, node.is_versioned):
|
||||
yield unique_id
|
||||
|
||||
|
||||
class TagSelectorMethod(SelectorMethod):
|
||||
def search(self, included_nodes: Set[UniqueId], selector: str) -> Iterator[UniqueId]:
|
||||
"""yields nodes from included that have the specified tag"""
|
||||
for node, real_node in self.all_nodes(included_nodes):
|
||||
if any(fnmatch(tag, selector) for tag in real_node.tags):
|
||||
yield node
|
||||
for unique_id, node in self.all_nodes(included_nodes):
|
||||
if any(fnmatch(tag, selector) for tag in node.tags):
|
||||
yield unique_id
|
||||
|
||||
|
||||
class GroupSelectorMethod(SelectorMethod):
|
||||
def search(self, included_nodes: Set[UniqueId], selector: str) -> Iterator[UniqueId]:
|
||||
"""yields nodes from included in the specified group"""
|
||||
for node, real_node in self.groupable_nodes(included_nodes):
|
||||
if selector == real_node.config.get("group"):
|
||||
yield node
|
||||
for unique_id, node in self.groupable_nodes(included_nodes):
|
||||
if selector == node.config.get("group"):
|
||||
yield unique_id
|
||||
|
||||
|
||||
class AccessSelectorMethod(SelectorMethod):
|
||||
def search(self, included_nodes: Set[UniqueId], selector: str) -> Iterator[UniqueId]:
|
||||
"""yields model nodes matching the specified access level"""
|
||||
for node, real_node in self.parsed_nodes(included_nodes):
|
||||
if not isinstance(real_node, ModelNode):
|
||||
for unique_id, node in self.parsed_nodes(included_nodes):
|
||||
if not isinstance(node, ModelNode):
|
||||
continue
|
||||
if selector == real_node.access:
|
||||
yield node
|
||||
if selector == node.access:
|
||||
yield unique_id
|
||||
|
||||
|
||||
class SourceSelectorMethod(SelectorMethod):
|
||||
@@ -262,14 +282,14 @@ class SourceSelectorMethod(SelectorMethod):
|
||||
).format(selector)
|
||||
raise DbtRuntimeError(msg)
|
||||
|
||||
for node, real_node in self.source_nodes(included_nodes):
|
||||
if not fnmatch(real_node.package_name, target_package):
|
||||
for unique_id, node in self.source_nodes(included_nodes):
|
||||
if not fnmatch(node.package_name, target_package):
|
||||
continue
|
||||
if not fnmatch(real_node.source_name, target_source):
|
||||
if not fnmatch(node.source_name, target_source):
|
||||
continue
|
||||
if not fnmatch(real_node.name, target_table):
|
||||
if not fnmatch(node.name, target_table):
|
||||
continue
|
||||
yield node
|
||||
yield unique_id
|
||||
|
||||
|
||||
class ExposureSelectorMethod(SelectorMethod):
|
||||
@@ -288,13 +308,38 @@ class ExposureSelectorMethod(SelectorMethod):
|
||||
).format(selector)
|
||||
raise DbtRuntimeError(msg)
|
||||
|
||||
for node, real_node in self.exposure_nodes(included_nodes):
|
||||
if not fnmatch(real_node.package_name, target_package):
|
||||
for unique_id, node in self.exposure_nodes(included_nodes):
|
||||
if not fnmatch(node.package_name, target_package):
|
||||
continue
|
||||
if not fnmatch(real_node.name, target_name):
|
||||
if not fnmatch(node.name, target_name):
|
||||
continue
|
||||
|
||||
yield node
|
||||
yield unique_id
|
||||
|
||||
|
||||
class PublicModelSelectorMethod(SelectorMethod):
|
||||
def search(self, included_nodes: Set[UniqueId], selector: str) -> Iterator[UniqueId]:
|
||||
parts = selector.split(".")
|
||||
target_package = SELECTOR_GLOB
|
||||
if len(parts) == 1:
|
||||
target_name = parts[0]
|
||||
elif len(parts) == 2:
|
||||
target_package, target_name = parts
|
||||
else:
|
||||
msg = (
|
||||
'Invalid public model selector value "{}". Public models must be of '
|
||||
"the form ${{public_model_name}} or "
|
||||
"${{public_model_package.public_model_name}}"
|
||||
).format(selector)
|
||||
raise DbtRuntimeError(msg)
|
||||
|
||||
for unique_id, node in self.public_model_nodes(included_nodes):
|
||||
if not fnmatch(node.package_name, target_package):
|
||||
continue
|
||||
if not fnmatch(node.name, target_name):
|
||||
continue
|
||||
|
||||
yield unique_id
|
||||
|
||||
|
||||
class MetricSelectorMethod(SelectorMethod):
|
||||
@@ -313,13 +358,13 @@ class MetricSelectorMethod(SelectorMethod):
|
||||
).format(selector)
|
||||
raise DbtRuntimeError(msg)
|
||||
|
||||
for node, real_node in self.metric_nodes(included_nodes):
|
||||
if not fnmatch(real_node.package_name, target_package):
|
||||
for unique_id, node in self.metric_nodes(included_nodes):
|
||||
if not fnmatch(node.package_name, target_package):
|
||||
continue
|
||||
if not fnmatch(real_node.name, target_name):
|
||||
if not fnmatch(node.name, target_name):
|
||||
continue
|
||||
|
||||
yield node
|
||||
yield unique_id
|
||||
|
||||
|
||||
class PathSelectorMethod(SelectorMethod):
|
||||
@@ -328,33 +373,37 @@ class PathSelectorMethod(SelectorMethod):
|
||||
# get project root from contextvar
|
||||
root = Path(cv_project_root.get())
|
||||
paths = set(p.relative_to(root) for p in root.glob(selector))
|
||||
for node, real_node in self.all_nodes(included_nodes):
|
||||
ofp = Path(real_node.original_file_path)
|
||||
for unique_id, node in self.all_nodes(included_nodes):
|
||||
if node.resource_type == NodeType.PublicModel: # public models have no path
|
||||
continue
|
||||
ofp = Path(node.original_file_path)
|
||||
if ofp in paths:
|
||||
yield node
|
||||
if hasattr(real_node, "patch_path") and real_node.patch_path: # type: ignore
|
||||
pfp = real_node.patch_path.split("://")[1] # type: ignore
|
||||
yield unique_id
|
||||
if hasattr(node, "patch_path") and node.patch_path: # type: ignore
|
||||
pfp = node.patch_path.split("://")[1] # type: ignore
|
||||
ymlfp = Path(pfp)
|
||||
if ymlfp in paths:
|
||||
yield node
|
||||
yield unique_id
|
||||
if any(parent in paths for parent in ofp.parents):
|
||||
yield node
|
||||
yield unique_id
|
||||
|
||||
|
||||
class FileSelectorMethod(SelectorMethod):
|
||||
def search(self, included_nodes: Set[UniqueId], selector: str) -> Iterator[UniqueId]:
|
||||
"""Yields nodes from included that match the given file name."""
|
||||
for node, real_node in self.all_nodes(included_nodes):
|
||||
if fnmatch(Path(real_node.original_file_path).name, selector):
|
||||
yield node
|
||||
for unique_id, node in self.all_nodes(included_nodes):
|
||||
if node.resource_type == NodeType.PublicModel: # public models have no file
|
||||
continue
|
||||
if fnmatch(Path(node.original_file_path).name, selector):
|
||||
yield unique_id
|
||||
|
||||
|
||||
class PackageSelectorMethod(SelectorMethod):
|
||||
def search(self, included_nodes: Set[UniqueId], selector: str) -> Iterator[UniqueId]:
|
||||
"""Yields nodes from included that have the specified package"""
|
||||
for node, real_node in self.all_nodes(included_nodes):
|
||||
if fnmatch(real_node.package_name, selector):
|
||||
yield node
|
||||
for unique_id, node in self.all_nodes(included_nodes):
|
||||
if fnmatch(node.package_name, selector):
|
||||
yield unique_id
|
||||
|
||||
|
||||
def _getattr_descend(obj: Any, attrs: List[str]) -> Any:
|
||||
@@ -396,9 +445,9 @@ class ConfigSelectorMethod(SelectorMethod):
|
||||
# search sources is kind of useless now source configs only have
|
||||
# 'enabled', which you can't really filter on anyway, but maybe we'll
|
||||
# add more someday, so search them anyway.
|
||||
for node, real_node in self.configurable_nodes(included_nodes):
|
||||
for unique_id, node in self.configurable_nodes(included_nodes):
|
||||
try:
|
||||
value = _getattr_descend(real_node.config, parts)
|
||||
value = _getattr_descend(node.config, parts)
|
||||
except AttributeError:
|
||||
continue
|
||||
else:
|
||||
@@ -408,7 +457,7 @@ class ConfigSelectorMethod(SelectorMethod):
|
||||
or (CaseInsensitive(selector) == "true" and True in value)
|
||||
or (CaseInsensitive(selector) == "false" and False in value)
|
||||
):
|
||||
yield node
|
||||
yield unique_id
|
||||
else:
|
||||
if (
|
||||
(selector == value)
|
||||
@@ -416,7 +465,7 @@ class ConfigSelectorMethod(SelectorMethod):
|
||||
or (CaseInsensitive(selector) == "false")
|
||||
and value is False
|
||||
):
|
||||
yield node
|
||||
yield unique_id
|
||||
|
||||
|
||||
class ResourceTypeSelectorMethod(SelectorMethod):
|
||||
@@ -425,17 +474,17 @@ class ResourceTypeSelectorMethod(SelectorMethod):
|
||||
resource_type = NodeType(selector)
|
||||
except ValueError as exc:
|
||||
raise DbtRuntimeError(f'Invalid resource_type selector "{selector}"') from exc
|
||||
for node, real_node in self.parsed_nodes(included_nodes):
|
||||
if real_node.resource_type == resource_type:
|
||||
yield node
|
||||
for unique_id, node in self.parsed_and_public_nodes(included_nodes):
|
||||
if node.resource_type == resource_type:
|
||||
yield unique_id
|
||||
|
||||
|
||||
class TestNameSelectorMethod(SelectorMethod):
|
||||
def search(self, included_nodes: Set[UniqueId], selector: str) -> Iterator[UniqueId]:
|
||||
for node, real_node in self.parsed_nodes(included_nodes):
|
||||
if real_node.resource_type == NodeType.Test and hasattr(real_node, "test_metadata"):
|
||||
if fnmatch(real_node.test_metadata.name, selector): # type: ignore[union-attr]
|
||||
yield node
|
||||
for unique_id, node in self.parsed_nodes(included_nodes):
|
||||
if node.resource_type == NodeType.Test and hasattr(node, "test_metadata"):
|
||||
if fnmatch(node.test_metadata.name, selector): # type: ignore[union-attr]
|
||||
yield unique_id
|
||||
|
||||
|
||||
class TestTypeSelectorMethod(SelectorMethod):
|
||||
@@ -451,9 +500,9 @@ class TestTypeSelectorMethod(SelectorMethod):
|
||||
f'Invalid test type selector {selector}: expected "generic" or ' '"singular"'
|
||||
)
|
||||
|
||||
for node, real_node in self.parsed_nodes(included_nodes):
|
||||
if isinstance(real_node, search_type):
|
||||
yield node
|
||||
for unique_id, node in self.parsed_nodes(included_nodes):
|
||||
if isinstance(node, search_type):
|
||||
yield unique_id
|
||||
|
||||
|
||||
class StateSelectorMethod(SelectorMethod):
|
||||
@@ -602,24 +651,24 @@ class StateSelectorMethod(SelectorMethod):
|
||||
|
||||
manifest: WritableManifest = self.previous_state.manifest
|
||||
|
||||
for node, real_node in self.all_nodes(included_nodes):
|
||||
for unique_id, node in self.all_nodes(included_nodes):
|
||||
previous_node: Optional[SelectorTarget] = None
|
||||
|
||||
if node in manifest.nodes:
|
||||
previous_node = manifest.nodes[node]
|
||||
elif node in manifest.sources:
|
||||
previous_node = manifest.sources[node]
|
||||
elif node in manifest.exposures:
|
||||
previous_node = manifest.exposures[node]
|
||||
elif node in manifest.metrics:
|
||||
previous_node = manifest.metrics[node]
|
||||
if unique_id in manifest.nodes:
|
||||
previous_node = manifest.nodes[unique_id]
|
||||
elif unique_id in manifest.sources:
|
||||
previous_node = manifest.sources[unique_id]
|
||||
elif unique_id in manifest.exposures:
|
||||
previous_node = manifest.exposures[unique_id]
|
||||
elif unique_id in manifest.metrics:
|
||||
previous_node = manifest.metrics[unique_id]
|
||||
|
||||
keyword_args = {}
|
||||
if checker.__name__ in ["same_contract", "check_modified_content"]:
|
||||
keyword_args["adapter_type"] = adapter_type # type: ignore
|
||||
|
||||
if checker(previous_node, real_node, **keyword_args): # type: ignore
|
||||
yield node
|
||||
if checker(previous_node, node, **keyword_args): # type: ignore
|
||||
yield unique_id
|
||||
|
||||
|
||||
class ResultSelectorMethod(SelectorMethod):
|
||||
@@ -629,9 +678,9 @@ class ResultSelectorMethod(SelectorMethod):
|
||||
matches = set(
|
||||
result.unique_id for result in self.previous_state.results if result.status == selector
|
||||
)
|
||||
for node, real_node in self.all_nodes(included_nodes):
|
||||
if node in matches:
|
||||
yield node
|
||||
for unique_id, node in self.all_nodes(included_nodes):
|
||||
if unique_id in matches:
|
||||
yield unique_id
|
||||
|
||||
|
||||
class SourceStatusSelectorMethod(SelectorMethod):
|
||||
@@ -683,37 +732,37 @@ class SourceStatusSelectorMethod(SelectorMethod):
|
||||
):
|
||||
matches.remove(unique_id)
|
||||
|
||||
for node, real_node in self.all_nodes(included_nodes):
|
||||
if node in matches:
|
||||
yield node
|
||||
for unique_id, node in self.all_nodes(included_nodes):
|
||||
if unique_id in matches:
|
||||
yield unique_id
|
||||
|
||||
|
||||
class VersionSelectorMethod(SelectorMethod):
|
||||
def search(self, included_nodes: Set[UniqueId], selector: str) -> Iterator[UniqueId]:
|
||||
for node, real_node in self.parsed_nodes(included_nodes):
|
||||
if isinstance(real_node, ModelNode):
|
||||
for unique_id, node in self.parsed_and_public_nodes(included_nodes):
|
||||
if isinstance(node, ModelNode):
|
||||
if selector == "latest":
|
||||
if real_node.is_latest_version:
|
||||
yield node
|
||||
if node.is_latest_version:
|
||||
yield unique_id
|
||||
elif selector == "prerelease":
|
||||
if (
|
||||
real_node.version
|
||||
and real_node.latest_version
|
||||
and UnparsedVersion(v=real_node.version)
|
||||
> UnparsedVersion(v=real_node.latest_version)
|
||||
node.version
|
||||
and node.latest_version
|
||||
and UnparsedVersion(v=node.version)
|
||||
> UnparsedVersion(v=node.latest_version)
|
||||
):
|
||||
yield node
|
||||
yield unique_id
|
||||
elif selector == "old":
|
||||
if (
|
||||
real_node.version
|
||||
and real_node.latest_version
|
||||
and UnparsedVersion(v=real_node.version)
|
||||
< UnparsedVersion(v=real_node.latest_version)
|
||||
node.version
|
||||
and node.latest_version
|
||||
and UnparsedVersion(v=node.version)
|
||||
< UnparsedVersion(v=node.latest_version)
|
||||
):
|
||||
yield node
|
||||
yield unique_id
|
||||
elif selector == "none":
|
||||
if real_node.version is None:
|
||||
yield node
|
||||
if node.version is None:
|
||||
yield unique_id
|
||||
else:
|
||||
raise DbtRuntimeError(
|
||||
f'Invalid version type selector {selector}: expected one of: "latest", "prerelease", "old", or "none"'
|
||||
@@ -740,6 +789,7 @@ class MethodManager:
|
||||
MethodName.Result: ResultSelectorMethod,
|
||||
MethodName.SourceStatus: SourceStatusSelectorMethod,
|
||||
MethodName.Version: VersionSelectorMethod,
|
||||
MethodName.PublicModel: PublicModelSelectorMethod,
|
||||
}
|
||||
|
||||
def __init__(
|
||||
|
||||
@@ -60,7 +60,7 @@ SelectionSpec = Union[
|
||||
|
||||
|
||||
@dataclass
|
||||
class SelectionCriteria:
|
||||
class SelectionCriteria(dbtClassMixin):
|
||||
raw: Any
|
||||
method: MethodName
|
||||
method_arguments: List[str]
|
||||
|
||||
@@ -33,6 +33,7 @@ class NodeType(StrEnum):
|
||||
Exposure = "exposure"
|
||||
Metric = "metric"
|
||||
Group = "group"
|
||||
PublicModel = "publicmodel"
|
||||
SemanticModel = "semanticmodel"
|
||||
|
||||
@classmethod
|
||||
@@ -55,12 +56,14 @@ class NodeType(StrEnum):
|
||||
cls.Model,
|
||||
cls.Seed,
|
||||
cls.Snapshot,
|
||||
cls.PublicModel,
|
||||
]
|
||||
|
||||
@classmethod
|
||||
def versioned(cls) -> List["NodeType"]:
|
||||
return [
|
||||
cls.Model,
|
||||
cls.PublicModel,
|
||||
]
|
||||
|
||||
@classmethod
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import json
|
||||
|
||||
from dbt.contracts.graph.nodes import Exposure, SourceDefinition, Metric
|
||||
from dbt.contracts.publication import PublicModel
|
||||
from dbt.flags import get_flags
|
||||
from dbt.graph import ResourceTypeSelector
|
||||
from dbt.task.runnable import GraphRunnableTask
|
||||
@@ -27,6 +28,7 @@ class ListTask(GraphRunnableTask):
|
||||
NodeType.Source,
|
||||
NodeType.Exposure,
|
||||
NodeType.Metric,
|
||||
NodeType.PublicModel,
|
||||
)
|
||||
)
|
||||
ALL_RESOURCE_VALUES = DEFAULT_RESOURCE_VALUES | frozenset((NodeType.Analysis,))
|
||||
@@ -56,7 +58,9 @@ class ListTask(GraphRunnableTask):
|
||||
)
|
||||
|
||||
def _iterate_selected_nodes(self):
|
||||
# For list command, either ResourceTypeSelector or TestSelector
|
||||
selector = self.get_node_selector()
|
||||
# Get selection spec matching arguments or use the DEFAULT_INCLUDES from dbt.graph.cli
|
||||
spec = self.get_selection_spec()
|
||||
nodes = sorted(selector.get_selected(spec))
|
||||
if not nodes:
|
||||
@@ -73,6 +77,8 @@ class ListTask(GraphRunnableTask):
|
||||
yield self.manifest.exposures[node]
|
||||
elif node in self.manifest.metrics:
|
||||
yield self.manifest.metrics[node]
|
||||
elif node in self.manifest.public_nodes:
|
||||
yield self.manifest.public_nodes[node]
|
||||
else:
|
||||
raise DbtRuntimeError(
|
||||
f'Got an unexpected result from node selection: "{node}"'
|
||||
@@ -96,6 +102,10 @@ class ListTask(GraphRunnableTask):
|
||||
# metrics are searched for by pkg.metric_name
|
||||
metric_selector = ".".join([node.package_name, node.name])
|
||||
yield f"metric:{metric_selector}"
|
||||
elif node.resource_type == NodeType.PublicModel:
|
||||
assert isinstance(node, PublicModel)
|
||||
pub_model_selector = ".".join([node.package_name, node.name])
|
||||
yield f"public:{pub_model_selector}"
|
||||
else:
|
||||
# everything else is from `fqn`
|
||||
yield ".".join(node.fqn)
|
||||
@@ -109,7 +119,7 @@ class ListTask(GraphRunnableTask):
|
||||
yield json.dumps(
|
||||
{
|
||||
k: v
|
||||
for k, v in node.to_dict(omit_none=False).items()
|
||||
for k, v in node.to_ls_dict(omit_none=False).items()
|
||||
if (
|
||||
k in self.args.output_keys
|
||||
if self.args.output_keys
|
||||
@@ -152,7 +162,7 @@ class ListTask(GraphRunnableTask):
|
||||
@property
|
||||
def resource_types(self):
|
||||
if self.args.models:
|
||||
return [NodeType.Model]
|
||||
return [NodeType.Model, NodeType.PublicModel]
|
||||
|
||||
if not self.args.resource_types:
|
||||
return list(self.DEFAULT_RESOURCE_VALUES)
|
||||
|
||||
@@ -118,6 +118,8 @@ class GraphRunnableTask(ConfiguredTask):
|
||||
spec = self.config.get_selector(default_selector_name)
|
||||
else:
|
||||
# use --select and --exclude args
|
||||
# This is used when no selection is specified, and will use the DEFAULT_INCLUDES
|
||||
# from dbt.graph.cli.
|
||||
spec = parse_difference(self.selection_arg, self.exclusion_arg, indirect_selection)
|
||||
return spec
|
||||
|
||||
|
||||
@@ -16,6 +16,7 @@ node_type_pluralizations = {
|
||||
NodeType.Exposure: "exposures",
|
||||
NodeType.Metric: "metrics",
|
||||
NodeType.Group: "groups",
|
||||
NodeType.PublicModel: "publicmodels",
|
||||
NodeType.SemanticModel: "semanticmodels",
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user