mirror of
https://github.com/dbt-labs/dbt-core
synced 2025-12-21 18:21:27 +00:00
Compare commits
6 Commits
enable-pos
...
trouze/ext
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
3b12a3667a | ||
|
|
70b169147e | ||
|
|
c4d8f3c695 | ||
|
|
5d37b84f8b | ||
|
|
3481c3cc45 | ||
|
|
83578afb9d |
6
.changes/unreleased/Features-20250627-094000.yaml
Normal file
6
.changes/unreleased/Features-20250627-094000.yaml
Normal file
@@ -0,0 +1,6 @@
|
||||
kind: Features
|
||||
body: Let ConfigSelectorMethod search through all nodes to return matching selections
|
||||
time: 2025-06-27T09:40:00.485654-05:00
|
||||
custom:
|
||||
Author: trouze
|
||||
Issue: "11607"
|
||||
@@ -22,7 +22,6 @@ from dbt.contracts.graph.nodes import (
|
||||
ManifestNode,
|
||||
Metric,
|
||||
ModelNode,
|
||||
ResultNode,
|
||||
SavedQuery,
|
||||
SemanticModel,
|
||||
SingularTestNode,
|
||||
@@ -205,11 +204,6 @@ class SelectorMethod(metaclass=abc.ABCMeta):
|
||||
self.saved_query_nodes(included_nodes),
|
||||
)
|
||||
|
||||
def configurable_nodes(
|
||||
self, included_nodes: Set[UniqueId]
|
||||
) -> Iterator[Tuple[UniqueId, ResultNode]]:
|
||||
yield from chain(self.parsed_nodes(included_nodes), self.source_nodes(included_nodes))
|
||||
|
||||
def non_source_nodes(
|
||||
self,
|
||||
included_nodes: Set[UniqueId],
|
||||
@@ -536,7 +530,7 @@ class ConfigSelectorMethod(SelectorMethod):
|
||||
# search sources is kind of useless now source configs only have
|
||||
# 'enabled', which you can't really filter on anyway, but maybe we'll
|
||||
# add more someday, so search them anyway.
|
||||
for unique_id, node in self.configurable_nodes(included_nodes):
|
||||
for unique_id, node in self.all_nodes(included_nodes):
|
||||
try:
|
||||
value = _getattr_descend(node.config, parts)
|
||||
except AttributeError:
|
||||
|
||||
@@ -109,7 +109,8 @@ users_sql = """
|
||||
{{
|
||||
config(
|
||||
materialized = 'table',
|
||||
tags=['bi', 'users']
|
||||
tags=['bi', 'users'],
|
||||
meta={'contains_pii':true}
|
||||
)
|
||||
}}
|
||||
|
||||
|
||||
140
tests/functional/graph_selection/test_config_selection.py
Normal file
140
tests/functional/graph_selection/test_config_selection.py
Normal file
@@ -0,0 +1,140 @@
|
||||
"""
|
||||
Functional tests for config selector functionality.
|
||||
|
||||
These tests validate that config selectors work correctly after the change from
|
||||
configurable_nodes() to all_nodes() in ConfigSelectorMethod.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
from dbt.tests.util import run_dbt
|
||||
from tests.functional.graph_selection.fixtures import (
|
||||
alternative_users_sql,
|
||||
base_users_sql,
|
||||
emails_alt_sql,
|
||||
emails_sql,
|
||||
nested_users_sql,
|
||||
never_selected_sql,
|
||||
schema_yml,
|
||||
subdir_sql,
|
||||
users_rollup_dependency_sql,
|
||||
users_rollup_sql,
|
||||
users_sql,
|
||||
)
|
||||
|
||||
|
||||
class TestConfigSelection:
|
||||
"""Test config selectors work on all node types including newly supported ones."""
|
||||
|
||||
@pytest.fixture(scope="class")
|
||||
def project_config_update(self):
|
||||
return {
|
||||
"config-version": 2,
|
||||
}
|
||||
|
||||
@pytest.fixture(scope="class")
|
||||
def models(self):
|
||||
return {
|
||||
"schema.yml": schema_yml,
|
||||
"base_users.sql": base_users_sql,
|
||||
"users.sql": users_sql,
|
||||
"users_rollup.sql": users_rollup_sql,
|
||||
"versioned_v3.sql": base_users_sql,
|
||||
"users_rollup_dependency.sql": users_rollup_dependency_sql,
|
||||
"emails.sql": emails_sql,
|
||||
"emails_alt.sql": emails_alt_sql,
|
||||
"alternative.users.sql": alternative_users_sql,
|
||||
"never_selected.sql": never_selected_sql,
|
||||
"test": {
|
||||
"subdir.sql": subdir_sql,
|
||||
"subdir": {"nested_users.sql": nested_users_sql},
|
||||
},
|
||||
"metricflow_time_spine.sql": "SELECT to_date('02/20/2023', 'mm/dd/yyyy') as date_day",
|
||||
"semantic_models.yml": """
|
||||
version: 2
|
||||
|
||||
semantic_models:
|
||||
- name: test_semantic_model
|
||||
label: "Test Semantic Model"
|
||||
model: ref('users')
|
||||
dimensions:
|
||||
- name: created_at
|
||||
type: time
|
||||
type_params:
|
||||
time_granularity: day
|
||||
measures:
|
||||
- name: total_count
|
||||
agg: count
|
||||
expr: 1
|
||||
entities:
|
||||
- name: id
|
||||
type: primary
|
||||
defaults:
|
||||
agg_time_dimension: created_at
|
||||
|
||||
metrics:
|
||||
- name: test_metric
|
||||
label: "Test Metric"
|
||||
type: simple
|
||||
config:
|
||||
enabled: true
|
||||
meta:
|
||||
metric_type: simple
|
||||
type_params:
|
||||
measure: total_count
|
||||
|
||||
saved_queries:
|
||||
- name: test_saved_query
|
||||
label: "Test Saved Query"
|
||||
config:
|
||||
enabled: true
|
||||
meta:
|
||||
query_type: basic
|
||||
contains_pii: true
|
||||
query_params:
|
||||
metrics:
|
||||
- test_metric
|
||||
group_by:
|
||||
- "Dimension('test_semantic_model__created_at')"
|
||||
exports:
|
||||
- name: test_export
|
||||
config:
|
||||
alias: test_export_alias
|
||||
export_as: table
|
||||
""",
|
||||
}
|
||||
|
||||
def test_config_selector_with_resource_type_filter(self, project):
|
||||
"""Test config selectors with resource type filters."""
|
||||
|
||||
results = run_dbt(["list", "--resource-type", "model", "--select", "config.enabled:true"])
|
||||
selected_nodes = set(results)
|
||||
|
||||
assert "saved_query:test.test_saved_query" not in selected_nodes
|
||||
assert "metric:test.test_metric" not in selected_nodes
|
||||
assert "semantic_model:test.test_semantic_model" not in selected_nodes
|
||||
|
||||
def test_config_enabled_true_selects_extended_nodes(self, project):
|
||||
"""Test that dbt ls -s config.enabled:true returns the test_saved_query.
|
||||
|
||||
This specific test validates that the saved query (which has config.enabled:true)
|
||||
is properly selected by the config selector. This demonstrates that the change from
|
||||
configurable_nodes() to all_nodes() allows config selectors to work on saved queries.
|
||||
"""
|
||||
|
||||
results = run_dbt(["list", "--select", "config.enabled:true"])
|
||||
selected_nodes = set(results)
|
||||
|
||||
assert "saved_query:test.test_saved_query" in selected_nodes
|
||||
assert "metric:test.test_metric" in selected_nodes
|
||||
assert "semantic_model:test.test_semantic_model" in selected_nodes
|
||||
|
||||
def test_config_meta_selection(self, project):
|
||||
""" """
|
||||
|
||||
results = run_dbt(["list", "--select", "config.meta.contains_pii:true"])
|
||||
selected_nodes = set(results)
|
||||
|
||||
assert "test.users" in selected_nodes
|
||||
assert "saved_query:test.test_saved_query" in selected_nodes
|
||||
assert "test.unique_users_id" in selected_nodes
|
||||
@@ -427,6 +427,128 @@ def test_select_config_meta(manifest):
|
||||
assert not search_manifest_using_method(manifest, list_method, "other") == {"table_model"}
|
||||
|
||||
|
||||
def test_select_config_enabled_on_all_nodes(manifest):
|
||||
"""Test that ConfigSelectorMethod can find enabled property on all node types.
|
||||
|
||||
This test verifies that the change from configurable_nodes() to all_nodes()
|
||||
allows the ConfigSelectorMethod to search through all node types, not just
|
||||
the previous subset of parsed_nodes and source_nodes.
|
||||
"""
|
||||
methods = MethodManager(manifest, None)
|
||||
method = methods.get_method("config", ["enabled"])
|
||||
assert isinstance(method, ConfigSelectorMethod)
|
||||
assert method.arguments == ["enabled"]
|
||||
|
||||
# All nodes should have enabled: true by default
|
||||
enabled_nodes = search_manifest_using_method(manifest, method, "true")
|
||||
|
||||
# Should include traditional configurable nodes (models, sources)
|
||||
assert "table_model" in enabled_nodes
|
||||
assert "view_model" in enabled_nodes
|
||||
assert "ext_raw.ext_source" in enabled_nodes
|
||||
|
||||
# Should now also include the new node types that were added via all_nodes()
|
||||
# Metrics, semantic models, saved queries should all have enabled: true by default
|
||||
assert "my_metric" in enabled_nodes
|
||||
assert "test_semantic_model" in enabled_nodes
|
||||
assert "test_saved_query" in enabled_nodes
|
||||
|
||||
|
||||
def test_select_config_searches_more_nodes_than_before(manifest):
|
||||
"""Test that ConfigSelectorMethod now searches more node types than before.
|
||||
|
||||
This test verifies that the change from configurable_nodes() to all_nodes()
|
||||
actually expands the set of searchable nodes beyond just models and sources.
|
||||
"""
|
||||
methods = MethodManager(manifest, None)
|
||||
|
||||
# Test that we can search for config properties that exist on various node types
|
||||
enabled_method = methods.get_method("config", ["enabled"])
|
||||
all_enabled_nodes = search_manifest_using_method(manifest, enabled_method, "true")
|
||||
|
||||
# Verify specific new node types are found (these were not in configurable_nodes())
|
||||
metrics = [node for node in all_enabled_nodes if "my_metric" in node]
|
||||
semantic_models = [node for node in all_enabled_nodes if "test_semantic_model" in node]
|
||||
saved_queries = [node for node in all_enabled_nodes if "test_saved_query" in node]
|
||||
|
||||
# With all_nodes(), we should find these new node types
|
||||
assert len(metrics) > 0, "Should find metric nodes (new with all_nodes)"
|
||||
assert len(semantic_models) > 0, "Should find semantic model nodes (new with all_nodes)"
|
||||
assert len(saved_queries) > 0, "Should find saved query nodes (new with all_nodes)"
|
||||
|
||||
# Specifically check that these new node types are found
|
||||
assert "my_metric" in all_enabled_nodes, "Metric should be searchable with config selector"
|
||||
assert (
|
||||
"test_semantic_model" in all_enabled_nodes
|
||||
), "Semantic model should be searchable with config selector"
|
||||
assert (
|
||||
"test_saved_query" in all_enabled_nodes
|
||||
), "Saved query should be searchable with config selector"
|
||||
|
||||
# The total should include many types of nodes due to all_nodes() expansion
|
||||
total_nodes = len(all_enabled_nodes)
|
||||
assert (
|
||||
total_nodes > 20
|
||||
), f"Should find many nodes with all_nodes() expansion, found {total_nodes}"
|
||||
|
||||
|
||||
def test_config_selector_finds_nodes_not_in_old_configurable_nodes(manifest):
|
||||
"""Test that ConfigSelectorMethod now finds nodes that were excluded by configurable_nodes().
|
||||
|
||||
Before the change, ConfigSelectorMethod.search() used configurable_nodes() which only
|
||||
included parsed_nodes() and source_nodes(). Now it uses all_nodes() which includes
|
||||
exposure_nodes(), metric_nodes(), unit_tests(), semantic_model_nodes(), and saved_query_nodes().
|
||||
|
||||
This test verifies that the new node types are now searchable.
|
||||
"""
|
||||
methods = MethodManager(manifest, None)
|
||||
method = methods.get_method("config", ["enabled"])
|
||||
|
||||
# Search for enabled nodes
|
||||
enabled_nodes = search_manifest_using_method(manifest, method, "true")
|
||||
|
||||
# These node types were NOT included in the old configurable_nodes() method
|
||||
# but should now be found via all_nodes()
|
||||
|
||||
# Metrics should be found (they have MetricConfig with enabled property)
|
||||
# search_name for metrics is just the metric name, not the full unique_id
|
||||
assert (
|
||||
"my_metric" in enabled_nodes
|
||||
), "Specific metric should be found by config selector (new functionality)"
|
||||
|
||||
# Semantic models should be found (they have SemanticModelConfig with enabled property)
|
||||
assert (
|
||||
"test_semantic_model" in enabled_nodes
|
||||
), "Specific semantic model should be found by config selector (new functionality)"
|
||||
|
||||
# Saved queries should be found (they have SavedQueryConfig with enabled property)
|
||||
assert (
|
||||
"test_saved_query" in enabled_nodes
|
||||
), "Specific saved query should be found by config selector (new functionality)"
|
||||
|
||||
# Unit tests should also be found (they have config.enabled)
|
||||
unit_tests_found = [node for node in enabled_nodes if "unit_test" in node]
|
||||
assert (
|
||||
len(unit_tests_found) > 0
|
||||
), "Unit tests should be found by config selector (new functionality)"
|
||||
|
||||
# Count the new node types we can find
|
||||
new_node_types = 0
|
||||
if "my_metric" in enabled_nodes:
|
||||
new_node_types += 1
|
||||
if "test_semantic_model" in enabled_nodes:
|
||||
new_node_types += 1
|
||||
if "test_saved_query" in enabled_nodes:
|
||||
new_node_types += 1
|
||||
new_node_types += len(unit_tests_found)
|
||||
|
||||
# This demonstrates the key change: ConfigSelectorMethod can now search through
|
||||
# many more node types than just the traditional "configurable" nodes
|
||||
assert (
|
||||
new_node_types >= 4
|
||||
), f"Should find at least 4 nodes from newly included types, found {new_node_types}"
|
||||
|
||||
|
||||
def test_select_test_name(manifest):
|
||||
methods = MethodManager(manifest, None)
|
||||
method = methods.get_method("test_name", [])
|
||||
|
||||
Reference in New Issue
Block a user