Compare commits

...

6 Commits

5 changed files with 271 additions and 8 deletions

View File

@@ -0,0 +1,6 @@
kind: Features
body: Let ConfigSelectorMethod search through all nodes to return matching selections
time: 2025-06-27T09:40:00.485654-05:00
custom:
Author: trouze
Issue: "11607"

View File

@@ -22,7 +22,6 @@ from dbt.contracts.graph.nodes import (
ManifestNode,
Metric,
ModelNode,
ResultNode,
SavedQuery,
SemanticModel,
SingularTestNode,
@@ -205,11 +204,6 @@ class SelectorMethod(metaclass=abc.ABCMeta):
self.saved_query_nodes(included_nodes),
)
def configurable_nodes(
self, included_nodes: Set[UniqueId]
) -> Iterator[Tuple[UniqueId, ResultNode]]:
yield from chain(self.parsed_nodes(included_nodes), self.source_nodes(included_nodes))
def non_source_nodes(
self,
included_nodes: Set[UniqueId],
@@ -536,7 +530,7 @@ class ConfigSelectorMethod(SelectorMethod):
# search sources is kind of useless now source configs only have
# 'enabled', which you can't really filter on anyway, but maybe we'll
# add more someday, so search them anyway.
for unique_id, node in self.configurable_nodes(included_nodes):
for unique_id, node in self.all_nodes(included_nodes):
try:
value = _getattr_descend(node.config, parts)
except AttributeError:

View File

@@ -109,7 +109,8 @@ users_sql = """
{{
config(
materialized = 'table',
tags=['bi', 'users']
tags=['bi', 'users'],
meta={'contains_pii':true}
)
}}

View File

@@ -0,0 +1,140 @@
"""
Functional tests for config selector functionality.
These tests validate that config selectors work correctly after the change from
configurable_nodes() to all_nodes() in ConfigSelectorMethod.
"""
import pytest
from dbt.tests.util import run_dbt
from tests.functional.graph_selection.fixtures import (
alternative_users_sql,
base_users_sql,
emails_alt_sql,
emails_sql,
nested_users_sql,
never_selected_sql,
schema_yml,
subdir_sql,
users_rollup_dependency_sql,
users_rollup_sql,
users_sql,
)
class TestConfigSelection:
"""Test config selectors work on all node types including newly supported ones."""
@pytest.fixture(scope="class")
def project_config_update(self):
return {
"config-version": 2,
}
@pytest.fixture(scope="class")
def models(self):
return {
"schema.yml": schema_yml,
"base_users.sql": base_users_sql,
"users.sql": users_sql,
"users_rollup.sql": users_rollup_sql,
"versioned_v3.sql": base_users_sql,
"users_rollup_dependency.sql": users_rollup_dependency_sql,
"emails.sql": emails_sql,
"emails_alt.sql": emails_alt_sql,
"alternative.users.sql": alternative_users_sql,
"never_selected.sql": never_selected_sql,
"test": {
"subdir.sql": subdir_sql,
"subdir": {"nested_users.sql": nested_users_sql},
},
"metricflow_time_spine.sql": "SELECT to_date('02/20/2023', 'mm/dd/yyyy') as date_day",
"semantic_models.yml": """
version: 2
semantic_models:
- name: test_semantic_model
label: "Test Semantic Model"
model: ref('users')
dimensions:
- name: created_at
type: time
type_params:
time_granularity: day
measures:
- name: total_count
agg: count
expr: 1
entities:
- name: id
type: primary
defaults:
agg_time_dimension: created_at
metrics:
- name: test_metric
label: "Test Metric"
type: simple
config:
enabled: true
meta:
metric_type: simple
type_params:
measure: total_count
saved_queries:
- name: test_saved_query
label: "Test Saved Query"
config:
enabled: true
meta:
query_type: basic
contains_pii: true
query_params:
metrics:
- test_metric
group_by:
- "Dimension('test_semantic_model__created_at')"
exports:
- name: test_export
config:
alias: test_export_alias
export_as: table
""",
}
def test_config_selector_with_resource_type_filter(self, project):
"""Test config selectors with resource type filters."""
results = run_dbt(["list", "--resource-type", "model", "--select", "config.enabled:true"])
selected_nodes = set(results)
assert "saved_query:test.test_saved_query" not in selected_nodes
assert "metric:test.test_metric" not in selected_nodes
assert "semantic_model:test.test_semantic_model" not in selected_nodes
def test_config_enabled_true_selects_extended_nodes(self, project):
"""Test that dbt ls -s config.enabled:true returns the test_saved_query.
This specific test validates that the saved query (which has config.enabled:true)
is properly selected by the config selector. This demonstrates that the change from
configurable_nodes() to all_nodes() allows config selectors to work on saved queries.
"""
results = run_dbt(["list", "--select", "config.enabled:true"])
selected_nodes = set(results)
assert "saved_query:test.test_saved_query" in selected_nodes
assert "metric:test.test_metric" in selected_nodes
assert "semantic_model:test.test_semantic_model" in selected_nodes
def test_config_meta_selection(self, project):
""" """
results = run_dbt(["list", "--select", "config.meta.contains_pii:true"])
selected_nodes = set(results)
assert "test.users" in selected_nodes
assert "saved_query:test.test_saved_query" in selected_nodes
assert "test.unique_users_id" in selected_nodes

View File

@@ -427,6 +427,128 @@ def test_select_config_meta(manifest):
assert not search_manifest_using_method(manifest, list_method, "other") == {"table_model"}
def test_select_config_enabled_on_all_nodes(manifest):
"""Test that ConfigSelectorMethod can find enabled property on all node types.
This test verifies that the change from configurable_nodes() to all_nodes()
allows the ConfigSelectorMethod to search through all node types, not just
the previous subset of parsed_nodes and source_nodes.
"""
methods = MethodManager(manifest, None)
method = methods.get_method("config", ["enabled"])
assert isinstance(method, ConfigSelectorMethod)
assert method.arguments == ["enabled"]
# All nodes should have enabled: true by default
enabled_nodes = search_manifest_using_method(manifest, method, "true")
# Should include traditional configurable nodes (models, sources)
assert "table_model" in enabled_nodes
assert "view_model" in enabled_nodes
assert "ext_raw.ext_source" in enabled_nodes
# Should now also include the new node types that were added via all_nodes()
# Metrics, semantic models, saved queries should all have enabled: true by default
assert "my_metric" in enabled_nodes
assert "test_semantic_model" in enabled_nodes
assert "test_saved_query" in enabled_nodes
def test_select_config_searches_more_nodes_than_before(manifest):
"""Test that ConfigSelectorMethod now searches more node types than before.
This test verifies that the change from configurable_nodes() to all_nodes()
actually expands the set of searchable nodes beyond just models and sources.
"""
methods = MethodManager(manifest, None)
# Test that we can search for config properties that exist on various node types
enabled_method = methods.get_method("config", ["enabled"])
all_enabled_nodes = search_manifest_using_method(manifest, enabled_method, "true")
# Verify specific new node types are found (these were not in configurable_nodes())
metrics = [node for node in all_enabled_nodes if "my_metric" in node]
semantic_models = [node for node in all_enabled_nodes if "test_semantic_model" in node]
saved_queries = [node for node in all_enabled_nodes if "test_saved_query" in node]
# With all_nodes(), we should find these new node types
assert len(metrics) > 0, "Should find metric nodes (new with all_nodes)"
assert len(semantic_models) > 0, "Should find semantic model nodes (new with all_nodes)"
assert len(saved_queries) > 0, "Should find saved query nodes (new with all_nodes)"
# Specifically check that these new node types are found
assert "my_metric" in all_enabled_nodes, "Metric should be searchable with config selector"
assert (
"test_semantic_model" in all_enabled_nodes
), "Semantic model should be searchable with config selector"
assert (
"test_saved_query" in all_enabled_nodes
), "Saved query should be searchable with config selector"
# The total should include many types of nodes due to all_nodes() expansion
total_nodes = len(all_enabled_nodes)
assert (
total_nodes > 20
), f"Should find many nodes with all_nodes() expansion, found {total_nodes}"
def test_config_selector_finds_nodes_not_in_old_configurable_nodes(manifest):
"""Test that ConfigSelectorMethod now finds nodes that were excluded by configurable_nodes().
Before the change, ConfigSelectorMethod.search() used configurable_nodes() which only
included parsed_nodes() and source_nodes(). Now it uses all_nodes() which includes
exposure_nodes(), metric_nodes(), unit_tests(), semantic_model_nodes(), and saved_query_nodes().
This test verifies that the new node types are now searchable.
"""
methods = MethodManager(manifest, None)
method = methods.get_method("config", ["enabled"])
# Search for enabled nodes
enabled_nodes = search_manifest_using_method(manifest, method, "true")
# These node types were NOT included in the old configurable_nodes() method
# but should now be found via all_nodes()
# Metrics should be found (they have MetricConfig with enabled property)
# search_name for metrics is just the metric name, not the full unique_id
assert (
"my_metric" in enabled_nodes
), "Specific metric should be found by config selector (new functionality)"
# Semantic models should be found (they have SemanticModelConfig with enabled property)
assert (
"test_semantic_model" in enabled_nodes
), "Specific semantic model should be found by config selector (new functionality)"
# Saved queries should be found (they have SavedQueryConfig with enabled property)
assert (
"test_saved_query" in enabled_nodes
), "Specific saved query should be found by config selector (new functionality)"
# Unit tests should also be found (they have config.enabled)
unit_tests_found = [node for node in enabled_nodes if "unit_test" in node]
assert (
len(unit_tests_found) > 0
), "Unit tests should be found by config selector (new functionality)"
# Count the new node types we can find
new_node_types = 0
if "my_metric" in enabled_nodes:
new_node_types += 1
if "test_semantic_model" in enabled_nodes:
new_node_types += 1
if "test_saved_query" in enabled_nodes:
new_node_types += 1
new_node_types += len(unit_tests_found)
# This demonstrates the key change: ConfigSelectorMethod can now search through
# many more node types than just the traditional "configurable" nodes
assert (
new_node_types >= 4
), f"Should find at least 4 nodes from newly included types, found {new_node_types}"
def test_select_test_name(manifest):
methods = MethodManager(manifest, None)
method = methods.get_method("test_name", [])