mirror of
https://github.com/dbt-labs/dbt-core
synced 2025-12-17 19:31:34 +00:00
Compare commits
9 Commits
v1.6.1
...
feature/40
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
8f41453f7b | ||
|
|
fa42880d6e | ||
|
|
e3f699ce6a | ||
|
|
be117691ef | ||
|
|
f2b6e65f00 | ||
|
|
d347fb8b59 | ||
|
|
6afc0abd0d | ||
|
|
47df8259af | ||
|
|
a8f8ff09af |
@@ -17,7 +17,7 @@ PROFILES_DIR = os.path.expanduser(
|
||||
STRICT_MODE = False # Only here for backwards compatibility
|
||||
FULL_REFRESH = False # subcommand
|
||||
STORE_FAILURES = False # subcommand
|
||||
GREEDY = None # subcommand
|
||||
INDIRECT_SELECTION = 'eager' # subcommand
|
||||
|
||||
# Global CLI commands
|
||||
USE_EXPERIMENTAL_PARSER = None
|
||||
@@ -95,7 +95,7 @@ MP_CONTEXT = _get_context()
|
||||
def set_from_args(args, user_config):
|
||||
global STRICT_MODE, FULL_REFRESH, WARN_ERROR, \
|
||||
USE_EXPERIMENTAL_PARSER, STATIC_PARSER, WRITE_JSON, PARTIAL_PARSE, \
|
||||
USE_COLORS, STORE_FAILURES, PROFILES_DIR, DEBUG, LOG_FORMAT, GREEDY, \
|
||||
USE_COLORS, STORE_FAILURES, PROFILES_DIR, DEBUG, LOG_FORMAT, INDIRECT_SELECTION, \
|
||||
VERSION_CHECK, FAIL_FAST, SEND_ANONYMOUS_USAGE_STATS, PRINTER_WIDTH, \
|
||||
WHICH
|
||||
|
||||
@@ -103,7 +103,7 @@ def set_from_args(args, user_config):
|
||||
# cli args without user_config or env var option
|
||||
FULL_REFRESH = getattr(args, 'full_refresh', FULL_REFRESH)
|
||||
STORE_FAILURES = getattr(args, 'store_failures', STORE_FAILURES)
|
||||
GREEDY = getattr(args, 'greedy', GREEDY)
|
||||
INDIRECT_SELECTION = getattr(args, 'indirect_selection', INDIRECT_SELECTION)
|
||||
WHICH = getattr(args, 'which', WHICH)
|
||||
|
||||
# global cli flags with env var and user_config alternatives
|
||||
|
||||
@@ -16,6 +16,7 @@ from .selector_spec import (
|
||||
SelectionIntersection,
|
||||
SelectionDifference,
|
||||
SelectionCriteria,
|
||||
IndirectSelection
|
||||
)
|
||||
|
||||
INTERSECTION_DELIMITER = ','
|
||||
@@ -25,7 +26,7 @@ DEFAULT_EXCLUDES: List[str] = []
|
||||
|
||||
|
||||
def parse_union(
|
||||
components: List[str], expect_exists: bool, greedy: bool = False
|
||||
components: List[str], expect_exists: bool, indirect_selection: IndirectSelection = IndirectSelection.Eager
|
||||
) -> SelectionUnion:
|
||||
# turn ['a b', 'c'] -> ['a', 'b', 'c']
|
||||
raw_specs = itertools.chain.from_iterable(
|
||||
@@ -36,7 +37,7 @@ def parse_union(
|
||||
# ['a', 'b', 'c,d'] -> union('a', 'b', intersection('c', 'd'))
|
||||
for raw_spec in raw_specs:
|
||||
intersection_components: List[SelectionSpec] = [
|
||||
SelectionCriteria.from_single_spec(part, greedy=greedy)
|
||||
SelectionCriteria.from_single_spec(part, indirect_selection=indirect_selection)
|
||||
for part in raw_spec.split(INTERSECTION_DELIMITER)
|
||||
]
|
||||
union_components.append(SelectionIntersection(
|
||||
@@ -52,21 +53,25 @@ def parse_union(
|
||||
|
||||
|
||||
def parse_union_from_default(
|
||||
raw: Optional[List[str]], default: List[str], greedy: bool = False
|
||||
raw: Optional[List[str]], default: List[str], indirect_selection: IndirectSelection = IndirectSelection.Eager
|
||||
) -> SelectionUnion:
|
||||
components: List[str]
|
||||
expect_exists: bool
|
||||
if raw is None:
|
||||
return parse_union(components=default, expect_exists=False, greedy=greedy)
|
||||
return parse_union(components=default, expect_exists=False, indirect_selection=indirect_selection)
|
||||
else:
|
||||
return parse_union(components=raw, expect_exists=True, greedy=greedy)
|
||||
return parse_union(components=raw, expect_exists=True, indirect_selection=indirect_selection)
|
||||
|
||||
|
||||
def parse_difference(
|
||||
include: Optional[List[str]], exclude: Optional[List[str]]
|
||||
) -> SelectionDifference:
|
||||
included = parse_union_from_default(include, DEFAULT_INCLUDES, greedy=bool(flags.GREEDY))
|
||||
excluded = parse_union_from_default(exclude, DEFAULT_EXCLUDES, greedy=True)
|
||||
included = parse_union_from_default(
|
||||
include,
|
||||
DEFAULT_INCLUDES,
|
||||
indirect_selection=IndirectSelection(flags.INDIRECT_SELECTION)
|
||||
)
|
||||
excluded = parse_union_from_default(exclude, DEFAULT_EXCLUDES, indirect_selection=IndirectSelection.Eager)
|
||||
return SelectionDifference(components=[included, excluded])
|
||||
|
||||
|
||||
@@ -148,7 +153,7 @@ def parse_union_definition(definition: Dict[str, Any]) -> SelectionSpec:
|
||||
union_def_parts = _get_list_dicts(definition, 'union')
|
||||
include, exclude = _parse_include_exclude_subdefs(union_def_parts)
|
||||
|
||||
union = SelectionUnion(components=include, greedy_warning=False)
|
||||
union = SelectionUnion(components=include)
|
||||
|
||||
if exclude is None:
|
||||
union.raw = definition
|
||||
@@ -156,8 +161,7 @@ def parse_union_definition(definition: Dict[str, Any]) -> SelectionSpec:
|
||||
else:
|
||||
return SelectionDifference(
|
||||
components=[union, exclude],
|
||||
raw=definition,
|
||||
greedy_warning=False
|
||||
raw=definition
|
||||
)
|
||||
|
||||
|
||||
@@ -166,7 +170,7 @@ def parse_intersection_definition(
|
||||
) -> SelectionSpec:
|
||||
intersection_def_parts = _get_list_dicts(definition, 'intersection')
|
||||
include, exclude = _parse_include_exclude_subdefs(intersection_def_parts)
|
||||
intersection = SelectionIntersection(components=include, greedy_warning=False)
|
||||
intersection = SelectionIntersection(components=include)
|
||||
|
||||
if exclude is None:
|
||||
intersection.raw = definition
|
||||
@@ -174,8 +178,7 @@ def parse_intersection_definition(
|
||||
else:
|
||||
return SelectionDifference(
|
||||
components=[intersection, exclude],
|
||||
raw=definition,
|
||||
greedy_warning=False
|
||||
raw=definition
|
||||
)
|
||||
|
||||
|
||||
@@ -209,7 +212,7 @@ def parse_dict_definition(definition: Dict[str, Any]) -> SelectionSpec:
|
||||
if diff_arg is None:
|
||||
return base
|
||||
else:
|
||||
return SelectionDifference(components=[base, diff_arg], greedy_warning=False)
|
||||
return SelectionDifference(components=[base, diff_arg])
|
||||
|
||||
|
||||
def parse_from_definition(
|
||||
|
||||
@@ -3,7 +3,7 @@ from typing import Set, List, Optional, Tuple
|
||||
from .graph import Graph, UniqueId
|
||||
from .queue import GraphQueue
|
||||
from .selector_methods import MethodManager
|
||||
from .selector_spec import SelectionCriteria, SelectionSpec
|
||||
from .selector_spec import SelectionCriteria, SelectionSpec, IndirectSelection
|
||||
|
||||
from dbt.logger import GLOBAL_LOGGER as logger
|
||||
from dbt.node_types import NodeType
|
||||
@@ -29,24 +29,6 @@ def alert_non_existence(raw_spec, nodes):
|
||||
)
|
||||
|
||||
|
||||
def alert_unused_nodes(raw_spec, node_names):
|
||||
summary_nodes_str = ("\n - ").join(node_names[:3])
|
||||
debug_nodes_str = ("\n - ").join(node_names)
|
||||
and_more_str = f"\n - and {len(node_names) - 3} more" if len(node_names) > 4 else ""
|
||||
summary_msg = (
|
||||
f"\nSome tests were excluded because at least one parent is not selected. "
|
||||
f"Use the --greedy flag to include them."
|
||||
f"\n - {summary_nodes_str}{and_more_str}"
|
||||
)
|
||||
logger.info(summary_msg)
|
||||
if len(node_names) > 4:
|
||||
debug_msg = (
|
||||
f"Full list of tests that were excluded:"
|
||||
f"\n - {debug_nodes_str}"
|
||||
)
|
||||
logger.debug(debug_msg)
|
||||
|
||||
|
||||
def can_select_indirectly(node):
|
||||
"""If a node is not selected itself, but its parent(s) are, it may qualify
|
||||
for indirect selection.
|
||||
@@ -113,7 +95,7 @@ class NodeSelector(MethodManager):
|
||||
neighbors = self.collect_specified_neighbors(spec, collected)
|
||||
direct_nodes, indirect_nodes = self.expand_selection(
|
||||
selected=(collected | neighbors),
|
||||
greedy=spec.greedy
|
||||
indirect_selection=spec.indirect_selection
|
||||
)
|
||||
return direct_nodes, indirect_nodes
|
||||
|
||||
@@ -217,21 +199,20 @@ class NodeSelector(MethodManager):
|
||||
}
|
||||
|
||||
def expand_selection(
|
||||
self, selected: Set[UniqueId], greedy: bool = False
|
||||
self, selected: Set[UniqueId], indirect_selection: IndirectSelection = IndirectSelection.Eager
|
||||
) -> Tuple[Set[UniqueId], Set[UniqueId]]:
|
||||
# Test selection can expand to include an implicitly/indirectly selected test.
|
||||
# In this way, `dbt test -m model_a` also includes tests that directly depend on `model_a`.
|
||||
# Expansion has two modes, GREEDY and NOT GREEDY.
|
||||
# Test selection by default expands to include an implicitly/indirectly selected tests.
|
||||
# `dbt test -m model_a` also includes tests that directly depend on `model_a`.
|
||||
# Expansion has two modes, EAGER and CAUTIOUS.
|
||||
#
|
||||
# GREEDY mode: If ANY parent is selected, select the test. We use this for EXCLUSION.
|
||||
# EAGER mode: If ANY parent is selected, select the test.
|
||||
#
|
||||
# NOT GREEDY mode:
|
||||
# CAUTIOUS mode:
|
||||
# - If ALL parents are selected, select the test.
|
||||
# - If ANY parent is missing, return it separately. We'll keep it around
|
||||
# for later and see if its other parents show up.
|
||||
# We use this for INCLUSION.
|
||||
# Users can also opt in to inclusive GREEDY mode by passing --greedy flag,
|
||||
# or by specifying `greedy: true` in a yaml selector
|
||||
# Users can opt out of inclusive EAGER mode by passing --indirect-selection cautious
|
||||
# CLI argument or by specifying `indirect_selection: true` in a yaml selector
|
||||
|
||||
direct_nodes = set(selected)
|
||||
indirect_nodes = set()
|
||||
@@ -241,7 +222,10 @@ class NodeSelector(MethodManager):
|
||||
node = self.manifest.nodes[unique_id]
|
||||
if can_select_indirectly(node):
|
||||
# should we add it in directly?
|
||||
if greedy or set(node.depends_on.nodes) <= set(selected):
|
||||
if (
|
||||
indirect_selection == IndirectSelection.Eager
|
||||
or set(node.depends_on.nodes) <= set(selected)
|
||||
):
|
||||
direct_nodes.add(unique_id)
|
||||
# if not:
|
||||
else:
|
||||
@@ -255,6 +239,10 @@ class NodeSelector(MethodManager):
|
||||
# Check tests previously selected indirectly to see if ALL their
|
||||
# parents are now present.
|
||||
|
||||
# performance: if identical, skip the processing below
|
||||
if set(direct_nodes) == set(indirect_nodes):
|
||||
return direct_nodes
|
||||
|
||||
selected = set(direct_nodes)
|
||||
|
||||
for unique_id in indirect_nodes:
|
||||
@@ -278,16 +266,6 @@ class NodeSelector(MethodManager):
|
||||
selected_nodes, indirect_only = self.select_nodes(spec)
|
||||
filtered_nodes = self.filter_selection(selected_nodes)
|
||||
|
||||
if indirect_only:
|
||||
filtered_unused_nodes = self.filter_selection(indirect_only)
|
||||
if filtered_unused_nodes and spec.greedy_warning:
|
||||
# log anything that didn't make the cut
|
||||
unused_node_names = []
|
||||
for unique_id in filtered_unused_nodes:
|
||||
name = self.manifest.nodes[unique_id].name
|
||||
unused_node_names.append(name)
|
||||
alert_unused_nodes(spec, unused_node_names)
|
||||
|
||||
return filtered_nodes
|
||||
|
||||
def get_graph_queue(self, spec: SelectionSpec) -> GraphQueue:
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
import os
|
||||
import re
|
||||
import enum
|
||||
from abc import ABCMeta, abstractmethod
|
||||
from dataclasses import dataclass
|
||||
from dbt.dataclass_schema import StrEnum
|
||||
|
||||
from typing import (
|
||||
Set, Iterator, List, Optional, Dict, Union, Any, Iterable, Tuple
|
||||
@@ -21,6 +23,10 @@ RAW_SELECTOR_PATTERN = re.compile(
|
||||
)
|
||||
SELECTOR_METHOD_SEPARATOR = '.'
|
||||
|
||||
class IndirectSelection(StrEnum):
|
||||
Eager = 'eager'
|
||||
Cautious = 'cautious'
|
||||
|
||||
|
||||
def _probably_path(value: str):
|
||||
"""Decide if value is probably a path. Windows has two path separators, so
|
||||
@@ -66,8 +72,7 @@ class SelectionCriteria:
|
||||
parents_depth: Optional[int]
|
||||
children: bool
|
||||
children_depth: Optional[int]
|
||||
greedy: bool = False
|
||||
greedy_warning: bool = False # do not raise warning for yaml selectors
|
||||
indirect_selection: IndirectSelection = IndirectSelection.Eager
|
||||
|
||||
def __post_init__(self):
|
||||
if self.children and self.childrens_parents:
|
||||
@@ -105,7 +110,7 @@ class SelectionCriteria:
|
||||
|
||||
@classmethod
|
||||
def selection_criteria_from_dict(
|
||||
cls, raw: Any, dct: Dict[str, Any], greedy: bool = False
|
||||
cls, raw: Any, dct: Dict[str, Any], indirect_selection: IndirectSelection = IndirectSelection.Eager
|
||||
) -> 'SelectionCriteria':
|
||||
if 'value' not in dct:
|
||||
raise RuntimeException(
|
||||
@@ -125,7 +130,7 @@ class SelectionCriteria:
|
||||
parents_depth=parents_depth,
|
||||
children=bool(dct.get('children')),
|
||||
children_depth=children_depth,
|
||||
greedy=(greedy or bool(dct.get('greedy'))),
|
||||
indirect_selection=(IndirectSelection(dct.get('indirect_selection') or indirect_selection)),
|
||||
)
|
||||
|
||||
@classmethod
|
||||
@@ -146,18 +151,22 @@ class SelectionCriteria:
|
||||
dct['parents'] = bool(dct.get('parents'))
|
||||
if 'children' in dct:
|
||||
dct['children'] = bool(dct.get('children'))
|
||||
if 'greedy' in dct:
|
||||
dct['greedy'] = bool(dct.get('greedy'))
|
||||
if 'indirect_selection' in dct:
|
||||
dct['indirect_selection'] = bool(dct.get('indirect_selection'))
|
||||
return dct
|
||||
|
||||
@classmethod
|
||||
def from_single_spec(cls, raw: str, greedy: bool = False) -> 'SelectionCriteria':
|
||||
def from_single_spec(cls, raw: str, indirect_selection: bool = True) -> 'SelectionCriteria':
|
||||
result = RAW_SELECTOR_PATTERN.match(raw)
|
||||
if result is None:
|
||||
# bad spec!
|
||||
raise RuntimeException(f'Invalid selector spec "{raw}"')
|
||||
|
||||
return cls.selection_criteria_from_dict(raw, result.groupdict(), greedy=greedy)
|
||||
return cls.selection_criteria_from_dict(
|
||||
raw,
|
||||
result.groupdict(),
|
||||
indirect_selection=indirect_selection
|
||||
)
|
||||
|
||||
|
||||
class BaseSelectionGroup(Iterable[SelectionSpec], metaclass=ABCMeta):
|
||||
@@ -165,12 +174,10 @@ class BaseSelectionGroup(Iterable[SelectionSpec], metaclass=ABCMeta):
|
||||
self,
|
||||
components: Iterable[SelectionSpec],
|
||||
expect_exists: bool = False,
|
||||
greedy_warning: bool = True,
|
||||
raw: Any = None,
|
||||
):
|
||||
self.components: List[SelectionSpec] = list(components)
|
||||
self.expect_exists = expect_exists
|
||||
self.greedy_warning = greedy_warning
|
||||
self.raw = raw
|
||||
|
||||
def __iter__(self) -> Iterator[SelectionSpec]:
|
||||
|
||||
@@ -392,13 +392,15 @@ def _build_build_subparser(subparsers, base_subparser):
|
||||
'''
|
||||
)
|
||||
sub.add_argument(
|
||||
'--greedy',
|
||||
action='store_true',
|
||||
'--indirect-selection',
|
||||
choices=['eager', 'cautious'],
|
||||
dest='indirect_selection',
|
||||
help='''
|
||||
Select all tests that touch the selected resources,
|
||||
even if they also depend on unselected resources
|
||||
'''
|
||||
Select all tests that are adjacent to selected resources,
|
||||
even if they those resources have been explicitly selected.
|
||||
''',
|
||||
)
|
||||
|
||||
resource_values: List[str] = [
|
||||
str(s) for s in build_task.BuildTask.ALL_RESOURCE_VALUES
|
||||
] + ['all']
|
||||
@@ -736,12 +738,14 @@ def _build_test_subparser(subparsers, base_subparser):
|
||||
'''
|
||||
)
|
||||
sub.add_argument(
|
||||
'--greedy',
|
||||
action='store_true',
|
||||
'--indirect-selection',
|
||||
choices=['eager', 'cautious'],
|
||||
default='eager',
|
||||
dest='indirect_selection',
|
||||
help='''
|
||||
Select all tests that touch the selected resources,
|
||||
even if they also depend on unselected resources
|
||||
'''
|
||||
Select all tests that are adjacent to selected resources,
|
||||
even if they those resources have been explicitly selected.
|
||||
''',
|
||||
)
|
||||
|
||||
sub.set_defaults(cls=test_task.TestTask, which='test', rpc_method='test')
|
||||
@@ -839,12 +843,13 @@ def _build_list_subparser(subparsers, base_subparser):
|
||||
required=False,
|
||||
)
|
||||
sub.add_argument(
|
||||
'--greedy',
|
||||
action='store_true',
|
||||
'--indirect-selection',
|
||||
choices=['eager', 'cautious'],
|
||||
dest='indirect_selection',
|
||||
help='''
|
||||
Select all tests that touch the selected resources,
|
||||
even if they also depend on unselected resources
|
||||
'''
|
||||
Select all tests that are adjacent to selected resources,
|
||||
even if they those resources have been explicitly selected.
|
||||
''',
|
||||
)
|
||||
_add_common_selector_arguments(sub)
|
||||
|
||||
|
||||
@@ -21,16 +21,14 @@ class TestSelectionExpansion(DBTIntegrationTest):
|
||||
"test-paths": ["tests"]
|
||||
}
|
||||
|
||||
def list_tests_and_assert(self, include, exclude, expected_tests, greedy=False, selector_name=None):
|
||||
def list_tests_and_assert(self, include, exclude, expected_tests, indirect_selection='eager', selector_name=None):
|
||||
list_args = [ 'ls', '--resource-type', 'test']
|
||||
if include:
|
||||
list_args.extend(('--select', include))
|
||||
if exclude:
|
||||
list_args.extend(('--exclude', exclude))
|
||||
if exclude:
|
||||
list_args.extend(('--exclude', exclude))
|
||||
if greedy:
|
||||
list_args.append('--greedy')
|
||||
if indirect_selection:
|
||||
list_args.extend(('--indirect-selection', indirect_selection))
|
||||
if selector_name:
|
||||
list_args.extend(('--selector', selector_name))
|
||||
|
||||
@@ -40,7 +38,7 @@ class TestSelectionExpansion(DBTIntegrationTest):
|
||||
test_names = [name.split('.')[-1] for name in listed]
|
||||
assert sorted(test_names) == sorted(expected_tests)
|
||||
|
||||
def run_tests_and_assert(self, include, exclude, expected_tests, greedy=False, selector_name=None):
|
||||
def run_tests_and_assert(self, include, exclude, expected_tests, indirect_selection='eager', selector_name=None):
|
||||
results = self.run_dbt(['run'])
|
||||
self.assertEqual(len(results), 2)
|
||||
|
||||
@@ -49,8 +47,8 @@ class TestSelectionExpansion(DBTIntegrationTest):
|
||||
test_args.extend(('--models', include))
|
||||
if exclude:
|
||||
test_args.extend(('--exclude', exclude))
|
||||
if greedy:
|
||||
test_args.append('--greedy')
|
||||
if indirect_selection:
|
||||
test_args.extend(('--indirect-selection', indirect_selection))
|
||||
if selector_name:
|
||||
test_args.extend(('--selector', selector_name))
|
||||
|
||||
@@ -79,7 +77,12 @@ class TestSelectionExpansion(DBTIntegrationTest):
|
||||
def test__postgres__model_a_alone(self):
|
||||
select = 'model_a'
|
||||
exclude = None
|
||||
expected = ['just_a','unique_model_a_fun']
|
||||
expected = [
|
||||
'cf_a_b', 'cf_a_src', 'just_a',
|
||||
'relationships_model_a_fun__fun__ref_model_b_',
|
||||
'relationships_model_a_fun__fun__source_my_src_my_tbl_',
|
||||
'unique_model_a_fun'
|
||||
]
|
||||
|
||||
self.list_tests_and_assert(select, exclude, expected)
|
||||
self.run_tests_and_assert(select, exclude, expected)
|
||||
@@ -89,10 +92,12 @@ class TestSelectionExpansion(DBTIntegrationTest):
|
||||
select = 'model_a model_b'
|
||||
exclude = None
|
||||
expected = [
|
||||
'cf_a_b','just_a','unique_model_a_fun',
|
||||
'relationships_model_a_fun__fun__ref_model_b_'
|
||||
'cf_a_b','cf_a_src','just_a','unique_model_a_fun',
|
||||
'relationships_model_a_fun__fun__ref_model_b_',
|
||||
'relationships_model_a_fun__fun__source_my_src_my_tbl_'
|
||||
]
|
||||
|
||||
|
||||
self.list_tests_and_assert(select, exclude, expected)
|
||||
self.run_tests_and_assert(select, exclude, expected)
|
||||
|
||||
@@ -101,8 +106,9 @@ class TestSelectionExpansion(DBTIntegrationTest):
|
||||
select = 'model_a source:*'
|
||||
exclude = None
|
||||
expected = [
|
||||
'cf_a_src','just_a','unique_model_a_fun',
|
||||
'cf_a_b','cf_a_src','just_a','unique_model_a_fun',
|
||||
'source_unique_my_src_my_tbl_fun',
|
||||
'relationships_model_a_fun__fun__ref_model_b_',
|
||||
'relationships_model_a_fun__fun__source_my_src_my_tbl_'
|
||||
]
|
||||
|
||||
@@ -126,14 +132,18 @@ class TestSelectionExpansion(DBTIntegrationTest):
|
||||
def test__postgres__model_a_exclude_specific_test(self):
|
||||
select = 'model_a'
|
||||
exclude = 'unique_model_a_fun'
|
||||
expected = ['just_a']
|
||||
expected = [
|
||||
'cf_a_b','cf_a_src','just_a',
|
||||
'relationships_model_a_fun__fun__ref_model_b_',
|
||||
'relationships_model_a_fun__fun__source_my_src_my_tbl_'
|
||||
]
|
||||
|
||||
self.list_tests_and_assert(select, exclude, expected)
|
||||
self.run_tests_and_assert(select, exclude, expected)
|
||||
|
||||
@use_profile('postgres')
|
||||
def test__postgres__only_schema(self):
|
||||
select = 'test_type:schema'
|
||||
def test__postgres__only_generic(self):
|
||||
select = 'test_type:generic'
|
||||
exclude = None
|
||||
expected = [
|
||||
'relationships_model_a_fun__fun__ref_model_b_',
|
||||
@@ -146,17 +156,37 @@ class TestSelectionExpansion(DBTIntegrationTest):
|
||||
self.run_tests_and_assert(select, exclude, expected)
|
||||
|
||||
@use_profile('postgres')
|
||||
def test__postgres__model_a_only_data(self):
|
||||
select = 'model_a,test_type:schema'
|
||||
def test__postgres__model_a_only_singular_unset(self):
|
||||
select = 'model_a,test_type:singular'
|
||||
exclude = None
|
||||
expected = ['unique_model_a_fun']
|
||||
expected = ['cf_a_b','cf_a_src','just_a']
|
||||
|
||||
self.list_tests_and_assert(select, exclude, expected)
|
||||
self.run_tests_and_assert(select, exclude, expected)
|
||||
|
||||
@use_profile('postgres')
|
||||
def test__postgres__only_data(self):
|
||||
select = 'test_type:data'
|
||||
def test__postgres__model_a_only_singular_eager(self):
|
||||
select = 'model_a,test_type:singular'
|
||||
exclude = None
|
||||
expected = ['cf_a_b','cf_a_src','just_a']
|
||||
indirect_selection = 'eager'
|
||||
|
||||
self.list_tests_and_assert(select, exclude, expected)
|
||||
self.run_tests_and_assert(select, exclude, expected)
|
||||
|
||||
@use_profile('postgres')
|
||||
def test__postgres__model_a_only_singular_cautious(self):
|
||||
select = 'model_a,test_type:singular'
|
||||
exclude = None
|
||||
expected = ['just_a']
|
||||
indirect_selection = 'cautious'
|
||||
|
||||
self.list_tests_and_assert(select, exclude, expected, indirect_selection=indirect_selection)
|
||||
self.run_tests_and_assert(select, exclude, expected, indirect_selection=indirect_selection)
|
||||
|
||||
@use_profile('postgres')
|
||||
def test__postgres__only_singular(self):
|
||||
select = 'test_type:singular'
|
||||
exclude = None
|
||||
expected = ['cf_a_b', 'cf_a_src', 'just_a']
|
||||
|
||||
@@ -164,10 +194,10 @@ class TestSelectionExpansion(DBTIntegrationTest):
|
||||
self.run_tests_and_assert(select, exclude, expected)
|
||||
|
||||
@use_profile('postgres')
|
||||
def test__postgres__model_a_only_data(self):
|
||||
select = 'model_a,test_type:data'
|
||||
def test__postgres__model_a_only_singular(self):
|
||||
select = 'model_a,test_type:singular'
|
||||
exclude = None
|
||||
expected = ['just_a']
|
||||
expected = ['cf_a_b','cf_a_src','just_a']
|
||||
|
||||
self.list_tests_and_assert(select, exclude, expected)
|
||||
self.run_tests_and_assert(select, exclude, expected)
|
||||
@@ -185,7 +215,10 @@ class TestSelectionExpansion(DBTIntegrationTest):
|
||||
def test__postgres__model_tag_test_name_intersection(self):
|
||||
select = 'tag:a_or_b,test_name:relationships'
|
||||
exclude = None
|
||||
expected = ['relationships_model_a_fun__fun__ref_model_b_']
|
||||
expected = [
|
||||
'relationships_model_a_fun__fun__ref_model_b_',
|
||||
'relationships_model_a_fun__fun__source_my_src_my_tbl_'
|
||||
]
|
||||
|
||||
self.list_tests_and_assert(select, exclude, expected)
|
||||
self.run_tests_and_assert(select, exclude, expected)
|
||||
@@ -225,16 +258,20 @@ class TestSelectionExpansion(DBTIntegrationTest):
|
||||
def test__postgres__exclude_data_test_tag(self):
|
||||
select = 'model_a'
|
||||
exclude = 'tag:data_test_tag'
|
||||
expected = ['unique_model_a_fun']
|
||||
expected = [
|
||||
'cf_a_b', 'cf_a_src',
|
||||
'relationships_model_a_fun__fun__ref_model_b_',
|
||||
'relationships_model_a_fun__fun__source_my_src_my_tbl_',
|
||||
'unique_model_a_fun'
|
||||
]
|
||||
|
||||
self.list_tests_and_assert(select, exclude, expected)
|
||||
self.run_tests_and_assert(select, exclude, expected)
|
||||
|
||||
@use_profile('postgres')
|
||||
def test__postgres__model_a_greedy(self):
|
||||
def test__postgres__model_a_indirect_selection(self):
|
||||
select = 'model_a'
|
||||
exclude = None
|
||||
greedy = True
|
||||
expected = [
|
||||
'cf_a_b', 'cf_a_src', 'just_a',
|
||||
'relationships_model_a_fun__fun__ref_model_b_',
|
||||
@@ -242,22 +279,22 @@ class TestSelectionExpansion(DBTIntegrationTest):
|
||||
'unique_model_a_fun'
|
||||
]
|
||||
|
||||
self.list_tests_and_assert(select, exclude, expected, greedy)
|
||||
self.run_tests_and_assert(select, exclude, expected, greedy=greedy)
|
||||
self.list_tests_and_assert(select, exclude, expected)
|
||||
self.run_tests_and_assert(select, exclude, expected)
|
||||
|
||||
@use_profile('postgres')
|
||||
def test__postgres__model_a_greedy_exclude_unique_tests(self):
|
||||
def test__postgres__model_a_indirect_selection_exclude_unique_tests(self):
|
||||
select = 'model_a'
|
||||
exclude = 'test_name:unique'
|
||||
greedy = True
|
||||
indirect_selection = 'eager'
|
||||
expected = [
|
||||
'cf_a_b', 'cf_a_src', 'just_a',
|
||||
'relationships_model_a_fun__fun__ref_model_b_',
|
||||
'relationships_model_a_fun__fun__source_my_src_my_tbl_',
|
||||
]
|
||||
|
||||
self.list_tests_and_assert(select, exclude, expected, greedy)
|
||||
self.run_tests_and_assert(select, exclude, expected, greedy=greedy)
|
||||
self.list_tests_and_assert(select, exclude, expected, indirect_selection)
|
||||
self.run_tests_and_assert(select, exclude, expected, indirect_selection=indirect_selection)
|
||||
|
||||
class TestExpansionWithSelectors(TestSelectionExpansion):
|
||||
|
||||
@@ -265,37 +302,24 @@ class TestExpansionWithSelectors(TestSelectionExpansion):
|
||||
def selectors_config(self):
|
||||
return yaml.safe_load('''
|
||||
selectors:
|
||||
- name: model_a_greedy_none
|
||||
- name: model_a_unset_indirect_selection
|
||||
definition:
|
||||
method: fqn
|
||||
value: model_a
|
||||
- name: model_a_greedy_false
|
||||
- name: model_a_no_indirect_selection
|
||||
definition:
|
||||
method: fqn
|
||||
value: model_a
|
||||
greedy: false
|
||||
- name: model_a_greedy_true
|
||||
indirect_selection: "cautious"
|
||||
- name: model_a_yes_indirect_selection
|
||||
definition:
|
||||
method: fqn
|
||||
value: model_a
|
||||
greedy: true
|
||||
indirect_selection: "eager"
|
||||
''')
|
||||
|
||||
@use_profile('postgres')
|
||||
def test__postgres__selector_model_a_not_greedy(self):
|
||||
expected = ['just_a','unique_model_a_fun']
|
||||
|
||||
# when greedy is not specified, so implicitly False
|
||||
self.list_tests_and_assert(include=None, exclude=None, expected_tests=expected, selector_name='model_a_greedy_none')
|
||||
self.run_tests_and_assert(include=None, exclude=None, expected_tests=expected, selector_name='model_a_greedy_none')
|
||||
|
||||
# when greedy is explicitly False
|
||||
self.list_tests_and_assert(include=None, exclude=None, expected_tests=expected, selector_name='model_a_greedy_false')
|
||||
self.run_tests_and_assert(include=None, exclude=None, expected_tests=expected, selector_name='model_a_greedy_false')
|
||||
|
||||
|
||||
@use_profile('postgres')
|
||||
def test__postgres__selector_model_a_yes_greedy(self):
|
||||
def test__postgres__selector_model_a_unset_indirect_selection(self):
|
||||
expected = [
|
||||
'cf_a_b', 'cf_a_src', 'just_a',
|
||||
'relationships_model_a_fun__fun__ref_model_b_',
|
||||
@@ -303,6 +327,25 @@ class TestExpansionWithSelectors(TestSelectionExpansion):
|
||||
'unique_model_a_fun'
|
||||
]
|
||||
|
||||
# when greedy is explicitly False
|
||||
self.list_tests_and_assert(include=None, exclude=None, expected_tests=expected, selector_name='model_a_greedy_true')
|
||||
self.run_tests_and_assert(include=None, exclude=None, expected_tests=expected, selector_name='model_a_greedy_true')
|
||||
self.list_tests_and_assert(include=None, exclude=None, expected_tests=expected, selector_name='model_a_unset_indirect_selection')
|
||||
self.run_tests_and_assert(include=None, exclude=None, expected_tests=expected, selector_name='model_a_unset_indirect_selection')
|
||||
|
||||
@use_profile('postgres')
|
||||
def test__postgres__selector_model_a_no_indirect_selection(self):
|
||||
expected = ['just_a','unique_model_a_fun']
|
||||
|
||||
self.list_tests_and_assert(include=None, exclude=None, expected_tests=expected, selector_name='model_a_no_indirect_selection')
|
||||
self.run_tests_and_assert(include=None, exclude=None, expected_tests=expected, selector_name='model_a_no_indirect_selection')
|
||||
|
||||
|
||||
@use_profile('postgres')
|
||||
def test__postgres__selector_model_a_yes_indirect_selection(self):
|
||||
expected = [
|
||||
'cf_a_b', 'cf_a_src', 'just_a',
|
||||
'relationships_model_a_fun__fun__ref_model_b_',
|
||||
'relationships_model_a_fun__fun__source_my_src_my_tbl_',
|
||||
'unique_model_a_fun'
|
||||
]
|
||||
|
||||
self.list_tests_and_assert(include=None, exclude=None, expected_tests=expected, selector_name='model_a_yes_indirect_selection')
|
||||
self.run_tests_and_assert(include=None, exclude=None, expected_tests=expected, selector_name='model_a_yes_indirect_selection')
|
||||
|
||||
Reference in New Issue
Block a user