mirror of
https://github.com/dbt-labs/dbt-core
synced 2025-12-20 13:51:28 +00:00
Compare commits
9 Commits
enable-pos
...
feature/40
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
8f41453f7b | ||
|
|
fa42880d6e | ||
|
|
e3f699ce6a | ||
|
|
be117691ef | ||
|
|
f2b6e65f00 | ||
|
|
d347fb8b59 | ||
|
|
6afc0abd0d | ||
|
|
47df8259af | ||
|
|
a8f8ff09af |
@@ -17,7 +17,7 @@ PROFILES_DIR = os.path.expanduser(
|
|||||||
STRICT_MODE = False # Only here for backwards compatibility
|
STRICT_MODE = False # Only here for backwards compatibility
|
||||||
FULL_REFRESH = False # subcommand
|
FULL_REFRESH = False # subcommand
|
||||||
STORE_FAILURES = False # subcommand
|
STORE_FAILURES = False # subcommand
|
||||||
GREEDY = None # subcommand
|
INDIRECT_SELECTION = 'eager' # subcommand
|
||||||
|
|
||||||
# Global CLI commands
|
# Global CLI commands
|
||||||
USE_EXPERIMENTAL_PARSER = None
|
USE_EXPERIMENTAL_PARSER = None
|
||||||
@@ -95,7 +95,7 @@ MP_CONTEXT = _get_context()
|
|||||||
def set_from_args(args, user_config):
|
def set_from_args(args, user_config):
|
||||||
global STRICT_MODE, FULL_REFRESH, WARN_ERROR, \
|
global STRICT_MODE, FULL_REFRESH, WARN_ERROR, \
|
||||||
USE_EXPERIMENTAL_PARSER, STATIC_PARSER, WRITE_JSON, PARTIAL_PARSE, \
|
USE_EXPERIMENTAL_PARSER, STATIC_PARSER, WRITE_JSON, PARTIAL_PARSE, \
|
||||||
USE_COLORS, STORE_FAILURES, PROFILES_DIR, DEBUG, LOG_FORMAT, GREEDY, \
|
USE_COLORS, STORE_FAILURES, PROFILES_DIR, DEBUG, LOG_FORMAT, INDIRECT_SELECTION, \
|
||||||
VERSION_CHECK, FAIL_FAST, SEND_ANONYMOUS_USAGE_STATS, PRINTER_WIDTH, \
|
VERSION_CHECK, FAIL_FAST, SEND_ANONYMOUS_USAGE_STATS, PRINTER_WIDTH, \
|
||||||
WHICH
|
WHICH
|
||||||
|
|
||||||
@@ -103,7 +103,7 @@ def set_from_args(args, user_config):
|
|||||||
# cli args without user_config or env var option
|
# cli args without user_config or env var option
|
||||||
FULL_REFRESH = getattr(args, 'full_refresh', FULL_REFRESH)
|
FULL_REFRESH = getattr(args, 'full_refresh', FULL_REFRESH)
|
||||||
STORE_FAILURES = getattr(args, 'store_failures', STORE_FAILURES)
|
STORE_FAILURES = getattr(args, 'store_failures', STORE_FAILURES)
|
||||||
GREEDY = getattr(args, 'greedy', GREEDY)
|
INDIRECT_SELECTION = getattr(args, 'indirect_selection', INDIRECT_SELECTION)
|
||||||
WHICH = getattr(args, 'which', WHICH)
|
WHICH = getattr(args, 'which', WHICH)
|
||||||
|
|
||||||
# global cli flags with env var and user_config alternatives
|
# global cli flags with env var and user_config alternatives
|
||||||
|
|||||||
@@ -16,6 +16,7 @@ from .selector_spec import (
|
|||||||
SelectionIntersection,
|
SelectionIntersection,
|
||||||
SelectionDifference,
|
SelectionDifference,
|
||||||
SelectionCriteria,
|
SelectionCriteria,
|
||||||
|
IndirectSelection
|
||||||
)
|
)
|
||||||
|
|
||||||
INTERSECTION_DELIMITER = ','
|
INTERSECTION_DELIMITER = ','
|
||||||
@@ -25,7 +26,7 @@ DEFAULT_EXCLUDES: List[str] = []
|
|||||||
|
|
||||||
|
|
||||||
def parse_union(
|
def parse_union(
|
||||||
components: List[str], expect_exists: bool, greedy: bool = False
|
components: List[str], expect_exists: bool, indirect_selection: IndirectSelection = IndirectSelection.Eager
|
||||||
) -> SelectionUnion:
|
) -> SelectionUnion:
|
||||||
# turn ['a b', 'c'] -> ['a', 'b', 'c']
|
# turn ['a b', 'c'] -> ['a', 'b', 'c']
|
||||||
raw_specs = itertools.chain.from_iterable(
|
raw_specs = itertools.chain.from_iterable(
|
||||||
@@ -36,7 +37,7 @@ def parse_union(
|
|||||||
# ['a', 'b', 'c,d'] -> union('a', 'b', intersection('c', 'd'))
|
# ['a', 'b', 'c,d'] -> union('a', 'b', intersection('c', 'd'))
|
||||||
for raw_spec in raw_specs:
|
for raw_spec in raw_specs:
|
||||||
intersection_components: List[SelectionSpec] = [
|
intersection_components: List[SelectionSpec] = [
|
||||||
SelectionCriteria.from_single_spec(part, greedy=greedy)
|
SelectionCriteria.from_single_spec(part, indirect_selection=indirect_selection)
|
||||||
for part in raw_spec.split(INTERSECTION_DELIMITER)
|
for part in raw_spec.split(INTERSECTION_DELIMITER)
|
||||||
]
|
]
|
||||||
union_components.append(SelectionIntersection(
|
union_components.append(SelectionIntersection(
|
||||||
@@ -52,21 +53,25 @@ def parse_union(
|
|||||||
|
|
||||||
|
|
||||||
def parse_union_from_default(
|
def parse_union_from_default(
|
||||||
raw: Optional[List[str]], default: List[str], greedy: bool = False
|
raw: Optional[List[str]], default: List[str], indirect_selection: IndirectSelection = IndirectSelection.Eager
|
||||||
) -> SelectionUnion:
|
) -> SelectionUnion:
|
||||||
components: List[str]
|
components: List[str]
|
||||||
expect_exists: bool
|
expect_exists: bool
|
||||||
if raw is None:
|
if raw is None:
|
||||||
return parse_union(components=default, expect_exists=False, greedy=greedy)
|
return parse_union(components=default, expect_exists=False, indirect_selection=indirect_selection)
|
||||||
else:
|
else:
|
||||||
return parse_union(components=raw, expect_exists=True, greedy=greedy)
|
return parse_union(components=raw, expect_exists=True, indirect_selection=indirect_selection)
|
||||||
|
|
||||||
|
|
||||||
def parse_difference(
|
def parse_difference(
|
||||||
include: Optional[List[str]], exclude: Optional[List[str]]
|
include: Optional[List[str]], exclude: Optional[List[str]]
|
||||||
) -> SelectionDifference:
|
) -> SelectionDifference:
|
||||||
included = parse_union_from_default(include, DEFAULT_INCLUDES, greedy=bool(flags.GREEDY))
|
included = parse_union_from_default(
|
||||||
excluded = parse_union_from_default(exclude, DEFAULT_EXCLUDES, greedy=True)
|
include,
|
||||||
|
DEFAULT_INCLUDES,
|
||||||
|
indirect_selection=IndirectSelection(flags.INDIRECT_SELECTION)
|
||||||
|
)
|
||||||
|
excluded = parse_union_from_default(exclude, DEFAULT_EXCLUDES, indirect_selection=IndirectSelection.Eager)
|
||||||
return SelectionDifference(components=[included, excluded])
|
return SelectionDifference(components=[included, excluded])
|
||||||
|
|
||||||
|
|
||||||
@@ -148,7 +153,7 @@ def parse_union_definition(definition: Dict[str, Any]) -> SelectionSpec:
|
|||||||
union_def_parts = _get_list_dicts(definition, 'union')
|
union_def_parts = _get_list_dicts(definition, 'union')
|
||||||
include, exclude = _parse_include_exclude_subdefs(union_def_parts)
|
include, exclude = _parse_include_exclude_subdefs(union_def_parts)
|
||||||
|
|
||||||
union = SelectionUnion(components=include, greedy_warning=False)
|
union = SelectionUnion(components=include)
|
||||||
|
|
||||||
if exclude is None:
|
if exclude is None:
|
||||||
union.raw = definition
|
union.raw = definition
|
||||||
@@ -156,8 +161,7 @@ def parse_union_definition(definition: Dict[str, Any]) -> SelectionSpec:
|
|||||||
else:
|
else:
|
||||||
return SelectionDifference(
|
return SelectionDifference(
|
||||||
components=[union, exclude],
|
components=[union, exclude],
|
||||||
raw=definition,
|
raw=definition
|
||||||
greedy_warning=False
|
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@@ -166,7 +170,7 @@ def parse_intersection_definition(
|
|||||||
) -> SelectionSpec:
|
) -> SelectionSpec:
|
||||||
intersection_def_parts = _get_list_dicts(definition, 'intersection')
|
intersection_def_parts = _get_list_dicts(definition, 'intersection')
|
||||||
include, exclude = _parse_include_exclude_subdefs(intersection_def_parts)
|
include, exclude = _parse_include_exclude_subdefs(intersection_def_parts)
|
||||||
intersection = SelectionIntersection(components=include, greedy_warning=False)
|
intersection = SelectionIntersection(components=include)
|
||||||
|
|
||||||
if exclude is None:
|
if exclude is None:
|
||||||
intersection.raw = definition
|
intersection.raw = definition
|
||||||
@@ -174,8 +178,7 @@ def parse_intersection_definition(
|
|||||||
else:
|
else:
|
||||||
return SelectionDifference(
|
return SelectionDifference(
|
||||||
components=[intersection, exclude],
|
components=[intersection, exclude],
|
||||||
raw=definition,
|
raw=definition
|
||||||
greedy_warning=False
|
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@@ -209,7 +212,7 @@ def parse_dict_definition(definition: Dict[str, Any]) -> SelectionSpec:
|
|||||||
if diff_arg is None:
|
if diff_arg is None:
|
||||||
return base
|
return base
|
||||||
else:
|
else:
|
||||||
return SelectionDifference(components=[base, diff_arg], greedy_warning=False)
|
return SelectionDifference(components=[base, diff_arg])
|
||||||
|
|
||||||
|
|
||||||
def parse_from_definition(
|
def parse_from_definition(
|
||||||
|
|||||||
@@ -3,7 +3,7 @@ from typing import Set, List, Optional, Tuple
|
|||||||
from .graph import Graph, UniqueId
|
from .graph import Graph, UniqueId
|
||||||
from .queue import GraphQueue
|
from .queue import GraphQueue
|
||||||
from .selector_methods import MethodManager
|
from .selector_methods import MethodManager
|
||||||
from .selector_spec import SelectionCriteria, SelectionSpec
|
from .selector_spec import SelectionCriteria, SelectionSpec, IndirectSelection
|
||||||
|
|
||||||
from dbt.logger import GLOBAL_LOGGER as logger
|
from dbt.logger import GLOBAL_LOGGER as logger
|
||||||
from dbt.node_types import NodeType
|
from dbt.node_types import NodeType
|
||||||
@@ -29,24 +29,6 @@ def alert_non_existence(raw_spec, nodes):
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def alert_unused_nodes(raw_spec, node_names):
|
|
||||||
summary_nodes_str = ("\n - ").join(node_names[:3])
|
|
||||||
debug_nodes_str = ("\n - ").join(node_names)
|
|
||||||
and_more_str = f"\n - and {len(node_names) - 3} more" if len(node_names) > 4 else ""
|
|
||||||
summary_msg = (
|
|
||||||
f"\nSome tests were excluded because at least one parent is not selected. "
|
|
||||||
f"Use the --greedy flag to include them."
|
|
||||||
f"\n - {summary_nodes_str}{and_more_str}"
|
|
||||||
)
|
|
||||||
logger.info(summary_msg)
|
|
||||||
if len(node_names) > 4:
|
|
||||||
debug_msg = (
|
|
||||||
f"Full list of tests that were excluded:"
|
|
||||||
f"\n - {debug_nodes_str}"
|
|
||||||
)
|
|
||||||
logger.debug(debug_msg)
|
|
||||||
|
|
||||||
|
|
||||||
def can_select_indirectly(node):
|
def can_select_indirectly(node):
|
||||||
"""If a node is not selected itself, but its parent(s) are, it may qualify
|
"""If a node is not selected itself, but its parent(s) are, it may qualify
|
||||||
for indirect selection.
|
for indirect selection.
|
||||||
@@ -113,7 +95,7 @@ class NodeSelector(MethodManager):
|
|||||||
neighbors = self.collect_specified_neighbors(spec, collected)
|
neighbors = self.collect_specified_neighbors(spec, collected)
|
||||||
direct_nodes, indirect_nodes = self.expand_selection(
|
direct_nodes, indirect_nodes = self.expand_selection(
|
||||||
selected=(collected | neighbors),
|
selected=(collected | neighbors),
|
||||||
greedy=spec.greedy
|
indirect_selection=spec.indirect_selection
|
||||||
)
|
)
|
||||||
return direct_nodes, indirect_nodes
|
return direct_nodes, indirect_nodes
|
||||||
|
|
||||||
@@ -217,21 +199,20 @@ class NodeSelector(MethodManager):
|
|||||||
}
|
}
|
||||||
|
|
||||||
def expand_selection(
|
def expand_selection(
|
||||||
self, selected: Set[UniqueId], greedy: bool = False
|
self, selected: Set[UniqueId], indirect_selection: IndirectSelection = IndirectSelection.Eager
|
||||||
) -> Tuple[Set[UniqueId], Set[UniqueId]]:
|
) -> Tuple[Set[UniqueId], Set[UniqueId]]:
|
||||||
# Test selection can expand to include an implicitly/indirectly selected test.
|
# Test selection by default expands to include an implicitly/indirectly selected tests.
|
||||||
# In this way, `dbt test -m model_a` also includes tests that directly depend on `model_a`.
|
# `dbt test -m model_a` also includes tests that directly depend on `model_a`.
|
||||||
# Expansion has two modes, GREEDY and NOT GREEDY.
|
# Expansion has two modes, EAGER and CAUTIOUS.
|
||||||
#
|
#
|
||||||
# GREEDY mode: If ANY parent is selected, select the test. We use this for EXCLUSION.
|
# EAGER mode: If ANY parent is selected, select the test.
|
||||||
#
|
#
|
||||||
# NOT GREEDY mode:
|
# CAUTIOUS mode:
|
||||||
# - If ALL parents are selected, select the test.
|
# - If ALL parents are selected, select the test.
|
||||||
# - If ANY parent is missing, return it separately. We'll keep it around
|
# - If ANY parent is missing, return it separately. We'll keep it around
|
||||||
# for later and see if its other parents show up.
|
# for later and see if its other parents show up.
|
||||||
# We use this for INCLUSION.
|
# Users can opt out of inclusive EAGER mode by passing --indirect-selection cautious
|
||||||
# Users can also opt in to inclusive GREEDY mode by passing --greedy flag,
|
# CLI argument or by specifying `indirect_selection: true` in a yaml selector
|
||||||
# or by specifying `greedy: true` in a yaml selector
|
|
||||||
|
|
||||||
direct_nodes = set(selected)
|
direct_nodes = set(selected)
|
||||||
indirect_nodes = set()
|
indirect_nodes = set()
|
||||||
@@ -241,7 +222,10 @@ class NodeSelector(MethodManager):
|
|||||||
node = self.manifest.nodes[unique_id]
|
node = self.manifest.nodes[unique_id]
|
||||||
if can_select_indirectly(node):
|
if can_select_indirectly(node):
|
||||||
# should we add it in directly?
|
# should we add it in directly?
|
||||||
if greedy or set(node.depends_on.nodes) <= set(selected):
|
if (
|
||||||
|
indirect_selection == IndirectSelection.Eager
|
||||||
|
or set(node.depends_on.nodes) <= set(selected)
|
||||||
|
):
|
||||||
direct_nodes.add(unique_id)
|
direct_nodes.add(unique_id)
|
||||||
# if not:
|
# if not:
|
||||||
else:
|
else:
|
||||||
@@ -255,6 +239,10 @@ class NodeSelector(MethodManager):
|
|||||||
# Check tests previously selected indirectly to see if ALL their
|
# Check tests previously selected indirectly to see if ALL their
|
||||||
# parents are now present.
|
# parents are now present.
|
||||||
|
|
||||||
|
# performance: if identical, skip the processing below
|
||||||
|
if set(direct_nodes) == set(indirect_nodes):
|
||||||
|
return direct_nodes
|
||||||
|
|
||||||
selected = set(direct_nodes)
|
selected = set(direct_nodes)
|
||||||
|
|
||||||
for unique_id in indirect_nodes:
|
for unique_id in indirect_nodes:
|
||||||
@@ -278,16 +266,6 @@ class NodeSelector(MethodManager):
|
|||||||
selected_nodes, indirect_only = self.select_nodes(spec)
|
selected_nodes, indirect_only = self.select_nodes(spec)
|
||||||
filtered_nodes = self.filter_selection(selected_nodes)
|
filtered_nodes = self.filter_selection(selected_nodes)
|
||||||
|
|
||||||
if indirect_only:
|
|
||||||
filtered_unused_nodes = self.filter_selection(indirect_only)
|
|
||||||
if filtered_unused_nodes and spec.greedy_warning:
|
|
||||||
# log anything that didn't make the cut
|
|
||||||
unused_node_names = []
|
|
||||||
for unique_id in filtered_unused_nodes:
|
|
||||||
name = self.manifest.nodes[unique_id].name
|
|
||||||
unused_node_names.append(name)
|
|
||||||
alert_unused_nodes(spec, unused_node_names)
|
|
||||||
|
|
||||||
return filtered_nodes
|
return filtered_nodes
|
||||||
|
|
||||||
def get_graph_queue(self, spec: SelectionSpec) -> GraphQueue:
|
def get_graph_queue(self, spec: SelectionSpec) -> GraphQueue:
|
||||||
|
|||||||
@@ -1,7 +1,9 @@
|
|||||||
import os
|
import os
|
||||||
import re
|
import re
|
||||||
|
import enum
|
||||||
from abc import ABCMeta, abstractmethod
|
from abc import ABCMeta, abstractmethod
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
|
from dbt.dataclass_schema import StrEnum
|
||||||
|
|
||||||
from typing import (
|
from typing import (
|
||||||
Set, Iterator, List, Optional, Dict, Union, Any, Iterable, Tuple
|
Set, Iterator, List, Optional, Dict, Union, Any, Iterable, Tuple
|
||||||
@@ -21,6 +23,10 @@ RAW_SELECTOR_PATTERN = re.compile(
|
|||||||
)
|
)
|
||||||
SELECTOR_METHOD_SEPARATOR = '.'
|
SELECTOR_METHOD_SEPARATOR = '.'
|
||||||
|
|
||||||
|
class IndirectSelection(StrEnum):
|
||||||
|
Eager = 'eager'
|
||||||
|
Cautious = 'cautious'
|
||||||
|
|
||||||
|
|
||||||
def _probably_path(value: str):
|
def _probably_path(value: str):
|
||||||
"""Decide if value is probably a path. Windows has two path separators, so
|
"""Decide if value is probably a path. Windows has two path separators, so
|
||||||
@@ -66,8 +72,7 @@ class SelectionCriteria:
|
|||||||
parents_depth: Optional[int]
|
parents_depth: Optional[int]
|
||||||
children: bool
|
children: bool
|
||||||
children_depth: Optional[int]
|
children_depth: Optional[int]
|
||||||
greedy: bool = False
|
indirect_selection: IndirectSelection = IndirectSelection.Eager
|
||||||
greedy_warning: bool = False # do not raise warning for yaml selectors
|
|
||||||
|
|
||||||
def __post_init__(self):
|
def __post_init__(self):
|
||||||
if self.children and self.childrens_parents:
|
if self.children and self.childrens_parents:
|
||||||
@@ -105,7 +110,7 @@ class SelectionCriteria:
|
|||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def selection_criteria_from_dict(
|
def selection_criteria_from_dict(
|
||||||
cls, raw: Any, dct: Dict[str, Any], greedy: bool = False
|
cls, raw: Any, dct: Dict[str, Any], indirect_selection: IndirectSelection = IndirectSelection.Eager
|
||||||
) -> 'SelectionCriteria':
|
) -> 'SelectionCriteria':
|
||||||
if 'value' not in dct:
|
if 'value' not in dct:
|
||||||
raise RuntimeException(
|
raise RuntimeException(
|
||||||
@@ -125,7 +130,7 @@ class SelectionCriteria:
|
|||||||
parents_depth=parents_depth,
|
parents_depth=parents_depth,
|
||||||
children=bool(dct.get('children')),
|
children=bool(dct.get('children')),
|
||||||
children_depth=children_depth,
|
children_depth=children_depth,
|
||||||
greedy=(greedy or bool(dct.get('greedy'))),
|
indirect_selection=(IndirectSelection(dct.get('indirect_selection') or indirect_selection)),
|
||||||
)
|
)
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
@@ -146,18 +151,22 @@ class SelectionCriteria:
|
|||||||
dct['parents'] = bool(dct.get('parents'))
|
dct['parents'] = bool(dct.get('parents'))
|
||||||
if 'children' in dct:
|
if 'children' in dct:
|
||||||
dct['children'] = bool(dct.get('children'))
|
dct['children'] = bool(dct.get('children'))
|
||||||
if 'greedy' in dct:
|
if 'indirect_selection' in dct:
|
||||||
dct['greedy'] = bool(dct.get('greedy'))
|
dct['indirect_selection'] = bool(dct.get('indirect_selection'))
|
||||||
return dct
|
return dct
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def from_single_spec(cls, raw: str, greedy: bool = False) -> 'SelectionCriteria':
|
def from_single_spec(cls, raw: str, indirect_selection: bool = True) -> 'SelectionCriteria':
|
||||||
result = RAW_SELECTOR_PATTERN.match(raw)
|
result = RAW_SELECTOR_PATTERN.match(raw)
|
||||||
if result is None:
|
if result is None:
|
||||||
# bad spec!
|
# bad spec!
|
||||||
raise RuntimeException(f'Invalid selector spec "{raw}"')
|
raise RuntimeException(f'Invalid selector spec "{raw}"')
|
||||||
|
|
||||||
return cls.selection_criteria_from_dict(raw, result.groupdict(), greedy=greedy)
|
return cls.selection_criteria_from_dict(
|
||||||
|
raw,
|
||||||
|
result.groupdict(),
|
||||||
|
indirect_selection=indirect_selection
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class BaseSelectionGroup(Iterable[SelectionSpec], metaclass=ABCMeta):
|
class BaseSelectionGroup(Iterable[SelectionSpec], metaclass=ABCMeta):
|
||||||
@@ -165,12 +174,10 @@ class BaseSelectionGroup(Iterable[SelectionSpec], metaclass=ABCMeta):
|
|||||||
self,
|
self,
|
||||||
components: Iterable[SelectionSpec],
|
components: Iterable[SelectionSpec],
|
||||||
expect_exists: bool = False,
|
expect_exists: bool = False,
|
||||||
greedy_warning: bool = True,
|
|
||||||
raw: Any = None,
|
raw: Any = None,
|
||||||
):
|
):
|
||||||
self.components: List[SelectionSpec] = list(components)
|
self.components: List[SelectionSpec] = list(components)
|
||||||
self.expect_exists = expect_exists
|
self.expect_exists = expect_exists
|
||||||
self.greedy_warning = greedy_warning
|
|
||||||
self.raw = raw
|
self.raw = raw
|
||||||
|
|
||||||
def __iter__(self) -> Iterator[SelectionSpec]:
|
def __iter__(self) -> Iterator[SelectionSpec]:
|
||||||
|
|||||||
@@ -392,13 +392,15 @@ def _build_build_subparser(subparsers, base_subparser):
|
|||||||
'''
|
'''
|
||||||
)
|
)
|
||||||
sub.add_argument(
|
sub.add_argument(
|
||||||
'--greedy',
|
'--indirect-selection',
|
||||||
action='store_true',
|
choices=['eager', 'cautious'],
|
||||||
|
dest='indirect_selection',
|
||||||
help='''
|
help='''
|
||||||
Select all tests that touch the selected resources,
|
Select all tests that are adjacent to selected resources,
|
||||||
even if they also depend on unselected resources
|
even if they those resources have been explicitly selected.
|
||||||
'''
|
''',
|
||||||
)
|
)
|
||||||
|
|
||||||
resource_values: List[str] = [
|
resource_values: List[str] = [
|
||||||
str(s) for s in build_task.BuildTask.ALL_RESOURCE_VALUES
|
str(s) for s in build_task.BuildTask.ALL_RESOURCE_VALUES
|
||||||
] + ['all']
|
] + ['all']
|
||||||
@@ -736,12 +738,14 @@ def _build_test_subparser(subparsers, base_subparser):
|
|||||||
'''
|
'''
|
||||||
)
|
)
|
||||||
sub.add_argument(
|
sub.add_argument(
|
||||||
'--greedy',
|
'--indirect-selection',
|
||||||
action='store_true',
|
choices=['eager', 'cautious'],
|
||||||
|
default='eager',
|
||||||
|
dest='indirect_selection',
|
||||||
help='''
|
help='''
|
||||||
Select all tests that touch the selected resources,
|
Select all tests that are adjacent to selected resources,
|
||||||
even if they also depend on unselected resources
|
even if they those resources have been explicitly selected.
|
||||||
'''
|
''',
|
||||||
)
|
)
|
||||||
|
|
||||||
sub.set_defaults(cls=test_task.TestTask, which='test', rpc_method='test')
|
sub.set_defaults(cls=test_task.TestTask, which='test', rpc_method='test')
|
||||||
@@ -839,12 +843,13 @@ def _build_list_subparser(subparsers, base_subparser):
|
|||||||
required=False,
|
required=False,
|
||||||
)
|
)
|
||||||
sub.add_argument(
|
sub.add_argument(
|
||||||
'--greedy',
|
'--indirect-selection',
|
||||||
action='store_true',
|
choices=['eager', 'cautious'],
|
||||||
|
dest='indirect_selection',
|
||||||
help='''
|
help='''
|
||||||
Select all tests that touch the selected resources,
|
Select all tests that are adjacent to selected resources,
|
||||||
even if they also depend on unselected resources
|
even if they those resources have been explicitly selected.
|
||||||
'''
|
''',
|
||||||
)
|
)
|
||||||
_add_common_selector_arguments(sub)
|
_add_common_selector_arguments(sub)
|
||||||
|
|
||||||
|
|||||||
@@ -21,16 +21,14 @@ class TestSelectionExpansion(DBTIntegrationTest):
|
|||||||
"test-paths": ["tests"]
|
"test-paths": ["tests"]
|
||||||
}
|
}
|
||||||
|
|
||||||
def list_tests_and_assert(self, include, exclude, expected_tests, greedy=False, selector_name=None):
|
def list_tests_and_assert(self, include, exclude, expected_tests, indirect_selection='eager', selector_name=None):
|
||||||
list_args = [ 'ls', '--resource-type', 'test']
|
list_args = [ 'ls', '--resource-type', 'test']
|
||||||
if include:
|
if include:
|
||||||
list_args.extend(('--select', include))
|
list_args.extend(('--select', include))
|
||||||
if exclude:
|
if exclude:
|
||||||
list_args.extend(('--exclude', exclude))
|
list_args.extend(('--exclude', exclude))
|
||||||
if exclude:
|
if indirect_selection:
|
||||||
list_args.extend(('--exclude', exclude))
|
list_args.extend(('--indirect-selection', indirect_selection))
|
||||||
if greedy:
|
|
||||||
list_args.append('--greedy')
|
|
||||||
if selector_name:
|
if selector_name:
|
||||||
list_args.extend(('--selector', selector_name))
|
list_args.extend(('--selector', selector_name))
|
||||||
|
|
||||||
@@ -40,7 +38,7 @@ class TestSelectionExpansion(DBTIntegrationTest):
|
|||||||
test_names = [name.split('.')[-1] for name in listed]
|
test_names = [name.split('.')[-1] for name in listed]
|
||||||
assert sorted(test_names) == sorted(expected_tests)
|
assert sorted(test_names) == sorted(expected_tests)
|
||||||
|
|
||||||
def run_tests_and_assert(self, include, exclude, expected_tests, greedy=False, selector_name=None):
|
def run_tests_and_assert(self, include, exclude, expected_tests, indirect_selection='eager', selector_name=None):
|
||||||
results = self.run_dbt(['run'])
|
results = self.run_dbt(['run'])
|
||||||
self.assertEqual(len(results), 2)
|
self.assertEqual(len(results), 2)
|
||||||
|
|
||||||
@@ -49,8 +47,8 @@ class TestSelectionExpansion(DBTIntegrationTest):
|
|||||||
test_args.extend(('--models', include))
|
test_args.extend(('--models', include))
|
||||||
if exclude:
|
if exclude:
|
||||||
test_args.extend(('--exclude', exclude))
|
test_args.extend(('--exclude', exclude))
|
||||||
if greedy:
|
if indirect_selection:
|
||||||
test_args.append('--greedy')
|
test_args.extend(('--indirect-selection', indirect_selection))
|
||||||
if selector_name:
|
if selector_name:
|
||||||
test_args.extend(('--selector', selector_name))
|
test_args.extend(('--selector', selector_name))
|
||||||
|
|
||||||
@@ -79,7 +77,12 @@ class TestSelectionExpansion(DBTIntegrationTest):
|
|||||||
def test__postgres__model_a_alone(self):
|
def test__postgres__model_a_alone(self):
|
||||||
select = 'model_a'
|
select = 'model_a'
|
||||||
exclude = None
|
exclude = None
|
||||||
expected = ['just_a','unique_model_a_fun']
|
expected = [
|
||||||
|
'cf_a_b', 'cf_a_src', 'just_a',
|
||||||
|
'relationships_model_a_fun__fun__ref_model_b_',
|
||||||
|
'relationships_model_a_fun__fun__source_my_src_my_tbl_',
|
||||||
|
'unique_model_a_fun'
|
||||||
|
]
|
||||||
|
|
||||||
self.list_tests_and_assert(select, exclude, expected)
|
self.list_tests_and_assert(select, exclude, expected)
|
||||||
self.run_tests_and_assert(select, exclude, expected)
|
self.run_tests_and_assert(select, exclude, expected)
|
||||||
@@ -89,10 +92,12 @@ class TestSelectionExpansion(DBTIntegrationTest):
|
|||||||
select = 'model_a model_b'
|
select = 'model_a model_b'
|
||||||
exclude = None
|
exclude = None
|
||||||
expected = [
|
expected = [
|
||||||
'cf_a_b','just_a','unique_model_a_fun',
|
'cf_a_b','cf_a_src','just_a','unique_model_a_fun',
|
||||||
'relationships_model_a_fun__fun__ref_model_b_'
|
'relationships_model_a_fun__fun__ref_model_b_',
|
||||||
|
'relationships_model_a_fun__fun__source_my_src_my_tbl_'
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
self.list_tests_and_assert(select, exclude, expected)
|
self.list_tests_and_assert(select, exclude, expected)
|
||||||
self.run_tests_and_assert(select, exclude, expected)
|
self.run_tests_and_assert(select, exclude, expected)
|
||||||
|
|
||||||
@@ -101,8 +106,9 @@ class TestSelectionExpansion(DBTIntegrationTest):
|
|||||||
select = 'model_a source:*'
|
select = 'model_a source:*'
|
||||||
exclude = None
|
exclude = None
|
||||||
expected = [
|
expected = [
|
||||||
'cf_a_src','just_a','unique_model_a_fun',
|
'cf_a_b','cf_a_src','just_a','unique_model_a_fun',
|
||||||
'source_unique_my_src_my_tbl_fun',
|
'source_unique_my_src_my_tbl_fun',
|
||||||
|
'relationships_model_a_fun__fun__ref_model_b_',
|
||||||
'relationships_model_a_fun__fun__source_my_src_my_tbl_'
|
'relationships_model_a_fun__fun__source_my_src_my_tbl_'
|
||||||
]
|
]
|
||||||
|
|
||||||
@@ -126,14 +132,18 @@ class TestSelectionExpansion(DBTIntegrationTest):
|
|||||||
def test__postgres__model_a_exclude_specific_test(self):
|
def test__postgres__model_a_exclude_specific_test(self):
|
||||||
select = 'model_a'
|
select = 'model_a'
|
||||||
exclude = 'unique_model_a_fun'
|
exclude = 'unique_model_a_fun'
|
||||||
expected = ['just_a']
|
expected = [
|
||||||
|
'cf_a_b','cf_a_src','just_a',
|
||||||
|
'relationships_model_a_fun__fun__ref_model_b_',
|
||||||
|
'relationships_model_a_fun__fun__source_my_src_my_tbl_'
|
||||||
|
]
|
||||||
|
|
||||||
self.list_tests_and_assert(select, exclude, expected)
|
self.list_tests_and_assert(select, exclude, expected)
|
||||||
self.run_tests_and_assert(select, exclude, expected)
|
self.run_tests_and_assert(select, exclude, expected)
|
||||||
|
|
||||||
@use_profile('postgres')
|
@use_profile('postgres')
|
||||||
def test__postgres__only_schema(self):
|
def test__postgres__only_generic(self):
|
||||||
select = 'test_type:schema'
|
select = 'test_type:generic'
|
||||||
exclude = None
|
exclude = None
|
||||||
expected = [
|
expected = [
|
||||||
'relationships_model_a_fun__fun__ref_model_b_',
|
'relationships_model_a_fun__fun__ref_model_b_',
|
||||||
@@ -146,17 +156,37 @@ class TestSelectionExpansion(DBTIntegrationTest):
|
|||||||
self.run_tests_and_assert(select, exclude, expected)
|
self.run_tests_and_assert(select, exclude, expected)
|
||||||
|
|
||||||
@use_profile('postgres')
|
@use_profile('postgres')
|
||||||
def test__postgres__model_a_only_data(self):
|
def test__postgres__model_a_only_singular_unset(self):
|
||||||
select = 'model_a,test_type:schema'
|
select = 'model_a,test_type:singular'
|
||||||
exclude = None
|
exclude = None
|
||||||
expected = ['unique_model_a_fun']
|
expected = ['cf_a_b','cf_a_src','just_a']
|
||||||
|
|
||||||
self.list_tests_and_assert(select, exclude, expected)
|
self.list_tests_and_assert(select, exclude, expected)
|
||||||
self.run_tests_and_assert(select, exclude, expected)
|
self.run_tests_and_assert(select, exclude, expected)
|
||||||
|
|
||||||
@use_profile('postgres')
|
@use_profile('postgres')
|
||||||
def test__postgres__only_data(self):
|
def test__postgres__model_a_only_singular_eager(self):
|
||||||
select = 'test_type:data'
|
select = 'model_a,test_type:singular'
|
||||||
|
exclude = None
|
||||||
|
expected = ['cf_a_b','cf_a_src','just_a']
|
||||||
|
indirect_selection = 'eager'
|
||||||
|
|
||||||
|
self.list_tests_and_assert(select, exclude, expected)
|
||||||
|
self.run_tests_and_assert(select, exclude, expected)
|
||||||
|
|
||||||
|
@use_profile('postgres')
|
||||||
|
def test__postgres__model_a_only_singular_cautious(self):
|
||||||
|
select = 'model_a,test_type:singular'
|
||||||
|
exclude = None
|
||||||
|
expected = ['just_a']
|
||||||
|
indirect_selection = 'cautious'
|
||||||
|
|
||||||
|
self.list_tests_and_assert(select, exclude, expected, indirect_selection=indirect_selection)
|
||||||
|
self.run_tests_and_assert(select, exclude, expected, indirect_selection=indirect_selection)
|
||||||
|
|
||||||
|
@use_profile('postgres')
|
||||||
|
def test__postgres__only_singular(self):
|
||||||
|
select = 'test_type:singular'
|
||||||
exclude = None
|
exclude = None
|
||||||
expected = ['cf_a_b', 'cf_a_src', 'just_a']
|
expected = ['cf_a_b', 'cf_a_src', 'just_a']
|
||||||
|
|
||||||
@@ -164,10 +194,10 @@ class TestSelectionExpansion(DBTIntegrationTest):
|
|||||||
self.run_tests_and_assert(select, exclude, expected)
|
self.run_tests_and_assert(select, exclude, expected)
|
||||||
|
|
||||||
@use_profile('postgres')
|
@use_profile('postgres')
|
||||||
def test__postgres__model_a_only_data(self):
|
def test__postgres__model_a_only_singular(self):
|
||||||
select = 'model_a,test_type:data'
|
select = 'model_a,test_type:singular'
|
||||||
exclude = None
|
exclude = None
|
||||||
expected = ['just_a']
|
expected = ['cf_a_b','cf_a_src','just_a']
|
||||||
|
|
||||||
self.list_tests_and_assert(select, exclude, expected)
|
self.list_tests_and_assert(select, exclude, expected)
|
||||||
self.run_tests_and_assert(select, exclude, expected)
|
self.run_tests_and_assert(select, exclude, expected)
|
||||||
@@ -185,7 +215,10 @@ class TestSelectionExpansion(DBTIntegrationTest):
|
|||||||
def test__postgres__model_tag_test_name_intersection(self):
|
def test__postgres__model_tag_test_name_intersection(self):
|
||||||
select = 'tag:a_or_b,test_name:relationships'
|
select = 'tag:a_or_b,test_name:relationships'
|
||||||
exclude = None
|
exclude = None
|
||||||
expected = ['relationships_model_a_fun__fun__ref_model_b_']
|
expected = [
|
||||||
|
'relationships_model_a_fun__fun__ref_model_b_',
|
||||||
|
'relationships_model_a_fun__fun__source_my_src_my_tbl_'
|
||||||
|
]
|
||||||
|
|
||||||
self.list_tests_and_assert(select, exclude, expected)
|
self.list_tests_and_assert(select, exclude, expected)
|
||||||
self.run_tests_and_assert(select, exclude, expected)
|
self.run_tests_and_assert(select, exclude, expected)
|
||||||
@@ -225,16 +258,20 @@ class TestSelectionExpansion(DBTIntegrationTest):
|
|||||||
def test__postgres__exclude_data_test_tag(self):
|
def test__postgres__exclude_data_test_tag(self):
|
||||||
select = 'model_a'
|
select = 'model_a'
|
||||||
exclude = 'tag:data_test_tag'
|
exclude = 'tag:data_test_tag'
|
||||||
expected = ['unique_model_a_fun']
|
expected = [
|
||||||
|
'cf_a_b', 'cf_a_src',
|
||||||
|
'relationships_model_a_fun__fun__ref_model_b_',
|
||||||
|
'relationships_model_a_fun__fun__source_my_src_my_tbl_',
|
||||||
|
'unique_model_a_fun'
|
||||||
|
]
|
||||||
|
|
||||||
self.list_tests_and_assert(select, exclude, expected)
|
self.list_tests_and_assert(select, exclude, expected)
|
||||||
self.run_tests_and_assert(select, exclude, expected)
|
self.run_tests_and_assert(select, exclude, expected)
|
||||||
|
|
||||||
@use_profile('postgres')
|
@use_profile('postgres')
|
||||||
def test__postgres__model_a_greedy(self):
|
def test__postgres__model_a_indirect_selection(self):
|
||||||
select = 'model_a'
|
select = 'model_a'
|
||||||
exclude = None
|
exclude = None
|
||||||
greedy = True
|
|
||||||
expected = [
|
expected = [
|
||||||
'cf_a_b', 'cf_a_src', 'just_a',
|
'cf_a_b', 'cf_a_src', 'just_a',
|
||||||
'relationships_model_a_fun__fun__ref_model_b_',
|
'relationships_model_a_fun__fun__ref_model_b_',
|
||||||
@@ -242,22 +279,22 @@ class TestSelectionExpansion(DBTIntegrationTest):
|
|||||||
'unique_model_a_fun'
|
'unique_model_a_fun'
|
||||||
]
|
]
|
||||||
|
|
||||||
self.list_tests_and_assert(select, exclude, expected, greedy)
|
self.list_tests_and_assert(select, exclude, expected)
|
||||||
self.run_tests_and_assert(select, exclude, expected, greedy=greedy)
|
self.run_tests_and_assert(select, exclude, expected)
|
||||||
|
|
||||||
@use_profile('postgres')
|
@use_profile('postgres')
|
||||||
def test__postgres__model_a_greedy_exclude_unique_tests(self):
|
def test__postgres__model_a_indirect_selection_exclude_unique_tests(self):
|
||||||
select = 'model_a'
|
select = 'model_a'
|
||||||
exclude = 'test_name:unique'
|
exclude = 'test_name:unique'
|
||||||
greedy = True
|
indirect_selection = 'eager'
|
||||||
expected = [
|
expected = [
|
||||||
'cf_a_b', 'cf_a_src', 'just_a',
|
'cf_a_b', 'cf_a_src', 'just_a',
|
||||||
'relationships_model_a_fun__fun__ref_model_b_',
|
'relationships_model_a_fun__fun__ref_model_b_',
|
||||||
'relationships_model_a_fun__fun__source_my_src_my_tbl_',
|
'relationships_model_a_fun__fun__source_my_src_my_tbl_',
|
||||||
]
|
]
|
||||||
|
|
||||||
self.list_tests_and_assert(select, exclude, expected, greedy)
|
self.list_tests_and_assert(select, exclude, expected, indirect_selection)
|
||||||
self.run_tests_and_assert(select, exclude, expected, greedy=greedy)
|
self.run_tests_and_assert(select, exclude, expected, indirect_selection=indirect_selection)
|
||||||
|
|
||||||
class TestExpansionWithSelectors(TestSelectionExpansion):
|
class TestExpansionWithSelectors(TestSelectionExpansion):
|
||||||
|
|
||||||
@@ -265,37 +302,24 @@ class TestExpansionWithSelectors(TestSelectionExpansion):
|
|||||||
def selectors_config(self):
|
def selectors_config(self):
|
||||||
return yaml.safe_load('''
|
return yaml.safe_load('''
|
||||||
selectors:
|
selectors:
|
||||||
- name: model_a_greedy_none
|
- name: model_a_unset_indirect_selection
|
||||||
definition:
|
definition:
|
||||||
method: fqn
|
method: fqn
|
||||||
value: model_a
|
value: model_a
|
||||||
- name: model_a_greedy_false
|
- name: model_a_no_indirect_selection
|
||||||
definition:
|
definition:
|
||||||
method: fqn
|
method: fqn
|
||||||
value: model_a
|
value: model_a
|
||||||
greedy: false
|
indirect_selection: "cautious"
|
||||||
- name: model_a_greedy_true
|
- name: model_a_yes_indirect_selection
|
||||||
definition:
|
definition:
|
||||||
method: fqn
|
method: fqn
|
||||||
value: model_a
|
value: model_a
|
||||||
greedy: true
|
indirect_selection: "eager"
|
||||||
''')
|
''')
|
||||||
|
|
||||||
@use_profile('postgres')
|
@use_profile('postgres')
|
||||||
def test__postgres__selector_model_a_not_greedy(self):
|
def test__postgres__selector_model_a_unset_indirect_selection(self):
|
||||||
expected = ['just_a','unique_model_a_fun']
|
|
||||||
|
|
||||||
# when greedy is not specified, so implicitly False
|
|
||||||
self.list_tests_and_assert(include=None, exclude=None, expected_tests=expected, selector_name='model_a_greedy_none')
|
|
||||||
self.run_tests_and_assert(include=None, exclude=None, expected_tests=expected, selector_name='model_a_greedy_none')
|
|
||||||
|
|
||||||
# when greedy is explicitly False
|
|
||||||
self.list_tests_and_assert(include=None, exclude=None, expected_tests=expected, selector_name='model_a_greedy_false')
|
|
||||||
self.run_tests_and_assert(include=None, exclude=None, expected_tests=expected, selector_name='model_a_greedy_false')
|
|
||||||
|
|
||||||
|
|
||||||
@use_profile('postgres')
|
|
||||||
def test__postgres__selector_model_a_yes_greedy(self):
|
|
||||||
expected = [
|
expected = [
|
||||||
'cf_a_b', 'cf_a_src', 'just_a',
|
'cf_a_b', 'cf_a_src', 'just_a',
|
||||||
'relationships_model_a_fun__fun__ref_model_b_',
|
'relationships_model_a_fun__fun__ref_model_b_',
|
||||||
@@ -303,6 +327,25 @@ class TestExpansionWithSelectors(TestSelectionExpansion):
|
|||||||
'unique_model_a_fun'
|
'unique_model_a_fun'
|
||||||
]
|
]
|
||||||
|
|
||||||
# when greedy is explicitly False
|
self.list_tests_and_assert(include=None, exclude=None, expected_tests=expected, selector_name='model_a_unset_indirect_selection')
|
||||||
self.list_tests_and_assert(include=None, exclude=None, expected_tests=expected, selector_name='model_a_greedy_true')
|
self.run_tests_and_assert(include=None, exclude=None, expected_tests=expected, selector_name='model_a_unset_indirect_selection')
|
||||||
self.run_tests_and_assert(include=None, exclude=None, expected_tests=expected, selector_name='model_a_greedy_true')
|
|
||||||
|
@use_profile('postgres')
|
||||||
|
def test__postgres__selector_model_a_no_indirect_selection(self):
|
||||||
|
expected = ['just_a','unique_model_a_fun']
|
||||||
|
|
||||||
|
self.list_tests_and_assert(include=None, exclude=None, expected_tests=expected, selector_name='model_a_no_indirect_selection')
|
||||||
|
self.run_tests_and_assert(include=None, exclude=None, expected_tests=expected, selector_name='model_a_no_indirect_selection')
|
||||||
|
|
||||||
|
|
||||||
|
@use_profile('postgres')
|
||||||
|
def test__postgres__selector_model_a_yes_indirect_selection(self):
|
||||||
|
expected = [
|
||||||
|
'cf_a_b', 'cf_a_src', 'just_a',
|
||||||
|
'relationships_model_a_fun__fun__ref_model_b_',
|
||||||
|
'relationships_model_a_fun__fun__source_my_src_my_tbl_',
|
||||||
|
'unique_model_a_fun'
|
||||||
|
]
|
||||||
|
|
||||||
|
self.list_tests_and_assert(include=None, exclude=None, expected_tests=expected, selector_name='model_a_yes_indirect_selection')
|
||||||
|
self.run_tests_and_assert(include=None, exclude=None, expected_tests=expected, selector_name='model_a_yes_indirect_selection')
|
||||||
|
|||||||
Reference in New Issue
Block a user