Compare commits

...

9 Commits

Author SHA1 Message Date
Jeremy Cohen
8f41453f7b Use enum for IndirectSelection 2021-10-28 12:08:53 +02:00
Mila Page
fa42880d6e All but selector test passing. 2021-10-27 19:51:38 -07:00
Mila Page
e3f699ce6a Tests expect eager on by default. 2021-10-27 18:08:18 -07:00
Mila Page
be117691ef Correct several failing tests now that default behavior was flipped. 2021-10-27 01:22:07 -07:00
Mila Page
f2b6e65f00 Update names and styling to fit test requirements. Add default value for option. 2021-10-27 00:59:42 -07:00
Mila Page
d347fb8b59 Update tests and add new case for when flag unset. 2021-10-26 22:42:35 -07:00
Mila Page
6afc0abd0d 1. Set testing flag default to True. 2. Improve arg parsing. 2021-10-26 22:20:46 -07:00
Mila Page
47df8259af Remove references to greedy and supporting functions. 2021-10-20 18:32:27 -07:00
Mila Page
a8f8ff09af Parser no longer takes greedy. Accepts indirect selection, a bool. 2021-10-20 17:54:19 -07:00
6 changed files with 173 additions and 137 deletions

View File

@@ -17,7 +17,7 @@ PROFILES_DIR = os.path.expanduser(
STRICT_MODE = False # Only here for backwards compatibility STRICT_MODE = False # Only here for backwards compatibility
FULL_REFRESH = False # subcommand FULL_REFRESH = False # subcommand
STORE_FAILURES = False # subcommand STORE_FAILURES = False # subcommand
GREEDY = None # subcommand INDIRECT_SELECTION = 'eager' # subcommand
# Global CLI commands # Global CLI commands
USE_EXPERIMENTAL_PARSER = None USE_EXPERIMENTAL_PARSER = None
@@ -95,7 +95,7 @@ MP_CONTEXT = _get_context()
def set_from_args(args, user_config): def set_from_args(args, user_config):
global STRICT_MODE, FULL_REFRESH, WARN_ERROR, \ global STRICT_MODE, FULL_REFRESH, WARN_ERROR, \
USE_EXPERIMENTAL_PARSER, STATIC_PARSER, WRITE_JSON, PARTIAL_PARSE, \ USE_EXPERIMENTAL_PARSER, STATIC_PARSER, WRITE_JSON, PARTIAL_PARSE, \
USE_COLORS, STORE_FAILURES, PROFILES_DIR, DEBUG, LOG_FORMAT, GREEDY, \ USE_COLORS, STORE_FAILURES, PROFILES_DIR, DEBUG, LOG_FORMAT, INDIRECT_SELECTION, \
VERSION_CHECK, FAIL_FAST, SEND_ANONYMOUS_USAGE_STATS, PRINTER_WIDTH, \ VERSION_CHECK, FAIL_FAST, SEND_ANONYMOUS_USAGE_STATS, PRINTER_WIDTH, \
WHICH WHICH
@@ -103,7 +103,7 @@ def set_from_args(args, user_config):
# cli args without user_config or env var option # cli args without user_config or env var option
FULL_REFRESH = getattr(args, 'full_refresh', FULL_REFRESH) FULL_REFRESH = getattr(args, 'full_refresh', FULL_REFRESH)
STORE_FAILURES = getattr(args, 'store_failures', STORE_FAILURES) STORE_FAILURES = getattr(args, 'store_failures', STORE_FAILURES)
GREEDY = getattr(args, 'greedy', GREEDY) INDIRECT_SELECTION = getattr(args, 'indirect_selection', INDIRECT_SELECTION)
WHICH = getattr(args, 'which', WHICH) WHICH = getattr(args, 'which', WHICH)
# global cli flags with env var and user_config alternatives # global cli flags with env var and user_config alternatives

View File

@@ -16,6 +16,7 @@ from .selector_spec import (
SelectionIntersection, SelectionIntersection,
SelectionDifference, SelectionDifference,
SelectionCriteria, SelectionCriteria,
IndirectSelection
) )
INTERSECTION_DELIMITER = ',' INTERSECTION_DELIMITER = ','
@@ -25,7 +26,7 @@ DEFAULT_EXCLUDES: List[str] = []
def parse_union( def parse_union(
components: List[str], expect_exists: bool, greedy: bool = False components: List[str], expect_exists: bool, indirect_selection: IndirectSelection = IndirectSelection.Eager
) -> SelectionUnion: ) -> SelectionUnion:
# turn ['a b', 'c'] -> ['a', 'b', 'c'] # turn ['a b', 'c'] -> ['a', 'b', 'c']
raw_specs = itertools.chain.from_iterable( raw_specs = itertools.chain.from_iterable(
@@ -36,7 +37,7 @@ def parse_union(
# ['a', 'b', 'c,d'] -> union('a', 'b', intersection('c', 'd')) # ['a', 'b', 'c,d'] -> union('a', 'b', intersection('c', 'd'))
for raw_spec in raw_specs: for raw_spec in raw_specs:
intersection_components: List[SelectionSpec] = [ intersection_components: List[SelectionSpec] = [
SelectionCriteria.from_single_spec(part, greedy=greedy) SelectionCriteria.from_single_spec(part, indirect_selection=indirect_selection)
for part in raw_spec.split(INTERSECTION_DELIMITER) for part in raw_spec.split(INTERSECTION_DELIMITER)
] ]
union_components.append(SelectionIntersection( union_components.append(SelectionIntersection(
@@ -52,21 +53,25 @@ def parse_union(
def parse_union_from_default( def parse_union_from_default(
raw: Optional[List[str]], default: List[str], greedy: bool = False raw: Optional[List[str]], default: List[str], indirect_selection: IndirectSelection = IndirectSelection.Eager
) -> SelectionUnion: ) -> SelectionUnion:
components: List[str] components: List[str]
expect_exists: bool expect_exists: bool
if raw is None: if raw is None:
return parse_union(components=default, expect_exists=False, greedy=greedy) return parse_union(components=default, expect_exists=False, indirect_selection=indirect_selection)
else: else:
return parse_union(components=raw, expect_exists=True, greedy=greedy) return parse_union(components=raw, expect_exists=True, indirect_selection=indirect_selection)
def parse_difference( def parse_difference(
include: Optional[List[str]], exclude: Optional[List[str]] include: Optional[List[str]], exclude: Optional[List[str]]
) -> SelectionDifference: ) -> SelectionDifference:
included = parse_union_from_default(include, DEFAULT_INCLUDES, greedy=bool(flags.GREEDY)) included = parse_union_from_default(
excluded = parse_union_from_default(exclude, DEFAULT_EXCLUDES, greedy=True) include,
DEFAULT_INCLUDES,
indirect_selection=IndirectSelection(flags.INDIRECT_SELECTION)
)
excluded = parse_union_from_default(exclude, DEFAULT_EXCLUDES, indirect_selection=IndirectSelection.Eager)
return SelectionDifference(components=[included, excluded]) return SelectionDifference(components=[included, excluded])
@@ -148,7 +153,7 @@ def parse_union_definition(definition: Dict[str, Any]) -> SelectionSpec:
union_def_parts = _get_list_dicts(definition, 'union') union_def_parts = _get_list_dicts(definition, 'union')
include, exclude = _parse_include_exclude_subdefs(union_def_parts) include, exclude = _parse_include_exclude_subdefs(union_def_parts)
union = SelectionUnion(components=include, greedy_warning=False) union = SelectionUnion(components=include)
if exclude is None: if exclude is None:
union.raw = definition union.raw = definition
@@ -156,8 +161,7 @@ def parse_union_definition(definition: Dict[str, Any]) -> SelectionSpec:
else: else:
return SelectionDifference( return SelectionDifference(
components=[union, exclude], components=[union, exclude],
raw=definition, raw=definition
greedy_warning=False
) )
@@ -166,7 +170,7 @@ def parse_intersection_definition(
) -> SelectionSpec: ) -> SelectionSpec:
intersection_def_parts = _get_list_dicts(definition, 'intersection') intersection_def_parts = _get_list_dicts(definition, 'intersection')
include, exclude = _parse_include_exclude_subdefs(intersection_def_parts) include, exclude = _parse_include_exclude_subdefs(intersection_def_parts)
intersection = SelectionIntersection(components=include, greedy_warning=False) intersection = SelectionIntersection(components=include)
if exclude is None: if exclude is None:
intersection.raw = definition intersection.raw = definition
@@ -174,8 +178,7 @@ def parse_intersection_definition(
else: else:
return SelectionDifference( return SelectionDifference(
components=[intersection, exclude], components=[intersection, exclude],
raw=definition, raw=definition
greedy_warning=False
) )
@@ -209,7 +212,7 @@ def parse_dict_definition(definition: Dict[str, Any]) -> SelectionSpec:
if diff_arg is None: if diff_arg is None:
return base return base
else: else:
return SelectionDifference(components=[base, diff_arg], greedy_warning=False) return SelectionDifference(components=[base, diff_arg])
def parse_from_definition( def parse_from_definition(

View File

@@ -3,7 +3,7 @@ from typing import Set, List, Optional, Tuple
from .graph import Graph, UniqueId from .graph import Graph, UniqueId
from .queue import GraphQueue from .queue import GraphQueue
from .selector_methods import MethodManager from .selector_methods import MethodManager
from .selector_spec import SelectionCriteria, SelectionSpec from .selector_spec import SelectionCriteria, SelectionSpec, IndirectSelection
from dbt.logger import GLOBAL_LOGGER as logger from dbt.logger import GLOBAL_LOGGER as logger
from dbt.node_types import NodeType from dbt.node_types import NodeType
@@ -29,24 +29,6 @@ def alert_non_existence(raw_spec, nodes):
) )
def alert_unused_nodes(raw_spec, node_names):
summary_nodes_str = ("\n - ").join(node_names[:3])
debug_nodes_str = ("\n - ").join(node_names)
and_more_str = f"\n - and {len(node_names) - 3} more" if len(node_names) > 4 else ""
summary_msg = (
f"\nSome tests were excluded because at least one parent is not selected. "
f"Use the --greedy flag to include them."
f"\n - {summary_nodes_str}{and_more_str}"
)
logger.info(summary_msg)
if len(node_names) > 4:
debug_msg = (
f"Full list of tests that were excluded:"
f"\n - {debug_nodes_str}"
)
logger.debug(debug_msg)
def can_select_indirectly(node): def can_select_indirectly(node):
"""If a node is not selected itself, but its parent(s) are, it may qualify """If a node is not selected itself, but its parent(s) are, it may qualify
for indirect selection. for indirect selection.
@@ -113,7 +95,7 @@ class NodeSelector(MethodManager):
neighbors = self.collect_specified_neighbors(spec, collected) neighbors = self.collect_specified_neighbors(spec, collected)
direct_nodes, indirect_nodes = self.expand_selection( direct_nodes, indirect_nodes = self.expand_selection(
selected=(collected | neighbors), selected=(collected | neighbors),
greedy=spec.greedy indirect_selection=spec.indirect_selection
) )
return direct_nodes, indirect_nodes return direct_nodes, indirect_nodes
@@ -217,21 +199,20 @@ class NodeSelector(MethodManager):
} }
def expand_selection( def expand_selection(
self, selected: Set[UniqueId], greedy: bool = False self, selected: Set[UniqueId], indirect_selection: IndirectSelection = IndirectSelection.Eager
) -> Tuple[Set[UniqueId], Set[UniqueId]]: ) -> Tuple[Set[UniqueId], Set[UniqueId]]:
# Test selection can expand to include an implicitly/indirectly selected test. # Test selection by default expands to include an implicitly/indirectly selected tests.
# In this way, `dbt test -m model_a` also includes tests that directly depend on `model_a`. # `dbt test -m model_a` also includes tests that directly depend on `model_a`.
# Expansion has two modes, GREEDY and NOT GREEDY. # Expansion has two modes, EAGER and CAUTIOUS.
# #
# GREEDY mode: If ANY parent is selected, select the test. We use this for EXCLUSION. # EAGER mode: If ANY parent is selected, select the test.
# #
# NOT GREEDY mode: # CAUTIOUS mode:
# - If ALL parents are selected, select the test. # - If ALL parents are selected, select the test.
# - If ANY parent is missing, return it separately. We'll keep it around # - If ANY parent is missing, return it separately. We'll keep it around
# for later and see if its other parents show up. # for later and see if its other parents show up.
# We use this for INCLUSION. # Users can opt out of inclusive EAGER mode by passing --indirect-selection cautious
# Users can also opt in to inclusive GREEDY mode by passing --greedy flag, # CLI argument or by specifying `indirect_selection: true` in a yaml selector
# or by specifying `greedy: true` in a yaml selector
direct_nodes = set(selected) direct_nodes = set(selected)
indirect_nodes = set() indirect_nodes = set()
@@ -241,7 +222,10 @@ class NodeSelector(MethodManager):
node = self.manifest.nodes[unique_id] node = self.manifest.nodes[unique_id]
if can_select_indirectly(node): if can_select_indirectly(node):
# should we add it in directly? # should we add it in directly?
if greedy or set(node.depends_on.nodes) <= set(selected): if (
indirect_selection == IndirectSelection.Eager
or set(node.depends_on.nodes) <= set(selected)
):
direct_nodes.add(unique_id) direct_nodes.add(unique_id)
# if not: # if not:
else: else:
@@ -255,6 +239,10 @@ class NodeSelector(MethodManager):
# Check tests previously selected indirectly to see if ALL their # Check tests previously selected indirectly to see if ALL their
# parents are now present. # parents are now present.
# performance: if identical, skip the processing below
if set(direct_nodes) == set(indirect_nodes):
return direct_nodes
selected = set(direct_nodes) selected = set(direct_nodes)
for unique_id in indirect_nodes: for unique_id in indirect_nodes:
@@ -278,16 +266,6 @@ class NodeSelector(MethodManager):
selected_nodes, indirect_only = self.select_nodes(spec) selected_nodes, indirect_only = self.select_nodes(spec)
filtered_nodes = self.filter_selection(selected_nodes) filtered_nodes = self.filter_selection(selected_nodes)
if indirect_only:
filtered_unused_nodes = self.filter_selection(indirect_only)
if filtered_unused_nodes and spec.greedy_warning:
# log anything that didn't make the cut
unused_node_names = []
for unique_id in filtered_unused_nodes:
name = self.manifest.nodes[unique_id].name
unused_node_names.append(name)
alert_unused_nodes(spec, unused_node_names)
return filtered_nodes return filtered_nodes
def get_graph_queue(self, spec: SelectionSpec) -> GraphQueue: def get_graph_queue(self, spec: SelectionSpec) -> GraphQueue:

View File

@@ -1,7 +1,9 @@
import os import os
import re import re
import enum
from abc import ABCMeta, abstractmethod from abc import ABCMeta, abstractmethod
from dataclasses import dataclass from dataclasses import dataclass
from dbt.dataclass_schema import StrEnum
from typing import ( from typing import (
Set, Iterator, List, Optional, Dict, Union, Any, Iterable, Tuple Set, Iterator, List, Optional, Dict, Union, Any, Iterable, Tuple
@@ -21,6 +23,10 @@ RAW_SELECTOR_PATTERN = re.compile(
) )
SELECTOR_METHOD_SEPARATOR = '.' SELECTOR_METHOD_SEPARATOR = '.'
class IndirectSelection(StrEnum):
Eager = 'eager'
Cautious = 'cautious'
def _probably_path(value: str): def _probably_path(value: str):
"""Decide if value is probably a path. Windows has two path separators, so """Decide if value is probably a path. Windows has two path separators, so
@@ -66,8 +72,7 @@ class SelectionCriteria:
parents_depth: Optional[int] parents_depth: Optional[int]
children: bool children: bool
children_depth: Optional[int] children_depth: Optional[int]
greedy: bool = False indirect_selection: IndirectSelection = IndirectSelection.Eager
greedy_warning: bool = False # do not raise warning for yaml selectors
def __post_init__(self): def __post_init__(self):
if self.children and self.childrens_parents: if self.children and self.childrens_parents:
@@ -105,7 +110,7 @@ class SelectionCriteria:
@classmethod @classmethod
def selection_criteria_from_dict( def selection_criteria_from_dict(
cls, raw: Any, dct: Dict[str, Any], greedy: bool = False cls, raw: Any, dct: Dict[str, Any], indirect_selection: IndirectSelection = IndirectSelection.Eager
) -> 'SelectionCriteria': ) -> 'SelectionCriteria':
if 'value' not in dct: if 'value' not in dct:
raise RuntimeException( raise RuntimeException(
@@ -125,7 +130,7 @@ class SelectionCriteria:
parents_depth=parents_depth, parents_depth=parents_depth,
children=bool(dct.get('children')), children=bool(dct.get('children')),
children_depth=children_depth, children_depth=children_depth,
greedy=(greedy or bool(dct.get('greedy'))), indirect_selection=(IndirectSelection(dct.get('indirect_selection') or indirect_selection)),
) )
@classmethod @classmethod
@@ -146,18 +151,22 @@ class SelectionCriteria:
dct['parents'] = bool(dct.get('parents')) dct['parents'] = bool(dct.get('parents'))
if 'children' in dct: if 'children' in dct:
dct['children'] = bool(dct.get('children')) dct['children'] = bool(dct.get('children'))
if 'greedy' in dct: if 'indirect_selection' in dct:
dct['greedy'] = bool(dct.get('greedy')) dct['indirect_selection'] = bool(dct.get('indirect_selection'))
return dct return dct
@classmethod @classmethod
def from_single_spec(cls, raw: str, greedy: bool = False) -> 'SelectionCriteria': def from_single_spec(cls, raw: str, indirect_selection: bool = True) -> 'SelectionCriteria':
result = RAW_SELECTOR_PATTERN.match(raw) result = RAW_SELECTOR_PATTERN.match(raw)
if result is None: if result is None:
# bad spec! # bad spec!
raise RuntimeException(f'Invalid selector spec "{raw}"') raise RuntimeException(f'Invalid selector spec "{raw}"')
return cls.selection_criteria_from_dict(raw, result.groupdict(), greedy=greedy) return cls.selection_criteria_from_dict(
raw,
result.groupdict(),
indirect_selection=indirect_selection
)
class BaseSelectionGroup(Iterable[SelectionSpec], metaclass=ABCMeta): class BaseSelectionGroup(Iterable[SelectionSpec], metaclass=ABCMeta):
@@ -165,12 +174,10 @@ class BaseSelectionGroup(Iterable[SelectionSpec], metaclass=ABCMeta):
self, self,
components: Iterable[SelectionSpec], components: Iterable[SelectionSpec],
expect_exists: bool = False, expect_exists: bool = False,
greedy_warning: bool = True,
raw: Any = None, raw: Any = None,
): ):
self.components: List[SelectionSpec] = list(components) self.components: List[SelectionSpec] = list(components)
self.expect_exists = expect_exists self.expect_exists = expect_exists
self.greedy_warning = greedy_warning
self.raw = raw self.raw = raw
def __iter__(self) -> Iterator[SelectionSpec]: def __iter__(self) -> Iterator[SelectionSpec]:

View File

@@ -392,13 +392,15 @@ def _build_build_subparser(subparsers, base_subparser):
''' '''
) )
sub.add_argument( sub.add_argument(
'--greedy', '--indirect-selection',
action='store_true', choices=['eager', 'cautious'],
dest='indirect_selection',
help=''' help='''
Select all tests that touch the selected resources, Select all tests that are adjacent to selected resources,
even if they also depend on unselected resources even if they those resources have been explicitly selected.
''' ''',
) )
resource_values: List[str] = [ resource_values: List[str] = [
str(s) for s in build_task.BuildTask.ALL_RESOURCE_VALUES str(s) for s in build_task.BuildTask.ALL_RESOURCE_VALUES
] + ['all'] ] + ['all']
@@ -736,12 +738,14 @@ def _build_test_subparser(subparsers, base_subparser):
''' '''
) )
sub.add_argument( sub.add_argument(
'--greedy', '--indirect-selection',
action='store_true', choices=['eager', 'cautious'],
default='eager',
dest='indirect_selection',
help=''' help='''
Select all tests that touch the selected resources, Select all tests that are adjacent to selected resources,
even if they also depend on unselected resources even if they those resources have been explicitly selected.
''' ''',
) )
sub.set_defaults(cls=test_task.TestTask, which='test', rpc_method='test') sub.set_defaults(cls=test_task.TestTask, which='test', rpc_method='test')
@@ -839,12 +843,13 @@ def _build_list_subparser(subparsers, base_subparser):
required=False, required=False,
) )
sub.add_argument( sub.add_argument(
'--greedy', '--indirect-selection',
action='store_true', choices=['eager', 'cautious'],
dest='indirect_selection',
help=''' help='''
Select all tests that touch the selected resources, Select all tests that are adjacent to selected resources,
even if they also depend on unselected resources even if they those resources have been explicitly selected.
''' ''',
) )
_add_common_selector_arguments(sub) _add_common_selector_arguments(sub)

View File

@@ -21,16 +21,14 @@ class TestSelectionExpansion(DBTIntegrationTest):
"test-paths": ["tests"] "test-paths": ["tests"]
} }
def list_tests_and_assert(self, include, exclude, expected_tests, greedy=False, selector_name=None): def list_tests_and_assert(self, include, exclude, expected_tests, indirect_selection='eager', selector_name=None):
list_args = [ 'ls', '--resource-type', 'test'] list_args = [ 'ls', '--resource-type', 'test']
if include: if include:
list_args.extend(('--select', include)) list_args.extend(('--select', include))
if exclude: if exclude:
list_args.extend(('--exclude', exclude)) list_args.extend(('--exclude', exclude))
if exclude: if indirect_selection:
list_args.extend(('--exclude', exclude)) list_args.extend(('--indirect-selection', indirect_selection))
if greedy:
list_args.append('--greedy')
if selector_name: if selector_name:
list_args.extend(('--selector', selector_name)) list_args.extend(('--selector', selector_name))
@@ -40,7 +38,7 @@ class TestSelectionExpansion(DBTIntegrationTest):
test_names = [name.split('.')[-1] for name in listed] test_names = [name.split('.')[-1] for name in listed]
assert sorted(test_names) == sorted(expected_tests) assert sorted(test_names) == sorted(expected_tests)
def run_tests_and_assert(self, include, exclude, expected_tests, greedy=False, selector_name=None): def run_tests_and_assert(self, include, exclude, expected_tests, indirect_selection='eager', selector_name=None):
results = self.run_dbt(['run']) results = self.run_dbt(['run'])
self.assertEqual(len(results), 2) self.assertEqual(len(results), 2)
@@ -49,8 +47,8 @@ class TestSelectionExpansion(DBTIntegrationTest):
test_args.extend(('--models', include)) test_args.extend(('--models', include))
if exclude: if exclude:
test_args.extend(('--exclude', exclude)) test_args.extend(('--exclude', exclude))
if greedy: if indirect_selection:
test_args.append('--greedy') test_args.extend(('--indirect-selection', indirect_selection))
if selector_name: if selector_name:
test_args.extend(('--selector', selector_name)) test_args.extend(('--selector', selector_name))
@@ -79,7 +77,12 @@ class TestSelectionExpansion(DBTIntegrationTest):
def test__postgres__model_a_alone(self): def test__postgres__model_a_alone(self):
select = 'model_a' select = 'model_a'
exclude = None exclude = None
expected = ['just_a','unique_model_a_fun'] expected = [
'cf_a_b', 'cf_a_src', 'just_a',
'relationships_model_a_fun__fun__ref_model_b_',
'relationships_model_a_fun__fun__source_my_src_my_tbl_',
'unique_model_a_fun'
]
self.list_tests_and_assert(select, exclude, expected) self.list_tests_and_assert(select, exclude, expected)
self.run_tests_and_assert(select, exclude, expected) self.run_tests_and_assert(select, exclude, expected)
@@ -89,10 +92,12 @@ class TestSelectionExpansion(DBTIntegrationTest):
select = 'model_a model_b' select = 'model_a model_b'
exclude = None exclude = None
expected = [ expected = [
'cf_a_b','just_a','unique_model_a_fun', 'cf_a_b','cf_a_src','just_a','unique_model_a_fun',
'relationships_model_a_fun__fun__ref_model_b_' 'relationships_model_a_fun__fun__ref_model_b_',
'relationships_model_a_fun__fun__source_my_src_my_tbl_'
] ]
self.list_tests_and_assert(select, exclude, expected) self.list_tests_and_assert(select, exclude, expected)
self.run_tests_and_assert(select, exclude, expected) self.run_tests_and_assert(select, exclude, expected)
@@ -101,8 +106,9 @@ class TestSelectionExpansion(DBTIntegrationTest):
select = 'model_a source:*' select = 'model_a source:*'
exclude = None exclude = None
expected = [ expected = [
'cf_a_src','just_a','unique_model_a_fun', 'cf_a_b','cf_a_src','just_a','unique_model_a_fun',
'source_unique_my_src_my_tbl_fun', 'source_unique_my_src_my_tbl_fun',
'relationships_model_a_fun__fun__ref_model_b_',
'relationships_model_a_fun__fun__source_my_src_my_tbl_' 'relationships_model_a_fun__fun__source_my_src_my_tbl_'
] ]
@@ -126,14 +132,18 @@ class TestSelectionExpansion(DBTIntegrationTest):
def test__postgres__model_a_exclude_specific_test(self): def test__postgres__model_a_exclude_specific_test(self):
select = 'model_a' select = 'model_a'
exclude = 'unique_model_a_fun' exclude = 'unique_model_a_fun'
expected = ['just_a'] expected = [
'cf_a_b','cf_a_src','just_a',
'relationships_model_a_fun__fun__ref_model_b_',
'relationships_model_a_fun__fun__source_my_src_my_tbl_'
]
self.list_tests_and_assert(select, exclude, expected) self.list_tests_and_assert(select, exclude, expected)
self.run_tests_and_assert(select, exclude, expected) self.run_tests_and_assert(select, exclude, expected)
@use_profile('postgres') @use_profile('postgres')
def test__postgres__only_schema(self): def test__postgres__only_generic(self):
select = 'test_type:schema' select = 'test_type:generic'
exclude = None exclude = None
expected = [ expected = [
'relationships_model_a_fun__fun__ref_model_b_', 'relationships_model_a_fun__fun__ref_model_b_',
@@ -146,17 +156,37 @@ class TestSelectionExpansion(DBTIntegrationTest):
self.run_tests_and_assert(select, exclude, expected) self.run_tests_and_assert(select, exclude, expected)
@use_profile('postgres') @use_profile('postgres')
def test__postgres__model_a_only_data(self): def test__postgres__model_a_only_singular_unset(self):
select = 'model_a,test_type:schema' select = 'model_a,test_type:singular'
exclude = None exclude = None
expected = ['unique_model_a_fun'] expected = ['cf_a_b','cf_a_src','just_a']
self.list_tests_and_assert(select, exclude, expected) self.list_tests_and_assert(select, exclude, expected)
self.run_tests_and_assert(select, exclude, expected) self.run_tests_and_assert(select, exclude, expected)
@use_profile('postgres') @use_profile('postgres')
def test__postgres__only_data(self): def test__postgres__model_a_only_singular_eager(self):
select = 'test_type:data' select = 'model_a,test_type:singular'
exclude = None
expected = ['cf_a_b','cf_a_src','just_a']
indirect_selection = 'eager'
self.list_tests_and_assert(select, exclude, expected)
self.run_tests_and_assert(select, exclude, expected)
@use_profile('postgres')
def test__postgres__model_a_only_singular_cautious(self):
select = 'model_a,test_type:singular'
exclude = None
expected = ['just_a']
indirect_selection = 'cautious'
self.list_tests_and_assert(select, exclude, expected, indirect_selection=indirect_selection)
self.run_tests_and_assert(select, exclude, expected, indirect_selection=indirect_selection)
@use_profile('postgres')
def test__postgres__only_singular(self):
select = 'test_type:singular'
exclude = None exclude = None
expected = ['cf_a_b', 'cf_a_src', 'just_a'] expected = ['cf_a_b', 'cf_a_src', 'just_a']
@@ -164,10 +194,10 @@ class TestSelectionExpansion(DBTIntegrationTest):
self.run_tests_and_assert(select, exclude, expected) self.run_tests_and_assert(select, exclude, expected)
@use_profile('postgres') @use_profile('postgres')
def test__postgres__model_a_only_data(self): def test__postgres__model_a_only_singular(self):
select = 'model_a,test_type:data' select = 'model_a,test_type:singular'
exclude = None exclude = None
expected = ['just_a'] expected = ['cf_a_b','cf_a_src','just_a']
self.list_tests_and_assert(select, exclude, expected) self.list_tests_and_assert(select, exclude, expected)
self.run_tests_and_assert(select, exclude, expected) self.run_tests_and_assert(select, exclude, expected)
@@ -185,7 +215,10 @@ class TestSelectionExpansion(DBTIntegrationTest):
def test__postgres__model_tag_test_name_intersection(self): def test__postgres__model_tag_test_name_intersection(self):
select = 'tag:a_or_b,test_name:relationships' select = 'tag:a_or_b,test_name:relationships'
exclude = None exclude = None
expected = ['relationships_model_a_fun__fun__ref_model_b_'] expected = [
'relationships_model_a_fun__fun__ref_model_b_',
'relationships_model_a_fun__fun__source_my_src_my_tbl_'
]
self.list_tests_and_assert(select, exclude, expected) self.list_tests_and_assert(select, exclude, expected)
self.run_tests_and_assert(select, exclude, expected) self.run_tests_and_assert(select, exclude, expected)
@@ -225,16 +258,20 @@ class TestSelectionExpansion(DBTIntegrationTest):
def test__postgres__exclude_data_test_tag(self): def test__postgres__exclude_data_test_tag(self):
select = 'model_a' select = 'model_a'
exclude = 'tag:data_test_tag' exclude = 'tag:data_test_tag'
expected = ['unique_model_a_fun'] expected = [
'cf_a_b', 'cf_a_src',
'relationships_model_a_fun__fun__ref_model_b_',
'relationships_model_a_fun__fun__source_my_src_my_tbl_',
'unique_model_a_fun'
]
self.list_tests_and_assert(select, exclude, expected) self.list_tests_and_assert(select, exclude, expected)
self.run_tests_and_assert(select, exclude, expected) self.run_tests_and_assert(select, exclude, expected)
@use_profile('postgres') @use_profile('postgres')
def test__postgres__model_a_greedy(self): def test__postgres__model_a_indirect_selection(self):
select = 'model_a' select = 'model_a'
exclude = None exclude = None
greedy = True
expected = [ expected = [
'cf_a_b', 'cf_a_src', 'just_a', 'cf_a_b', 'cf_a_src', 'just_a',
'relationships_model_a_fun__fun__ref_model_b_', 'relationships_model_a_fun__fun__ref_model_b_',
@@ -242,22 +279,22 @@ class TestSelectionExpansion(DBTIntegrationTest):
'unique_model_a_fun' 'unique_model_a_fun'
] ]
self.list_tests_and_assert(select, exclude, expected, greedy) self.list_tests_and_assert(select, exclude, expected)
self.run_tests_and_assert(select, exclude, expected, greedy=greedy) self.run_tests_and_assert(select, exclude, expected)
@use_profile('postgres') @use_profile('postgres')
def test__postgres__model_a_greedy_exclude_unique_tests(self): def test__postgres__model_a_indirect_selection_exclude_unique_tests(self):
select = 'model_a' select = 'model_a'
exclude = 'test_name:unique' exclude = 'test_name:unique'
greedy = True indirect_selection = 'eager'
expected = [ expected = [
'cf_a_b', 'cf_a_src', 'just_a', 'cf_a_b', 'cf_a_src', 'just_a',
'relationships_model_a_fun__fun__ref_model_b_', 'relationships_model_a_fun__fun__ref_model_b_',
'relationships_model_a_fun__fun__source_my_src_my_tbl_', 'relationships_model_a_fun__fun__source_my_src_my_tbl_',
] ]
self.list_tests_and_assert(select, exclude, expected, greedy) self.list_tests_and_assert(select, exclude, expected, indirect_selection)
self.run_tests_and_assert(select, exclude, expected, greedy=greedy) self.run_tests_and_assert(select, exclude, expected, indirect_selection=indirect_selection)
class TestExpansionWithSelectors(TestSelectionExpansion): class TestExpansionWithSelectors(TestSelectionExpansion):
@@ -265,37 +302,24 @@ class TestExpansionWithSelectors(TestSelectionExpansion):
def selectors_config(self): def selectors_config(self):
return yaml.safe_load(''' return yaml.safe_load('''
selectors: selectors:
- name: model_a_greedy_none - name: model_a_unset_indirect_selection
definition: definition:
method: fqn method: fqn
value: model_a value: model_a
- name: model_a_greedy_false - name: model_a_no_indirect_selection
definition: definition:
method: fqn method: fqn
value: model_a value: model_a
greedy: false indirect_selection: "cautious"
- name: model_a_greedy_true - name: model_a_yes_indirect_selection
definition: definition:
method: fqn method: fqn
value: model_a value: model_a
greedy: true indirect_selection: "eager"
''') ''')
@use_profile('postgres') @use_profile('postgres')
def test__postgres__selector_model_a_not_greedy(self): def test__postgres__selector_model_a_unset_indirect_selection(self):
expected = ['just_a','unique_model_a_fun']
# when greedy is not specified, so implicitly False
self.list_tests_and_assert(include=None, exclude=None, expected_tests=expected, selector_name='model_a_greedy_none')
self.run_tests_and_assert(include=None, exclude=None, expected_tests=expected, selector_name='model_a_greedy_none')
# when greedy is explicitly False
self.list_tests_and_assert(include=None, exclude=None, expected_tests=expected, selector_name='model_a_greedy_false')
self.run_tests_and_assert(include=None, exclude=None, expected_tests=expected, selector_name='model_a_greedy_false')
@use_profile('postgres')
def test__postgres__selector_model_a_yes_greedy(self):
expected = [ expected = [
'cf_a_b', 'cf_a_src', 'just_a', 'cf_a_b', 'cf_a_src', 'just_a',
'relationships_model_a_fun__fun__ref_model_b_', 'relationships_model_a_fun__fun__ref_model_b_',
@@ -303,6 +327,25 @@ class TestExpansionWithSelectors(TestSelectionExpansion):
'unique_model_a_fun' 'unique_model_a_fun'
] ]
# when greedy is explicitly False self.list_tests_and_assert(include=None, exclude=None, expected_tests=expected, selector_name='model_a_unset_indirect_selection')
self.list_tests_and_assert(include=None, exclude=None, expected_tests=expected, selector_name='model_a_greedy_true') self.run_tests_and_assert(include=None, exclude=None, expected_tests=expected, selector_name='model_a_unset_indirect_selection')
self.run_tests_and_assert(include=None, exclude=None, expected_tests=expected, selector_name='model_a_greedy_true')
@use_profile('postgres')
def test__postgres__selector_model_a_no_indirect_selection(self):
expected = ['just_a','unique_model_a_fun']
self.list_tests_and_assert(include=None, exclude=None, expected_tests=expected, selector_name='model_a_no_indirect_selection')
self.run_tests_and_assert(include=None, exclude=None, expected_tests=expected, selector_name='model_a_no_indirect_selection')
@use_profile('postgres')
def test__postgres__selector_model_a_yes_indirect_selection(self):
expected = [
'cf_a_b', 'cf_a_src', 'just_a',
'relationships_model_a_fun__fun__ref_model_b_',
'relationships_model_a_fun__fun__source_my_src_my_tbl_',
'unique_model_a_fun'
]
self.list_tests_and_assert(include=None, exclude=None, expected_tests=expected, selector_name='model_a_yes_indirect_selection')
self.run_tests_and_assert(include=None, exclude=None, expected_tests=expected, selector_name='model_a_yes_indirect_selection')