mirror of
https://github.com/dbt-labs/dbt-core
synced 2025-12-19 01:41:28 +00:00
Compare commits
23 Commits
enable-pos
...
v0.21.0rc1
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
641b0fa365 | ||
|
|
8876afdb14 | ||
|
|
a97a9c9942 | ||
|
|
0070cd99de | ||
|
|
78a1bbe3c7 | ||
|
|
cde82fa2b1 | ||
|
|
34cec7c7b0 | ||
|
|
db5caf97ae | ||
|
|
847046171e | ||
|
|
5dd37a9fb8 | ||
|
|
a2bdd08d88 | ||
|
|
1807526d0a | ||
|
|
362770f5bd | ||
|
|
af38f51041 | ||
|
|
efc8ece12e | ||
|
|
7471f07431 | ||
|
|
6fa30d10ea | ||
|
|
35150f914f | ||
|
|
b477be9eff | ||
|
|
b67e877cc1 | ||
|
|
1c066cd680 | ||
|
|
ec97b46caf | ||
|
|
b5bb354929 |
@@ -1,5 +1,5 @@
|
||||
[bumpversion]
|
||||
current_version = 0.21.0b2
|
||||
current_version = 0.21.0rc1
|
||||
parse = (?P<major>\d+)
|
||||
\.(?P<minor>\d+)
|
||||
\.(?P<patch>\d+)
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
## dbt 0.21.0 (Release TBD)
|
||||
|
||||
## dbt 0.21.0b2 (August 19, 2021)
|
||||
## dbt 0.21.0rc1 (September 20, 2021)
|
||||
|
||||
|
||||
|
||||
|
||||
### Features
|
||||
@@ -16,6 +18,7 @@
|
||||
- Added default field in the `selectors.yml` to allow user to define default selector ([#3448](https://github.com/dbt-labs/dbt/issues/3448), [#3875](https://github.com/dbt-labs/dbt/issues/3875), [#3892](https://github.com/dbt-labs/dbt/issues/3892))
|
||||
- Added timing and thread information to sources.json artifact ([#3804](https://github.com/dbt-labs/dbt/issues/3804), [#3894](https://github.com/dbt-labs/dbt/pull/3894))
|
||||
- Update cli and rpc flags for the `build` task to align with other commands (`--resource-type`, `--store-failures`) ([#3596](https://github.com/dbt-labs/dbt/issues/3596), [#3884](https://github.com/dbt-labs/dbt/pull/3884))
|
||||
- Log tests that are not indirectly selected. Add `--greedy` flag to `test`, `list`, `build` and `greedy` property in yaml selectors ([#3723](https://github.com/dbt-labs/dbt/pull/3723), [#3833](https://github.com/dbt-labs/dbt/pull/3833))
|
||||
|
||||
### Fixes
|
||||
|
||||
@@ -44,7 +47,7 @@ Contributors:
|
||||
- [@dbrtly](https://github.com/dbrtly) ([#3834](https://github.com/dbt-labs/dbt/pull/3834))
|
||||
- [@swanderz](https://github.com/swanderz) [#3623](https://github.com/dbt-labs/dbt/pull/3623)
|
||||
- [@JasonGluck](https://github.com/JasonGluck) ([#3582](https://github.com/dbt-labs/dbt/pull/3582))
|
||||
- [@joellabes](https://github.com/joellabes) ([#3669](https://github.com/dbt-labs/dbt/pull/3669))
|
||||
- [@joellabes](https://github.com/joellabes) ([#3669](https://github.com/dbt-labs/dbt/pull/3669), [#3833](https://github.com/dbt-labs/dbt/pull/3833))
|
||||
- [@juma-adoreme](https://github.com/juma-adoreme) ([#3838](https://github.com/dbt-labs/dbt/pull/3838))
|
||||
- [@annafil](https://github.com/annafil) ([#3825](https://github.com/dbt-labs/dbt/pull/3825))
|
||||
- [@AndreasTA-AW](https://github.com/AndreasTA-AW) ([#3691](https://github.com/dbt-labs/dbt/pull/3691))
|
||||
@@ -52,6 +55,7 @@ Contributors:
|
||||
- [@TeddyCr](https://github.com/TeddyCr) ([#3448](https://github.com/dbt-labs/dbt/pull/3865))
|
||||
- [@sdebruyn](https://github.com/sdebruyn) ([#3906](https://github.com/dbt-labs/dbt/pull/3906))
|
||||
|
||||
|
||||
## dbt 0.21.0b2 (August 19, 2021)
|
||||
|
||||
### Features
|
||||
|
||||
@@ -18,6 +18,7 @@ WRITE_JSON = None
|
||||
PARTIAL_PARSE = None
|
||||
USE_COLORS = None
|
||||
STORE_FAILURES = None
|
||||
GREEDY = None
|
||||
|
||||
|
||||
def env_set_truthy(key: str) -> Optional[str]:
|
||||
@@ -56,7 +57,7 @@ MP_CONTEXT = _get_context()
|
||||
def reset():
|
||||
global STRICT_MODE, FULL_REFRESH, USE_CACHE, WARN_ERROR, TEST_NEW_PARSER, \
|
||||
USE_EXPERIMENTAL_PARSER, WRITE_JSON, PARTIAL_PARSE, MP_CONTEXT, USE_COLORS, \
|
||||
STORE_FAILURES
|
||||
STORE_FAILURES, GREEDY
|
||||
|
||||
STRICT_MODE = False
|
||||
FULL_REFRESH = False
|
||||
@@ -69,12 +70,13 @@ def reset():
|
||||
MP_CONTEXT = _get_context()
|
||||
USE_COLORS = True
|
||||
STORE_FAILURES = False
|
||||
GREEDY = False
|
||||
|
||||
|
||||
def set_from_args(args):
|
||||
global STRICT_MODE, FULL_REFRESH, USE_CACHE, WARN_ERROR, TEST_NEW_PARSER, \
|
||||
USE_EXPERIMENTAL_PARSER, WRITE_JSON, PARTIAL_PARSE, MP_CONTEXT, USE_COLORS, \
|
||||
STORE_FAILURES
|
||||
STORE_FAILURES, GREEDY
|
||||
|
||||
USE_CACHE = getattr(args, 'use_cache', USE_CACHE)
|
||||
|
||||
@@ -99,6 +101,7 @@ def set_from_args(args):
|
||||
USE_COLORS = use_colors_override
|
||||
|
||||
STORE_FAILURES = getattr(args, 'store_failures', STORE_FAILURES)
|
||||
GREEDY = getattr(args, 'greedy', GREEDY)
|
||||
|
||||
|
||||
# initialize everything to the defaults on module load
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
# special support for CLI argument parsing.
|
||||
from dbt import flags
|
||||
import itertools
|
||||
from dbt.clients.yaml_helper import yaml, Loader, Dumper # noqa: F401
|
||||
|
||||
@@ -66,7 +67,7 @@ def parse_union_from_default(
|
||||
def parse_difference(
|
||||
include: Optional[List[str]], exclude: Optional[List[str]]
|
||||
) -> SelectionDifference:
|
||||
included = parse_union_from_default(include, DEFAULT_INCLUDES)
|
||||
included = parse_union_from_default(include, DEFAULT_INCLUDES, greedy=bool(flags.GREEDY))
|
||||
excluded = parse_union_from_default(exclude, DEFAULT_EXCLUDES, greedy=True)
|
||||
return SelectionDifference(components=[included, excluded])
|
||||
|
||||
@@ -180,7 +181,7 @@ def parse_union_definition(definition: Dict[str, Any]) -> SelectionSpec:
|
||||
union_def_parts = _get_list_dicts(definition, 'union')
|
||||
include, exclude = _parse_include_exclude_subdefs(union_def_parts)
|
||||
|
||||
union = SelectionUnion(components=include)
|
||||
union = SelectionUnion(components=include, greedy_warning=False)
|
||||
|
||||
if exclude is None:
|
||||
union.raw = definition
|
||||
@@ -188,7 +189,8 @@ def parse_union_definition(definition: Dict[str, Any]) -> SelectionSpec:
|
||||
else:
|
||||
return SelectionDifference(
|
||||
components=[union, exclude],
|
||||
raw=definition
|
||||
raw=definition,
|
||||
greedy_warning=False
|
||||
)
|
||||
|
||||
|
||||
@@ -197,7 +199,7 @@ def parse_intersection_definition(
|
||||
) -> SelectionSpec:
|
||||
intersection_def_parts = _get_list_dicts(definition, 'intersection')
|
||||
include, exclude = _parse_include_exclude_subdefs(intersection_def_parts)
|
||||
intersection = SelectionIntersection(components=include)
|
||||
intersection = SelectionIntersection(components=include, greedy_warning=False)
|
||||
|
||||
if exclude is None:
|
||||
intersection.raw = definition
|
||||
@@ -205,7 +207,8 @@ def parse_intersection_definition(
|
||||
else:
|
||||
return SelectionDifference(
|
||||
components=[intersection, exclude],
|
||||
raw=definition
|
||||
raw=definition,
|
||||
greedy_warning=False
|
||||
)
|
||||
|
||||
|
||||
@@ -239,7 +242,7 @@ def parse_dict_definition(definition: Dict[str, Any]) -> SelectionSpec:
|
||||
if diff_arg is None:
|
||||
return base
|
||||
else:
|
||||
return SelectionDifference(components=[base, diff_arg])
|
||||
return SelectionDifference(components=[base, diff_arg], greedy_warning=False)
|
||||
|
||||
|
||||
def parse_from_definition(
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
|
||||
from typing import Set, List, Optional, Tuple
|
||||
|
||||
from .graph import Graph, UniqueId
|
||||
@@ -30,6 +29,24 @@ def alert_non_existence(raw_spec, nodes):
|
||||
)
|
||||
|
||||
|
||||
def alert_unused_nodes(raw_spec, node_names):
|
||||
summary_nodes_str = ("\n - ").join(node_names[:3])
|
||||
debug_nodes_str = ("\n - ").join(node_names)
|
||||
and_more_str = f"\n - and {len(node_names) - 3} more" if len(node_names) > 4 else ""
|
||||
summary_msg = (
|
||||
f"\nSome tests were excluded because at least one parent is not selected. "
|
||||
f"Use the --greedy flag to include them."
|
||||
f"\n - {summary_nodes_str}{and_more_str}"
|
||||
)
|
||||
logger.info(summary_msg)
|
||||
if len(node_names) > 4:
|
||||
debug_msg = (
|
||||
f"Full list of tests that were excluded:"
|
||||
f"\n - {debug_nodes_str}"
|
||||
)
|
||||
logger.debug(debug_msg)
|
||||
|
||||
|
||||
def can_select_indirectly(node):
|
||||
"""If a node is not selected itself, but its parent(s) are, it may qualify
|
||||
for indirect selection.
|
||||
@@ -151,16 +168,16 @@ class NodeSelector(MethodManager):
|
||||
|
||||
return direct_nodes, indirect_nodes
|
||||
|
||||
def select_nodes(self, spec: SelectionSpec) -> Set[UniqueId]:
|
||||
def select_nodes(self, spec: SelectionSpec) -> Tuple[Set[UniqueId], Set[UniqueId]]:
|
||||
"""Select the nodes in the graph according to the spec.
|
||||
|
||||
This is the main point of entry for turning a spec into a set of nodes:
|
||||
- Recurse through spec, select by criteria, combine by set operation
|
||||
- Return final (unfiltered) selection set
|
||||
"""
|
||||
|
||||
direct_nodes, indirect_nodes = self.select_nodes_recursively(spec)
|
||||
return direct_nodes
|
||||
indirect_only = indirect_nodes.difference(direct_nodes)
|
||||
return direct_nodes, indirect_only
|
||||
|
||||
def _is_graph_member(self, unique_id: UniqueId) -> bool:
|
||||
if unique_id in self.manifest.sources:
|
||||
@@ -213,6 +230,8 @@ class NodeSelector(MethodManager):
|
||||
# - If ANY parent is missing, return it separately. We'll keep it around
|
||||
# for later and see if its other parents show up.
|
||||
# We use this for INCLUSION.
|
||||
# Users can also opt in to inclusive GREEDY mode by passing --greedy flag,
|
||||
# or by specifying `greedy: true` in a yaml selector
|
||||
|
||||
direct_nodes = set(selected)
|
||||
indirect_nodes = set()
|
||||
@@ -251,15 +270,24 @@ class NodeSelector(MethodManager):
|
||||
|
||||
- node selection. Based on the include/exclude sets, the set
|
||||
of matched unique IDs is returned
|
||||
- expand the graph at each leaf node, before combination
|
||||
- selectors might override this. for example, this is where
|
||||
tests are added
|
||||
- includes direct + indirect selection (for tests)
|
||||
- filtering:
|
||||
- selectors can filter the nodes after all of them have been
|
||||
selected
|
||||
"""
|
||||
selected_nodes = self.select_nodes(spec)
|
||||
selected_nodes, indirect_only = self.select_nodes(spec)
|
||||
filtered_nodes = self.filter_selection(selected_nodes)
|
||||
|
||||
if indirect_only:
|
||||
filtered_unused_nodes = self.filter_selection(indirect_only)
|
||||
if filtered_unused_nodes and spec.greedy_warning:
|
||||
# log anything that didn't make the cut
|
||||
unused_node_names = []
|
||||
for unique_id in filtered_unused_nodes:
|
||||
name = self.manifest.nodes[unique_id].name
|
||||
unused_node_names.append(name)
|
||||
alert_unused_nodes(spec, unused_node_names)
|
||||
|
||||
return filtered_nodes
|
||||
|
||||
def get_graph_queue(self, spec: SelectionSpec) -> GraphQueue:
|
||||
|
||||
@@ -67,6 +67,7 @@ class SelectionCriteria:
|
||||
children: bool
|
||||
children_depth: Optional[int]
|
||||
greedy: bool = False
|
||||
greedy_warning: bool = False # do not raise warning for yaml selectors
|
||||
|
||||
def __post_init__(self):
|
||||
if self.children and self.childrens_parents:
|
||||
@@ -124,11 +125,11 @@ class SelectionCriteria:
|
||||
parents_depth=parents_depth,
|
||||
children=bool(dct.get('children')),
|
||||
children_depth=children_depth,
|
||||
greedy=greedy
|
||||
greedy=(greedy or bool(dct.get('greedy'))),
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def dict_from_single_spec(cls, raw: str, greedy: bool = False):
|
||||
def dict_from_single_spec(cls, raw: str):
|
||||
result = RAW_SELECTOR_PATTERN.match(raw)
|
||||
if result is None:
|
||||
return {'error': 'Invalid selector spec'}
|
||||
@@ -145,6 +146,8 @@ class SelectionCriteria:
|
||||
dct['parents'] = bool(dct.get('parents'))
|
||||
if 'children' in dct:
|
||||
dct['children'] = bool(dct.get('children'))
|
||||
if 'greedy' in dct:
|
||||
dct['greedy'] = bool(dct.get('greedy'))
|
||||
return dct
|
||||
|
||||
@classmethod
|
||||
@@ -162,10 +165,12 @@ class BaseSelectionGroup(Iterable[SelectionSpec], metaclass=ABCMeta):
|
||||
self,
|
||||
components: Iterable[SelectionSpec],
|
||||
expect_exists: bool = False,
|
||||
greedy_warning: bool = True,
|
||||
raw: Any = None,
|
||||
):
|
||||
self.components: List[SelectionSpec] = list(components)
|
||||
self.expect_exists = expect_exists
|
||||
self.greedy_warning = greedy_warning
|
||||
self.raw = raw
|
||||
|
||||
def __iter__(self) -> Iterator[SelectionSpec]:
|
||||
|
||||
@@ -406,6 +406,14 @@ def _build_build_subparser(subparsers, base_subparser):
|
||||
Store test results (failing rows) in the database
|
||||
'''
|
||||
)
|
||||
sub.add_argument(
|
||||
'--greedy',
|
||||
action='store_true',
|
||||
help='''
|
||||
Select all tests that touch the selected resources,
|
||||
even if they also depend on unselected resources
|
||||
'''
|
||||
)
|
||||
resource_values: List[str] = [
|
||||
str(s) for s in build_task.BuildTask.ALL_RESOURCE_VALUES
|
||||
] + ['all']
|
||||
@@ -637,7 +645,7 @@ def _add_table_mutability_arguments(*subparsers):
|
||||
'--full-refresh',
|
||||
action='store_true',
|
||||
help='''
|
||||
If specified, DBT will drop incremental models and
|
||||
If specified, dbt will drop incremental models and
|
||||
fully-recalculate the incremental table from the model definition.
|
||||
'''
|
||||
)
|
||||
@@ -753,6 +761,14 @@ def _build_test_subparser(subparsers, base_subparser):
|
||||
Store test results (failing rows) in the database
|
||||
'''
|
||||
)
|
||||
sub.add_argument(
|
||||
'--greedy',
|
||||
action='store_true',
|
||||
help='''
|
||||
Select all tests that touch the selected resources,
|
||||
even if they also depend on unselected resources
|
||||
'''
|
||||
)
|
||||
|
||||
sub.set_defaults(cls=test_task.TestTask, which='test', rpc_method='test')
|
||||
return sub
|
||||
@@ -878,6 +894,14 @@ def _build_list_subparser(subparsers, base_subparser):
|
||||
metavar='SELECTOR',
|
||||
required=False,
|
||||
)
|
||||
sub.add_argument(
|
||||
'--greedy',
|
||||
action='store_true',
|
||||
help='''
|
||||
Select all tests that touch the selected resources,
|
||||
even if they also depend on unselected resources
|
||||
'''
|
||||
)
|
||||
_add_common_selector_arguments(sub)
|
||||
|
||||
return sub
|
||||
|
||||
@@ -438,7 +438,7 @@ class GraphRunnableTask(ManifestTask):
|
||||
)
|
||||
|
||||
if len(self._flattened_nodes) == 0:
|
||||
logger.warning("WARNING: Nothing to do. Try checking your model "
|
||||
logger.warning("\nWARNING: Nothing to do. Try checking your model "
|
||||
"configs and model specification args")
|
||||
result = self.get_result(
|
||||
results=[],
|
||||
|
||||
@@ -96,5 +96,5 @@ def _get_dbt_plugins_info():
|
||||
yield plugin_name, mod.version
|
||||
|
||||
|
||||
__version__ = '0.21.0b2'
|
||||
__version__ = '0.21.0rc1'
|
||||
installed = get_installed_version()
|
||||
|
||||
@@ -284,12 +284,12 @@ def parse_args(argv=None):
|
||||
parser.add_argument('adapter')
|
||||
parser.add_argument('--title-case', '-t', default=None)
|
||||
parser.add_argument('--dependency', action='append')
|
||||
parser.add_argument('--dbt-core-version', default='0.21.0b2')
|
||||
parser.add_argument('--dbt-core-version', default='0.21.0rc1')
|
||||
parser.add_argument('--email')
|
||||
parser.add_argument('--author')
|
||||
parser.add_argument('--url')
|
||||
parser.add_argument('--sql', action='store_true')
|
||||
parser.add_argument('--package-version', default='0.21.0b2')
|
||||
parser.add_argument('--package-version', default='0.21.0rc1')
|
||||
parser.add_argument('--project-version', default='1.0')
|
||||
parser.add_argument(
|
||||
'--no-dependency', action='store_false', dest='set_dependency'
|
||||
|
||||
@@ -24,7 +24,7 @@ def read(fname):
|
||||
|
||||
|
||||
package_name = "dbt-core"
|
||||
package_version = "0.21.0b2"
|
||||
package_version = "0.21.0rc1"
|
||||
description = """dbt (data build tool) is a command line tool that helps \
|
||||
analysts and engineers transform data in their warehouse more effectively"""
|
||||
|
||||
|
||||
75
docker/requirements/requirements.0.21.0rc1.txt
Normal file
75
docker/requirements/requirements.0.21.0rc1.txt
Normal file
@@ -0,0 +1,75 @@
|
||||
agate==1.6.1
|
||||
asn1crypto==1.4.0
|
||||
attrs==21.2.0
|
||||
azure-common==1.1.27
|
||||
azure-core==1.18.0
|
||||
azure-storage-blob==12.8.1
|
||||
Babel==2.9.1
|
||||
boto3==1.18.44
|
||||
botocore==1.21.44
|
||||
cachetools==4.2.2
|
||||
certifi==2021.5.30
|
||||
cffi==1.14.6
|
||||
chardet==4.0.0
|
||||
charset-normalizer==2.0.6
|
||||
colorama==0.4.4
|
||||
cryptography==3.4.8
|
||||
google-api-core==1.31.2
|
||||
google-auth==1.35.0
|
||||
google-cloud-bigquery==2.26.0
|
||||
google-cloud-core==1.7.2
|
||||
google-crc32c==1.1.2
|
||||
google-resumable-media==2.0.2
|
||||
googleapis-common-protos==1.53.0
|
||||
grpcio==1.40.0
|
||||
hologram==0.0.14
|
||||
idna==3.2
|
||||
importlib-metadata==4.8.1
|
||||
isodate==0.6.0
|
||||
jeepney==0.7.1
|
||||
Jinja2==2.11.3
|
||||
jmespath==0.10.0
|
||||
json-rpc==1.13.0
|
||||
jsonschema==3.1.1
|
||||
keyring==21.8.0
|
||||
leather==0.3.3
|
||||
Logbook==1.5.3
|
||||
MarkupSafe==2.0.1
|
||||
mashumaro==2.5
|
||||
minimal-snowplow-tracker==0.0.2
|
||||
msgpack==1.0.2
|
||||
msrest==0.6.21
|
||||
networkx==2.6.3
|
||||
oauthlib==3.1.1
|
||||
oscrypto==1.2.1
|
||||
packaging==20.9
|
||||
parsedatetime==2.6
|
||||
proto-plus==1.19.0
|
||||
protobuf==3.18.0
|
||||
psycopg2-binary==2.9.1
|
||||
pyasn1==0.4.8
|
||||
pyasn1-modules==0.2.8
|
||||
pycparser==2.20
|
||||
pycryptodomex==3.10.1
|
||||
PyJWT==2.1.0
|
||||
pyOpenSSL==20.0.1
|
||||
pyparsing==2.4.7
|
||||
pyrsistent==0.18.0
|
||||
python-dateutil==2.8.2
|
||||
python-slugify==5.0.2
|
||||
pytimeparse==1.1.8
|
||||
pytz==2021.1
|
||||
PyYAML==5.4.1
|
||||
requests==2.26.0
|
||||
requests-oauthlib==1.3.0
|
||||
rsa==4.7.2
|
||||
s3transfer==0.5.0
|
||||
SecretStorage==3.3.1
|
||||
six==1.16.0
|
||||
snowflake-connector-python==2.5.1
|
||||
sqlparse==0.4.2
|
||||
text-unidecode==1.3
|
||||
typing-extensions==3.10.0.2
|
||||
urllib3==1.26.6
|
||||
Werkzeug==2.0.1
|
||||
zipp==3.5.0
|
||||
@@ -1 +1 @@
|
||||
version = '0.21.0b2'
|
||||
version = '0.21.0rc1'
|
||||
|
||||
@@ -20,7 +20,7 @@ except ImportError:
|
||||
|
||||
|
||||
package_name = "dbt-bigquery"
|
||||
package_version = "0.21.0b2"
|
||||
package_version = "0.21.0rc1"
|
||||
description = """The bigquery adapter plugin for dbt (data build tool)"""
|
||||
|
||||
this_directory = os.path.abspath(os.path.dirname(__file__))
|
||||
|
||||
@@ -1 +1 @@
|
||||
version = '0.21.0b2'
|
||||
version = '0.21.0rc1'
|
||||
|
||||
@@ -41,7 +41,7 @@ def _dbt_psycopg2_name():
|
||||
|
||||
|
||||
package_name = "dbt-postgres"
|
||||
package_version = "0.21.0b2"
|
||||
package_version = "0.21.0rc1"
|
||||
description = """The postgres adpter plugin for dbt (data build tool)"""
|
||||
|
||||
this_directory = os.path.abspath(os.path.dirname(__file__))
|
||||
|
||||
@@ -1 +1 @@
|
||||
version = '0.21.0b2'
|
||||
version = '0.21.0rc1'
|
||||
|
||||
@@ -20,7 +20,7 @@ except ImportError:
|
||||
|
||||
|
||||
package_name = "dbt-redshift"
|
||||
package_version = "0.21.0b2"
|
||||
package_version = "0.21.0rc1"
|
||||
description = """The redshift adapter plugin for dbt (data build tool)"""
|
||||
|
||||
this_directory = os.path.abspath(os.path.dirname(__file__))
|
||||
|
||||
@@ -1 +1 @@
|
||||
version = '0.21.0b2'
|
||||
version = '0.21.0rc1'
|
||||
|
||||
@@ -20,7 +20,7 @@ except ImportError:
|
||||
|
||||
|
||||
package_name = "dbt-snowflake"
|
||||
package_version = "0.21.0b2"
|
||||
package_version = "0.21.0rc1"
|
||||
description = """The snowflake adapter plugin for dbt (data build tool)"""
|
||||
|
||||
this_directory = os.path.abspath(os.path.dirname(__file__))
|
||||
|
||||
2
setup.py
2
setup.py
@@ -24,7 +24,7 @@ with open(os.path.join(this_directory, 'README.md')) as f:
|
||||
|
||||
|
||||
package_name = "dbt"
|
||||
package_version = "0.21.0b2"
|
||||
package_version = "0.21.0rc1"
|
||||
description = """With dbt, data analysts and engineers can build analytics \
|
||||
the way engineers build applications."""
|
||||
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
from test.integration.base import DBTIntegrationTest, FakeArgs, use_profile
|
||||
import yaml
|
||||
|
||||
from dbt.task.test import TestTask
|
||||
from dbt.task.list import ListTask
|
||||
@@ -20,12 +21,18 @@ class TestSelectionExpansion(DBTIntegrationTest):
|
||||
"test-paths": ["tests"]
|
||||
}
|
||||
|
||||
def list_tests_and_assert(self, include, exclude, expected_tests):
|
||||
def list_tests_and_assert(self, include, exclude, expected_tests, greedy=False, selector_name=None):
|
||||
list_args = [ 'ls', '--resource-type', 'test']
|
||||
if include:
|
||||
list_args.extend(('--select', include))
|
||||
if exclude:
|
||||
list_args.extend(('--exclude', exclude))
|
||||
if exclude:
|
||||
list_args.extend(('--exclude', exclude))
|
||||
if greedy:
|
||||
list_args.append('--greedy')
|
||||
if selector_name:
|
||||
list_args.extend(('--selector', selector_name))
|
||||
|
||||
listed = self.run_dbt(list_args)
|
||||
assert len(listed) == len(expected_tests)
|
||||
@@ -34,7 +41,7 @@ class TestSelectionExpansion(DBTIntegrationTest):
|
||||
assert sorted(test_names) == sorted(expected_tests)
|
||||
|
||||
def run_tests_and_assert(
|
||||
self, include, exclude, expected_tests, schema = False, data = False
|
||||
self, include, exclude, expected_tests, schema=False, data=False, greedy=False, selector_name=None
|
||||
):
|
||||
results = self.run_dbt(['run'])
|
||||
self.assertEqual(len(results), 2)
|
||||
@@ -48,6 +55,10 @@ class TestSelectionExpansion(DBTIntegrationTest):
|
||||
test_args.append('--schema')
|
||||
if data:
|
||||
test_args.append('--data')
|
||||
if greedy:
|
||||
test_args.append('--greedy')
|
||||
if selector_name:
|
||||
test_args.extend(('--selector', selector_name))
|
||||
|
||||
results = self.run_dbt(test_args)
|
||||
tests_run = [r.node.name for r in results]
|
||||
@@ -228,3 +239,80 @@ class TestSelectionExpansion(DBTIntegrationTest):
|
||||
|
||||
self.list_tests_and_assert(select, exclude, expected)
|
||||
self.run_tests_and_assert(select, exclude, expected)
|
||||
|
||||
@use_profile('postgres')
|
||||
def test__postgres__model_a_greedy(self):
|
||||
select = 'model_a'
|
||||
exclude = None
|
||||
greedy = True
|
||||
expected = [
|
||||
'cf_a_b', 'cf_a_src', 'just_a',
|
||||
'relationships_model_a_fun__fun__ref_model_b_',
|
||||
'relationships_model_a_fun__fun__source_my_src_my_tbl_',
|
||||
'unique_model_a_fun'
|
||||
]
|
||||
|
||||
self.list_tests_and_assert(select, exclude, expected, greedy)
|
||||
self.run_tests_and_assert(select, exclude, expected, greedy=greedy)
|
||||
|
||||
@use_profile('postgres')
|
||||
def test__postgres__model_a_greedy_exclude_unique_tests(self):
|
||||
select = 'model_a'
|
||||
exclude = 'test_name:unique'
|
||||
greedy = True
|
||||
expected = [
|
||||
'cf_a_b', 'cf_a_src', 'just_a',
|
||||
'relationships_model_a_fun__fun__ref_model_b_',
|
||||
'relationships_model_a_fun__fun__source_my_src_my_tbl_',
|
||||
]
|
||||
|
||||
self.list_tests_and_assert(select, exclude, expected, greedy)
|
||||
self.run_tests_and_assert(select, exclude, expected, greedy=greedy)
|
||||
|
||||
class TestExpansionWithSelectors(TestSelectionExpansion):
|
||||
|
||||
@property
|
||||
def selectors_config(self):
|
||||
return yaml.safe_load('''
|
||||
selectors:
|
||||
- name: model_a_greedy_none
|
||||
definition:
|
||||
method: fqn
|
||||
value: model_a
|
||||
- name: model_a_greedy_false
|
||||
definition:
|
||||
method: fqn
|
||||
value: model_a
|
||||
greedy: false
|
||||
- name: model_a_greedy_true
|
||||
definition:
|
||||
method: fqn
|
||||
value: model_a
|
||||
greedy: true
|
||||
''')
|
||||
|
||||
@use_profile('postgres')
|
||||
def test__postgres__selector_model_a_not_greedy(self):
|
||||
expected = ['just_a','unique_model_a_fun']
|
||||
|
||||
# when greedy is not specified, so implicitly False
|
||||
self.list_tests_and_assert(include=None, exclude=None, expected_tests=expected, selector_name='model_a_greedy_none')
|
||||
self.run_tests_and_assert(include=None, exclude=None, expected_tests=expected, selector_name='model_a_greedy_none')
|
||||
|
||||
# when greedy is explicitly False
|
||||
self.list_tests_and_assert(include=None, exclude=None, expected_tests=expected, selector_name='model_a_greedy_false')
|
||||
self.run_tests_and_assert(include=None, exclude=None, expected_tests=expected, selector_name='model_a_greedy_false')
|
||||
|
||||
|
||||
@use_profile('postgres')
|
||||
def test__postgres__selector_model_a_yes_greedy(self):
|
||||
expected = [
|
||||
'cf_a_b', 'cf_a_src', 'just_a',
|
||||
'relationships_model_a_fun__fun__ref_model_b_',
|
||||
'relationships_model_a_fun__fun__source_my_src_my_tbl_',
|
||||
'unique_model_a_fun'
|
||||
]
|
||||
|
||||
# when greedy is explicitly False
|
||||
self.list_tests_and_assert(include=None, exclude=None, expected_tests=expected, selector_name='model_a_greedy_true')
|
||||
self.run_tests_and_assert(include=None, exclude=None, expected_tests=expected, selector_name='model_a_greedy_true')
|
||||
|
||||
@@ -120,7 +120,7 @@ def test_run_specs(include, exclude, expected):
|
||||
manifest = _get_manifest(graph)
|
||||
selector = graph_selector.NodeSelector(graph, manifest)
|
||||
spec = graph_cli.parse_difference(include, exclude)
|
||||
selected = selector.select_nodes(spec)
|
||||
selected, _ = selector.select_nodes(spec)
|
||||
|
||||
assert selected == expected
|
||||
|
||||
|
||||
Reference in New Issue
Block a user