mirror of
https://github.com/dbt-labs/dbt-core
synced 2025-12-17 19:31:34 +00:00
Compare commits
1 Commits
v1.5.1
...
feature/re
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
2d69a9b114 |
@@ -169,36 +169,6 @@ class Compiler:
|
||||
relation_cls = adapter.Relation
|
||||
return relation_cls.add_ephemeral_prefix(name)
|
||||
|
||||
def _get_compiled_model(
|
||||
self,
|
||||
manifest: Manifest,
|
||||
cte_id: str,
|
||||
extra_context: Dict[str, Any],
|
||||
) -> NonSourceCompiledNode:
|
||||
|
||||
if cte_id not in manifest.nodes:
|
||||
raise InternalException(
|
||||
f'During compilation, found a cte reference that could not be '
|
||||
f'resolved: {cte_id}'
|
||||
)
|
||||
cte_model = manifest.nodes[cte_id]
|
||||
if getattr(cte_model, 'compiled', False):
|
||||
assert isinstance(cte_model, tuple(COMPILED_TYPES.values()))
|
||||
return cast(NonSourceCompiledNode, cte_model)
|
||||
elif cte_model.is_ephemeral_model:
|
||||
# this must be some kind of parsed node that we can compile.
|
||||
# we know it's not a parsed source definition
|
||||
assert isinstance(cte_model, tuple(COMPILED_TYPES))
|
||||
# update the node so
|
||||
node = self.compile_node(cte_model, manifest, extra_context)
|
||||
manifest.sync_update_node(node)
|
||||
return node
|
||||
else:
|
||||
raise InternalException(
|
||||
f'During compilation, found an uncompiled cte that '
|
||||
f'was not an ephemeral model: {cte_id}'
|
||||
)
|
||||
|
||||
def _inject_ctes_into_sql(self, sql: str, ctes: List[InjectedCTE]) -> str:
|
||||
"""
|
||||
`ctes` is a list of InjectedCTEs like:
|
||||
@@ -260,26 +230,6 @@ class Compiler:
|
||||
|
||||
return str(parsed)
|
||||
|
||||
def _model_prepend_ctes(
|
||||
self,
|
||||
model: NonSourceCompiledNode,
|
||||
prepended_ctes: List[InjectedCTE]
|
||||
) -> NonSourceCompiledNode:
|
||||
if model.compiled_sql is None:
|
||||
raise RuntimeException(
|
||||
'Cannot prepend ctes to an unparsed node', model
|
||||
)
|
||||
injected_sql = self._inject_ctes_into_sql(
|
||||
model.compiled_sql,
|
||||
prepended_ctes,
|
||||
)
|
||||
|
||||
model.extra_ctes_injected = True
|
||||
model.extra_ctes = prepended_ctes
|
||||
model.injected_sql = injected_sql
|
||||
model.validate(model.to_dict())
|
||||
return model
|
||||
|
||||
def _get_dbt_test_name(self) -> str:
|
||||
return 'dbt__CTE__INTERNAL_test'
|
||||
|
||||
@@ -287,7 +237,7 @@ class Compiler:
|
||||
self,
|
||||
model: NonSourceCompiledNode,
|
||||
manifest: Manifest,
|
||||
extra_context: Dict[str, Any],
|
||||
extra_context: Optional[Dict[str, Any]],
|
||||
) -> Tuple[NonSourceCompiledNode, List[InjectedCTE]]:
|
||||
if model.extra_ctes_injected:
|
||||
return (model, model.extra_ctes)
|
||||
@@ -303,41 +253,73 @@ class Compiler:
|
||||
dbt_test_name = self._get_dbt_test_name()
|
||||
|
||||
for cte in model.extra_ctes:
|
||||
new_prep_ctes: List[InjectedCTE]
|
||||
orig_compiled_sql: Optional[str]
|
||||
if cte.id == dbt_test_name:
|
||||
sql = cte.sql
|
||||
orig_compiled_sql = sql
|
||||
else:
|
||||
cte_model = self._get_compiled_model(
|
||||
manifest,
|
||||
cte.id,
|
||||
extra_context,
|
||||
)
|
||||
cte_model, new_prepended_ctes = self._recursively_prepend_ctes(
|
||||
cte_model, manifest, extra_context
|
||||
)
|
||||
_extend_prepended_ctes(prepended_ctes, new_prepended_ctes)
|
||||
cte_model = manifest.nodes[cte.id]
|
||||
if getattr(cte_model, 'compiled', False):
|
||||
assert isinstance(cte_model, tuple(
|
||||
COMPILED_TYPES.values()))
|
||||
cte_model = cast(NonSourceCompiledNode, cte_model)
|
||||
orig_compiled_sql = cte_model.compiled_sql
|
||||
cte_model, new_prep_ctes = self._recursively_prepend_ctes(
|
||||
cte_model, manifest, extra_context
|
||||
)
|
||||
elif cte_model.is_ephemeral_model:
|
||||
assert isinstance(cte_model, tuple(COMPILED_TYPES))
|
||||
compiled_node = self._compile_node(
|
||||
cte_model, manifest, extra_context)
|
||||
|
||||
orig_compiled_sql = compiled_node.compiled_sql
|
||||
cte_model, new_prep_ctes = self._recursively_prepend_ctes(
|
||||
compiled_node, manifest, extra_context
|
||||
)
|
||||
manifest.sync_update_node(cte_model)
|
||||
else:
|
||||
raise InternalException(
|
||||
f'During compilation, found an uncompiled cte that '
|
||||
f'was not an ephemeral model: {cte.id}'
|
||||
)
|
||||
|
||||
_extend_prepended_ctes(prepended_ctes, new_prep_ctes)
|
||||
|
||||
new_cte_name = self.add_ephemeral_prefix(cte_model.name)
|
||||
sql = f' {new_cte_name} as (\n{cte_model.compiled_sql}\n)'
|
||||
sql = f' {new_cte_name} as (\n{orig_compiled_sql}\n)'
|
||||
|
||||
_add_prepended_cte(prepended_ctes, InjectedCTE(id=cte.id, sql=sql))
|
||||
|
||||
model = self._model_prepend_ctes(model, prepended_ctes)
|
||||
if model.compiled_sql is None:
|
||||
raise RuntimeException(
|
||||
'Cannot inject ctes into an unparsed node', model
|
||||
)
|
||||
injected_sql = self._inject_ctes_into_sql(
|
||||
model.compiled_sql,
|
||||
prepended_ctes,
|
||||
)
|
||||
model.extra_ctes_injected = True
|
||||
model.extra_ctes = prepended_ctes
|
||||
model.compiled_sql = injected_sql
|
||||
model.validate(model.to_dict())
|
||||
|
||||
manifest.update_node(model)
|
||||
|
||||
return model, prepended_ctes
|
||||
|
||||
def _insert_ctes(
|
||||
def _add_extra_ctes(
|
||||
self,
|
||||
compiled_node: NonSourceCompiledNode,
|
||||
manifest: Manifest,
|
||||
extra_context: Dict[str, Any],
|
||||
) -> NonSourceCompiledNode:
|
||||
"""Insert the CTEs for the model."""
|
||||
):
|
||||
"""Add extra CTEs for the model."""
|
||||
|
||||
# for data tests, we need to insert a special CTE at the end of the
|
||||
# list containing the test query, and then have the "real" query be a
|
||||
# select count(*) from that model.
|
||||
# the benefit of doing it this way is that _insert_ctes() can be
|
||||
# the benefit of doing it this way is that _add_extra_ctes() can be
|
||||
# rewritten for different adapters to handle databses that don't
|
||||
# support CTEs, or at least don't have full support.
|
||||
if isinstance(compiled_node, CompiledDataTestNode):
|
||||
@@ -352,11 +334,7 @@ class Compiler:
|
||||
compiled_node.extra_ctes.append(cte)
|
||||
compiled_node.compiled_sql = f'\nselect count(*) from {name}'
|
||||
|
||||
injected_node, _ = self._recursively_prepend_ctes(
|
||||
compiled_node, manifest, extra_context
|
||||
)
|
||||
return injected_node
|
||||
|
||||
# this is called by both 'compile_node' and '_recursively_prepend_ctes'
|
||||
def _compile_node(
|
||||
self,
|
||||
node: ManifestNode,
|
||||
@@ -374,7 +352,6 @@ class Compiler:
|
||||
'compiled_sql': None,
|
||||
'extra_ctes_injected': False,
|
||||
'extra_ctes': [],
|
||||
'injected_sql': None,
|
||||
})
|
||||
compiled_node = _compiled_type_for(node).from_dict(data)
|
||||
|
||||
@@ -390,11 +367,13 @@ class Compiler:
|
||||
|
||||
compiled_node.compiled = True
|
||||
|
||||
injected_node = self._insert_ctes(
|
||||
# add extra ctes, such as for a test node or possibly
|
||||
# adapter specific ctes
|
||||
self._add_extra_ctes(
|
||||
compiled_node, manifest, extra_context
|
||||
)
|
||||
|
||||
return injected_node
|
||||
return compiled_node
|
||||
|
||||
def write_graph_file(self, linker: Linker, manifest: Manifest):
|
||||
filename = graph_file_name
|
||||
@@ -454,18 +433,16 @@ class Compiler:
|
||||
return node
|
||||
logger.debug(f'Writing injected SQL for node "{node.unique_id}"')
|
||||
|
||||
if node.injected_sql is None:
|
||||
# this should not really happen, but it'd be a shame to crash
|
||||
# over it
|
||||
if node.compiled_sql is None:
|
||||
logger.error(
|
||||
f'Compiled node "{node.unique_id}" had no injected_sql, '
|
||||
f'Compiled node "{node.unique_id}" had no compiled_sql, '
|
||||
'cannot write sql!'
|
||||
)
|
||||
else:
|
||||
node.build_path = node.write_node(
|
||||
self.config.target_path,
|
||||
'compiled',
|
||||
node.injected_sql
|
||||
node.compiled_sql
|
||||
)
|
||||
return node
|
||||
|
||||
@@ -476,15 +453,19 @@ class Compiler:
|
||||
extra_context: Optional[Dict[str, Any]] = None,
|
||||
write: bool = True,
|
||||
) -> NonSourceCompiledNode:
|
||||
node = self._compile_node(node, manifest, extra_context)
|
||||
compiled_node = self._compile_node(node, manifest, extra_context)
|
||||
|
||||
if write and _is_writable(node):
|
||||
self._write_node(node)
|
||||
return node
|
||||
injected_node, _ = self._recursively_prepend_ctes(
|
||||
compiled_node, manifest, extra_context
|
||||
)
|
||||
|
||||
if write and _is_writable(injected_node):
|
||||
self._write_node(injected_node)
|
||||
return injected_node
|
||||
|
||||
|
||||
def _is_writable(node):
|
||||
if not node.injected_sql:
|
||||
if not node.compiled_sql:
|
||||
return False
|
||||
|
||||
if node.resource_type == NodeType.Snapshot:
|
||||
|
||||
@@ -1217,7 +1217,7 @@ class ModelContext(ProviderContext):
|
||||
|
||||
@contextproperty
|
||||
def sql(self) -> Optional[str]:
|
||||
return getattr(self.model, 'injected_sql', None)
|
||||
return getattr(self.model, 'compiled_sql', None)
|
||||
|
||||
@contextproperty
|
||||
def database(self) -> str:
|
||||
|
||||
@@ -42,7 +42,6 @@ class CompiledNode(ParsedNode, CompiledNodeMixin):
|
||||
compiled_sql: Optional[str] = None
|
||||
extra_ctes_injected: bool = False
|
||||
extra_ctes: List[InjectedCTE] = field(default_factory=list)
|
||||
injected_sql: Optional[str] = None
|
||||
|
||||
def set_cte(self, cte_id: str, sql: str):
|
||||
"""This is the equivalent of what self.extra_ctes[cte_id] = sql would
|
||||
|
||||
@@ -132,7 +132,7 @@ class RuntimeException(RuntimeError, Exception):
|
||||
result.update({
|
||||
'raw_sql': self.node.raw_sql,
|
||||
# the node isn't always compiled, but if it is, include that!
|
||||
'compiled_sql': getattr(self.node, 'injected_sql', None),
|
||||
'compiled_sql': getattr(self.node, 'compiled_sql', None),
|
||||
})
|
||||
return result
|
||||
|
||||
|
||||
@@ -214,7 +214,7 @@
|
||||
|
||||
{% if not target_relation_exists %}
|
||||
|
||||
{% set build_sql = build_snapshot_table(strategy, model['injected_sql']) %}
|
||||
{% set build_sql = build_snapshot_table(strategy, model['compiled_sql']) %}
|
||||
{% set final_sql = create_table_as(False, target_relation, build_sql) %}
|
||||
|
||||
{% else %}
|
||||
|
||||
@@ -106,7 +106,7 @@
|
||||
|
||||
|
||||
{% macro snapshot_check_all_get_existing_columns(node, target_exists) -%}
|
||||
{%- set query_columns = get_columns_in_query(node['injected_sql']) -%}
|
||||
{%- set query_columns = get_columns_in_query(node['compiled_sql']) -%}
|
||||
{%- if not target_exists -%}
|
||||
{# no table yet -> return whatever the query does #}
|
||||
{{ return([false, query_columns]) }}
|
||||
|
||||
@@ -65,7 +65,7 @@ class RPCCompileRunner(GenericRPCRunner[RemoteCompileResult]):
|
||||
def execute(self, compiled_node, manifest) -> RemoteCompileResult:
|
||||
return RemoteCompileResult(
|
||||
raw_sql=compiled_node.raw_sql,
|
||||
compiled_sql=compiled_node.injected_sql,
|
||||
compiled_sql=compiled_node.compiled_sql,
|
||||
node=compiled_node,
|
||||
timing=[], # this will get added later
|
||||
logs=[],
|
||||
@@ -88,7 +88,7 @@ class RPCCompileRunner(GenericRPCRunner[RemoteCompileResult]):
|
||||
class RPCExecuteRunner(GenericRPCRunner[RemoteRunResult]):
|
||||
def execute(self, compiled_node, manifest) -> RemoteRunResult:
|
||||
_, execute_result = self.adapter.execute(
|
||||
compiled_node.injected_sql, fetch=True
|
||||
compiled_node.compiled_sql, fetch=True
|
||||
)
|
||||
|
||||
table = ResultTable(
|
||||
@@ -98,7 +98,7 @@ class RPCExecuteRunner(GenericRPCRunner[RemoteRunResult]):
|
||||
|
||||
return RemoteRunResult(
|
||||
raw_sql=compiled_node.raw_sql,
|
||||
compiled_sql=compiled_node.injected_sql,
|
||||
compiled_sql=compiled_node.compiled_sql,
|
||||
node=compiled_node,
|
||||
table=table,
|
||||
timing=[],
|
||||
|
||||
@@ -255,7 +255,7 @@ class RunTask(CompileTask):
|
||||
def get_hook_sql(self, adapter, hook, idx, num_hooks, extra_context):
|
||||
compiler = adapter.get_compiler()
|
||||
compiled = compiler.compile_node(hook, self.manifest, extra_context)
|
||||
statement = compiled.injected_sql
|
||||
statement = compiled.compiled_sql
|
||||
hook_index = hook.index or num_hooks
|
||||
hook_obj = get_hook(statement, index=hook_index)
|
||||
return hook_obj.sql or ''
|
||||
|
||||
@@ -42,7 +42,7 @@ class TestRunner(CompileRunner):
|
||||
|
||||
def execute_data_test(self, test: CompiledDataTestNode):
|
||||
res, table = self.adapter.execute(
|
||||
test.injected_sql, auto_begin=True, fetch=True
|
||||
test.compiled_sql, auto_begin=True, fetch=True
|
||||
)
|
||||
|
||||
num_rows = len(table.rows)
|
||||
@@ -59,7 +59,7 @@ class TestRunner(CompileRunner):
|
||||
|
||||
def execute_schema_test(self, test: CompiledSchemaTestNode):
|
||||
res, table = self.adapter.execute(
|
||||
test.injected_sql,
|
||||
test.compiled_sql,
|
||||
auto_begin=True,
|
||||
fetch=True,
|
||||
)
|
||||
|
||||
@@ -385,7 +385,7 @@ class BigQueryAdapter(BaseAdapter):
|
||||
model_database = model.get('database')
|
||||
model_schema = model.get('schema')
|
||||
model_alias = model.get('alias')
|
||||
model_sql = model.get('injected_sql')
|
||||
model_sql = model.get('compiled_sql')
|
||||
|
||||
logger.debug("Model SQL ({}):\n{}".format(model_alias, model_sql))
|
||||
self.connections.create_view(
|
||||
@@ -505,7 +505,7 @@ class BigQueryAdapter(BaseAdapter):
|
||||
decorator=None):
|
||||
|
||||
if sql_override is None:
|
||||
sql_override = model.get('injected_sql')
|
||||
sql_override = model.get('compiled_sql')
|
||||
|
||||
if flags.STRICT_MODE:
|
||||
connection = self.connections.get_thread_connection()
|
||||
|
||||
@@ -11,7 +11,7 @@
|
||||
{{ log(table_start_time ~ ' | -> Running for day ' ~ date, info=True) }}
|
||||
{% endif %}
|
||||
|
||||
{% set fixed_sql = model['injected_sql'] | replace('[DBT__PARTITION_DATE]', date) %}
|
||||
{% set fixed_sql = model['compiled_sql'] | replace('[DBT__PARTITION_DATE]', date) %}
|
||||
{% set _ = adapter.execute_model(model, 'table', fixed_sql, decorator=date) %}
|
||||
{% endfor %}
|
||||
|
||||
|
||||
@@ -130,7 +130,7 @@ class TestCLIInvocationWithProfilesDir(ModelCopyingIntegrationTest):
|
||||
|
||||
# make sure the test runs against `custom_schema`
|
||||
for test_result in res:
|
||||
self.assertTrue(self.custom_schema, test_result.node.injected_sql)
|
||||
self.assertTrue(self.custom_schema, test_result.node.compiled_sql)
|
||||
|
||||
|
||||
class TestCLIInvocationWithProjectDir(ModelCopyingIntegrationTest):
|
||||
|
||||
@@ -130,17 +130,6 @@ class TestDocsGenerate(DBTIntegrationTest):
|
||||
def models(self):
|
||||
return self.dir("models")
|
||||
|
||||
@property
|
||||
def packages_config(self):
|
||||
return {
|
||||
'packages': [
|
||||
{
|
||||
'git': 'https://github.com/fishtown-analytics/dbt-integration-project',
|
||||
'revision': 'dbt/0.17.0',
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
@property
|
||||
def project_config(self):
|
||||
return {
|
||||
@@ -1114,7 +1103,6 @@ class TestDocsGenerate(DBTIntegrationTest):
|
||||
'compiled_sql': ANY,
|
||||
'extra_ctes_injected': True,
|
||||
'extra_ctes': [],
|
||||
'injected_sql': ANY,
|
||||
'checksum': self._checksum_file(model_sql_path),
|
||||
'unrendered_config': unrendered_model_config,
|
||||
},
|
||||
@@ -1188,7 +1176,6 @@ class TestDocsGenerate(DBTIntegrationTest):
|
||||
'compiled_sql': ANY,
|
||||
'extra_ctes_injected': True,
|
||||
'extra_ctes': [],
|
||||
'injected_sql': ANY,
|
||||
'checksum': self._checksum_file(second_model_sql_path),
|
||||
'unrendered_config': unrendered_second_config
|
||||
},
|
||||
@@ -1264,7 +1251,6 @@ class TestDocsGenerate(DBTIntegrationTest):
|
||||
'compiled_sql': '',
|
||||
'extra_ctes_injected': True,
|
||||
'extra_ctes': [],
|
||||
'injected_sql': '',
|
||||
'checksum': self._checksum_file(seed_path),
|
||||
'unrendered_config': unrendered_seed_config,
|
||||
},
|
||||
@@ -1301,7 +1287,6 @@ class TestDocsGenerate(DBTIntegrationTest):
|
||||
'compiled_sql': AnyStringWith('count(*)'),
|
||||
'extra_ctes_injected': True,
|
||||
'extra_ctes': [],
|
||||
'injected_sql': AnyStringWith('count(*)'),
|
||||
'test_metadata': {
|
||||
'namespace': None,
|
||||
'name': 'not_null',
|
||||
@@ -1346,7 +1331,6 @@ class TestDocsGenerate(DBTIntegrationTest):
|
||||
'compiled_sql': AnyStringWith('select 0'),
|
||||
'extra_ctes_injected': True,
|
||||
'extra_ctes': [],
|
||||
'injected_sql': AnyStringWith('select 0'),
|
||||
'test_metadata': {
|
||||
'namespace': 'test',
|
||||
'name': 'nothing',
|
||||
@@ -1390,7 +1374,6 @@ class TestDocsGenerate(DBTIntegrationTest):
|
||||
'compiled_sql': AnyStringWith('count(*)'),
|
||||
'extra_ctes_injected': True,
|
||||
'extra_ctes': [],
|
||||
'injected_sql': AnyStringWith('count(*)'),
|
||||
'test_metadata': {
|
||||
'namespace': None,
|
||||
'name': 'unique',
|
||||
@@ -1587,7 +1570,6 @@ class TestDocsGenerate(DBTIntegrationTest):
|
||||
'compiled_sql': ANY,
|
||||
'extra_ctes_injected': True,
|
||||
'extra_ctes': [],
|
||||
'injected_sql': ANY,
|
||||
'checksum': self._checksum_file(ephemeral_copy_path),
|
||||
'unrendered_config': self.unrendered_model_config(materialized='ephemeral'),
|
||||
},
|
||||
@@ -1645,7 +1627,6 @@ class TestDocsGenerate(DBTIntegrationTest):
|
||||
'compiled_sql': ANY,
|
||||
'extra_ctes_injected': True,
|
||||
'extra_ctes': [ANY],
|
||||
'injected_sql': ANY,
|
||||
'checksum': self._checksum_file(ephemeral_summary_path),
|
||||
'unrendered_config': self.unrendered_model_config(materialized='table'),
|
||||
},
|
||||
@@ -1702,7 +1683,6 @@ class TestDocsGenerate(DBTIntegrationTest):
|
||||
'compiled_sql': ANY,
|
||||
'extra_ctes_injected': True,
|
||||
'extra_ctes': [],
|
||||
'injected_sql': ANY,
|
||||
'checksum': self._checksum_file(view_summary_path),
|
||||
'unrendered_config': self.unrendered_model_config(materialized='view'),
|
||||
},
|
||||
@@ -1776,7 +1756,6 @@ class TestDocsGenerate(DBTIntegrationTest):
|
||||
'compiled_sql': '',
|
||||
'extra_ctes_injected': True,
|
||||
'extra_ctes': [],
|
||||
'injected_sql': '',
|
||||
'checksum': self._checksum_file(seed_path),
|
||||
'unrendered_config': self.unrendered_seed_config(),
|
||||
},
|
||||
@@ -2047,7 +2026,6 @@ class TestDocsGenerate(DBTIntegrationTest):
|
||||
'compiled_sql': ANY,
|
||||
'extra_ctes_injected': True,
|
||||
'extra_ctes': [],
|
||||
'injected_sql': ANY,
|
||||
'checksum': self._checksum_file(clustered_sql_path),
|
||||
'unrendered_config': self.unrendered_model_config(
|
||||
cluster_by=['first_name'],
|
||||
@@ -2129,7 +2107,6 @@ class TestDocsGenerate(DBTIntegrationTest):
|
||||
'compiled_sql': ANY,
|
||||
'extra_ctes_injected': True,
|
||||
'extra_ctes': [],
|
||||
'injected_sql': ANY,
|
||||
'checksum': self._checksum_file(multi_clustered_sql_path),
|
||||
'unrendered_config': self.unrendered_model_config(
|
||||
cluster_by=['first_name', 'email'],
|
||||
@@ -2210,7 +2187,6 @@ class TestDocsGenerate(DBTIntegrationTest):
|
||||
'compiled_sql': ANY,
|
||||
'extra_ctes_injected': True,
|
||||
'extra_ctes': [],
|
||||
'injected_sql': ANY,
|
||||
'checksum': self._checksum_file(nested_view_sql_path),
|
||||
'unrendered_config': self.unrendered_model_config(materialized='view'),
|
||||
},
|
||||
@@ -2246,7 +2222,6 @@ class TestDocsGenerate(DBTIntegrationTest):
|
||||
'compiled_sql': ANY,
|
||||
'extra_ctes_injected': True,
|
||||
'extra_ctes': [],
|
||||
'injected_sql': ANY,
|
||||
'checksum': self._checksum_file(nested_table_sql_path),
|
||||
'unrendered_config': self.unrendered_model_config(materialized='table'),
|
||||
},
|
||||
@@ -2323,7 +2298,6 @@ class TestDocsGenerate(DBTIntegrationTest):
|
||||
'compiled_sql': '',
|
||||
'extra_ctes_injected': True,
|
||||
'extra_ctes': [],
|
||||
'injected_sql': '',
|
||||
'checksum': self._checksum_file(seed_path),
|
||||
'unrendered_config': self.unrendered_seed_config(),
|
||||
},
|
||||
@@ -2459,7 +2433,6 @@ class TestDocsGenerate(DBTIntegrationTest):
|
||||
'compiled_sql': ANY,
|
||||
'extra_ctes_injected': True,
|
||||
'extra_ctes': [],
|
||||
'injected_sql': ANY,
|
||||
'checksum': self._checksum_file(model_sql_path),
|
||||
'unrendered_config': self.unrendered_model_config(bind=False, materialized='view'),
|
||||
},
|
||||
@@ -2536,7 +2509,6 @@ class TestDocsGenerate(DBTIntegrationTest):
|
||||
'compiled_sql': ANY,
|
||||
'extra_ctes_injected': True,
|
||||
'extra_ctes': [],
|
||||
'injected_sql': ANY,
|
||||
'checksum': self._checksum_file(seed_path),
|
||||
'unrendered_config': self.unrendered_seed_config(),
|
||||
},
|
||||
@@ -2710,7 +2682,6 @@ class TestDocsGenerate(DBTIntegrationTest):
|
||||
'extra_ctes': [],
|
||||
'extra_ctes_injected': True,
|
||||
'fqn': ['test', 'model'],
|
||||
'injected_sql': compiled_sql,
|
||||
'meta': {},
|
||||
'name': 'model',
|
||||
'original_file_path': model_sql_path,
|
||||
@@ -2799,7 +2770,6 @@ class TestDocsGenerate(DBTIntegrationTest):
|
||||
'extra_ctes': [],
|
||||
'extra_ctes_injected': True,
|
||||
'fqn': ['test', 'second_model'],
|
||||
'injected_sql': compiled_sql,
|
||||
'meta': {},
|
||||
'name': 'second_model',
|
||||
'original_file_path': second_model_sql_path,
|
||||
@@ -2883,7 +2853,6 @@ class TestDocsGenerate(DBTIntegrationTest):
|
||||
'extra_ctes': [],
|
||||
'extra_ctes_injected': True,
|
||||
'fqn': ['test', 'seed'],
|
||||
'injected_sql': '',
|
||||
'meta': {},
|
||||
'name': 'seed',
|
||||
'original_file_path': seed_path,
|
||||
@@ -2930,7 +2899,6 @@ class TestDocsGenerate(DBTIntegrationTest):
|
||||
'extra_ctes': [],
|
||||
'extra_ctes_injected': True,
|
||||
'fqn': ['test', 'schema_test', 'not_null_model_id'],
|
||||
'injected_sql': AnyStringWith('id is null'),
|
||||
'meta': {},
|
||||
'name': 'not_null_model_id',
|
||||
'original_file_path': model_schema_yml_path,
|
||||
@@ -2985,7 +2953,6 @@ class TestDocsGenerate(DBTIntegrationTest):
|
||||
'extra_ctes': [],
|
||||
'extra_ctes_injected': True,
|
||||
'fqn': ['test', 'schema_test', 'test_nothing_model_'],
|
||||
'injected_sql': AnyStringWith('select 0'),
|
||||
'meta': {},
|
||||
'name': 'test_nothing_model_',
|
||||
'original_file_path': model_schema_yml_path,
|
||||
@@ -3039,7 +3006,6 @@ class TestDocsGenerate(DBTIntegrationTest):
|
||||
'extra_ctes': [],
|
||||
'extra_ctes_injected': True,
|
||||
'fqn': ['test', 'schema_test', 'unique_model_id'],
|
||||
'injected_sql': AnyStringWith('count(*)'),
|
||||
'meta': {},
|
||||
'name': 'unique_model_id',
|
||||
'original_file_path': model_schema_yml_path,
|
||||
@@ -3129,7 +3095,7 @@ class TestDocsGenerate(DBTIntegrationTest):
|
||||
},
|
||||
},
|
||||
'compiled': True,
|
||||
'compiled_sql': ephemeral_compiled_sql,
|
||||
'compiled_sql': ephemeral_injected_sql,
|
||||
'config': self.rendered_model_config(materialized='table'),
|
||||
'sources': [],
|
||||
'depends_on': {
|
||||
@@ -3146,7 +3112,6 @@ class TestDocsGenerate(DBTIntegrationTest):
|
||||
],
|
||||
'extra_ctes_injected': True,
|
||||
'fqn': ['test', 'ephemeral_summary'],
|
||||
'injected_sql': ephemeral_injected_sql,
|
||||
'meta': {},
|
||||
'name': 'ephemeral_summary',
|
||||
'original_file_path': ephemeral_summary_path,
|
||||
@@ -3220,7 +3185,6 @@ class TestDocsGenerate(DBTIntegrationTest):
|
||||
'extra_ctes': [],
|
||||
'extra_ctes_injected': True,
|
||||
'fqn': ['test', 'view_summary'],
|
||||
'injected_sql': view_compiled_sql,
|
||||
'meta': {},
|
||||
'name': 'view_summary',
|
||||
'original_file_path': view_summary_path,
|
||||
@@ -3308,7 +3272,6 @@ class TestDocsGenerate(DBTIntegrationTest):
|
||||
'extra_ctes': [],
|
||||
'extra_ctes_injected': True,
|
||||
'fqn': ['test', 'seed'],
|
||||
'injected_sql': '',
|
||||
'meta': {},
|
||||
'name': 'seed',
|
||||
'original_file_path': seed_path,
|
||||
|
||||
@@ -64,8 +64,8 @@ class TestDeferState(DBTIntegrationTest):
|
||||
|
||||
# with state it should work though
|
||||
results = self.run_dbt(['run', '-m', 'view_model', '--state', 'state', '--defer', '--target', 'otherschema'])
|
||||
assert self.other_schema not in results[0].node.injected_sql
|
||||
assert self.unique_schema() in results[0].node.injected_sql
|
||||
assert self.other_schema not in results[0].node.compiled_sql
|
||||
assert self.unique_schema() in results[0].node.compiled_sql
|
||||
|
||||
with open('target/manifest.json') as fp:
|
||||
data = json.load(fp)
|
||||
|
||||
@@ -119,7 +119,6 @@ class CompilerTest(unittest.TestCase):
|
||||
compiled=True,
|
||||
extra_ctes_injected=False,
|
||||
extra_ctes=[InjectedCTE(id='model.root.ephemeral', sql='select * from source_table')],
|
||||
injected_sql='',
|
||||
compiled_sql=(
|
||||
'with cte as (select * from something_else) '
|
||||
'select * from __dbt__CTE__ephemeral'),
|
||||
@@ -147,7 +146,6 @@ class CompilerTest(unittest.TestCase):
|
||||
compiled_sql='select * from source_table',
|
||||
extra_ctes_injected=False,
|
||||
extra_ctes=[],
|
||||
injected_sql='',
|
||||
checksum=FileHash.from_contents(''),
|
||||
),
|
||||
},
|
||||
@@ -168,7 +166,7 @@ class CompilerTest(unittest.TestCase):
|
||||
self.assertEqual(result, manifest.nodes['model.root.view'])
|
||||
self.assertEqual(result.extra_ctes_injected, True)
|
||||
self.assertEqualIgnoreWhitespace(
|
||||
result.injected_sql,
|
||||
result.compiled_sql,
|
||||
('with __dbt__CTE__ephemeral as ('
|
||||
'select * from source_table'
|
||||
'), cte as (select * from something_else) '
|
||||
@@ -204,7 +202,6 @@ class CompilerTest(unittest.TestCase):
|
||||
compiled=True,
|
||||
extra_ctes_injected=False,
|
||||
extra_ctes=[],
|
||||
injected_sql='',
|
||||
compiled_sql=('with cte as (select * from something_else) '
|
||||
'select * from source_table'),
|
||||
checksum=FileHash.from_contents(''),
|
||||
@@ -230,7 +227,6 @@ class CompilerTest(unittest.TestCase):
|
||||
compiled=True,
|
||||
extra_ctes_injected=False,
|
||||
extra_ctes=[],
|
||||
injected_sql='',
|
||||
compiled_sql=('select * from source_table'),
|
||||
checksum=FileHash.from_contents(''),
|
||||
),
|
||||
@@ -246,15 +242,14 @@ class CompilerTest(unittest.TestCase):
|
||||
result, _ = compiler._recursively_prepend_ctes(
|
||||
manifest.nodes['model.root.view'],
|
||||
manifest,
|
||||
{}
|
||||
)
|
||||
{})
|
||||
|
||||
self.assertEqual(
|
||||
result,
|
||||
manifest.nodes.get('model.root.view'))
|
||||
self.assertTrue(result.extra_ctes_injected)
|
||||
self.assertEqualIgnoreWhitespace(
|
||||
result.injected_sql,
|
||||
result.compiled_sql,
|
||||
manifest.nodes.get('model.root.view').compiled_sql)
|
||||
|
||||
compiler = dbt.compilation.Compiler(self.config)
|
||||
@@ -268,7 +263,7 @@ class CompilerTest(unittest.TestCase):
|
||||
manifest.nodes.get('model.root.view_no_cte'))
|
||||
self.assertTrue(result.extra_ctes_injected)
|
||||
self.assertEqualIgnoreWhitespace(
|
||||
result.injected_sql,
|
||||
result.compiled_sql,
|
||||
manifest.nodes.get('model.root.view_no_cte').compiled_sql)
|
||||
|
||||
def test__prepend_ctes(self):
|
||||
@@ -298,7 +293,6 @@ class CompilerTest(unittest.TestCase):
|
||||
compiled=True,
|
||||
extra_ctes_injected=False,
|
||||
extra_ctes=[InjectedCTE(id='model.root.ephemeral', sql='select * from source_table')],
|
||||
injected_sql='',
|
||||
compiled_sql='select * from __dbt__CTE__ephemeral',
|
||||
checksum=FileHash.from_contents(''),
|
||||
),
|
||||
@@ -323,7 +317,6 @@ class CompilerTest(unittest.TestCase):
|
||||
compiled=True,
|
||||
extra_ctes_injected=False,
|
||||
extra_ctes=[],
|
||||
injected_sql='',
|
||||
compiled_sql='select * from source_table',
|
||||
checksum=FileHash.from_contents(''),
|
||||
),
|
||||
@@ -347,7 +340,7 @@ class CompilerTest(unittest.TestCase):
|
||||
|
||||
self.assertTrue(result.extra_ctes_injected)
|
||||
self.assertEqualIgnoreWhitespace(
|
||||
result.injected_sql,
|
||||
result.compiled_sql,
|
||||
('with __dbt__CTE__ephemeral as ('
|
||||
'select * from source_table'
|
||||
') '
|
||||
@@ -397,7 +390,6 @@ class CompilerTest(unittest.TestCase):
|
||||
raw_sql='select * from source_table',
|
||||
compiled=True,
|
||||
compiled_sql='select * from source_table',
|
||||
injected_sql='select * from source_table',
|
||||
extra_ctes_injected=True,
|
||||
extra_ctes=[],
|
||||
checksum=FileHash.from_contents(''),
|
||||
@@ -426,7 +418,6 @@ class CompilerTest(unittest.TestCase):
|
||||
compiled=True,
|
||||
extra_ctes_injected=False,
|
||||
extra_ctes=[InjectedCTE(id='model.root.ephemeral', sql='select * from source_table')],
|
||||
injected_sql='',
|
||||
compiled_sql='select * from __dbt__CTE__ephemeral',
|
||||
checksum=FileHash.from_contents(''),
|
||||
),
|
||||
@@ -448,7 +439,6 @@ class CompilerTest(unittest.TestCase):
|
||||
manifest,
|
||||
{}
|
||||
)
|
||||
compile_node.assert_called_once_with(parsed_ephemeral, manifest, {})
|
||||
|
||||
self.assertEqual(result,
|
||||
manifest.nodes.get('model.root.view'))
|
||||
@@ -456,7 +446,7 @@ class CompilerTest(unittest.TestCase):
|
||||
self.assertTrue(manifest.nodes['model.root.ephemeral'].compiled)
|
||||
self.assertTrue(result.extra_ctes_injected)
|
||||
self.assertEqualIgnoreWhitespace(
|
||||
result.injected_sql,
|
||||
result.compiled_sql,
|
||||
('with __dbt__CTE__ephemeral as ('
|
||||
'select * from source_table'
|
||||
') '
|
||||
@@ -491,7 +481,6 @@ class CompilerTest(unittest.TestCase):
|
||||
compiled=True,
|
||||
extra_ctes_injected=False,
|
||||
extra_ctes=[InjectedCTE(id='model.root.ephemeral', sql=None)],
|
||||
injected_sql=None,
|
||||
compiled_sql='select * from __dbt__CTE__ephemeral',
|
||||
checksum=FileHash.from_contents(''),
|
||||
|
||||
@@ -554,7 +543,7 @@ class CompilerTest(unittest.TestCase):
|
||||
self.assertEqual(result, manifest.nodes['model.root.view'])
|
||||
self.assertTrue(result.extra_ctes_injected)
|
||||
self.assertEqualIgnoreWhitespace(
|
||||
result.injected_sql,
|
||||
result.compiled_sql,
|
||||
('with __dbt__CTE__ephemeral_level_two as ('
|
||||
'select * from source_table'
|
||||
'), __dbt__CTE__ephemeral as ('
|
||||
|
||||
@@ -74,10 +74,9 @@ def basic_compiled_model():
|
||||
config=NodeConfig(),
|
||||
meta={},
|
||||
compiled=True,
|
||||
compiled_sql='select * from whatever',
|
||||
extra_ctes=[InjectedCTE('whatever', 'select * from other')],
|
||||
extra_ctes_injected=True,
|
||||
injected_sql='with whatever as (select * from other) select * from whatever',
|
||||
compiled_sql='with whatever as (select * from other) select * from whatever',
|
||||
checksum=FileHash.from_contents(''),
|
||||
unrendered_config={}
|
||||
)
|
||||
@@ -183,10 +182,9 @@ def basic_compiled_dict():
|
||||
'columns': {},
|
||||
'meta': {},
|
||||
'compiled': True,
|
||||
'compiled_sql': 'select * from whatever',
|
||||
'extra_ctes': [{'id': 'whatever', 'sql': 'select * from other'}],
|
||||
'extra_ctes_injected': True,
|
||||
'injected_sql': 'with whatever as (select * from other) select * from whatever',
|
||||
'compiled_sql': 'with whatever as (select * from other) select * from whatever',
|
||||
'checksum': {'name': 'sha256', 'checksum': 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855'},
|
||||
'unrendered_config': {}
|
||||
}
|
||||
@@ -375,10 +373,9 @@ def basic_compiled_schema_test_node():
|
||||
config=TestConfig(severity='warn'),
|
||||
meta={},
|
||||
compiled=True,
|
||||
compiled_sql='select * from whatever',
|
||||
extra_ctes=[InjectedCTE('whatever', 'select * from other')],
|
||||
extra_ctes_injected=True,
|
||||
injected_sql='with whatever as (select * from other) select * from whatever',
|
||||
compiled_sql='with whatever as (select * from other) select * from whatever',
|
||||
column_name='id',
|
||||
test_metadata=TestMetadata(namespace=None, name='foo', kwargs={}),
|
||||
checksum=FileHash.from_contents(''),
|
||||
@@ -474,10 +471,9 @@ def basic_compiled_schema_test_dict():
|
||||
'columns': {},
|
||||
'meta': {},
|
||||
'compiled': True,
|
||||
'compiled_sql': 'select * from whatever',
|
||||
'extra_ctes': [{'id': 'whatever', 'sql': 'select * from other'}],
|
||||
'extra_ctes_injected': True,
|
||||
'injected_sql': 'with whatever as (select * from other) select * from whatever',
|
||||
'compiled_sql': 'with whatever as (select * from other) select * from whatever',
|
||||
'column_name': 'id',
|
||||
'test_metadata': {
|
||||
'name': 'foo',
|
||||
|
||||
@@ -38,7 +38,6 @@ REQUIRED_PARSED_NODE_KEYS = frozenset({
|
||||
|
||||
REQUIRED_COMPILED_NODE_KEYS = frozenset(REQUIRED_PARSED_NODE_KEYS | {
|
||||
'compiled', 'extra_ctes_injected', 'extra_ctes', 'compiled_sql',
|
||||
'injected_sql',
|
||||
})
|
||||
|
||||
|
||||
@@ -482,7 +481,6 @@ class MixedManifestTest(unittest.TestCase):
|
||||
compiled=True,
|
||||
compiled_sql='also does not matter',
|
||||
extra_ctes_injected=True,
|
||||
injected_sql=None,
|
||||
extra_ctes=[],
|
||||
checksum=FileHash.empty(),
|
||||
),
|
||||
@@ -508,7 +506,6 @@ class MixedManifestTest(unittest.TestCase):
|
||||
compiled=True,
|
||||
compiled_sql='also does not matter',
|
||||
extra_ctes_injected=True,
|
||||
injected_sql='and this also does not matter',
|
||||
extra_ctes=[],
|
||||
checksum=FileHash.empty(),
|
||||
),
|
||||
|
||||
Reference in New Issue
Block a user