Compare commits

...

1 Commits

Author SHA1 Message Date
Gerda Shank
2d69a9b114 Remove injected_sql, use compiled_sql instead 2020-10-09 18:34:14 -04:00
17 changed files with 93 additions and 168 deletions

View File

@@ -169,36 +169,6 @@ class Compiler:
relation_cls = adapter.Relation
return relation_cls.add_ephemeral_prefix(name)
def _get_compiled_model(
self,
manifest: Manifest,
cte_id: str,
extra_context: Dict[str, Any],
) -> NonSourceCompiledNode:
if cte_id not in manifest.nodes:
raise InternalException(
f'During compilation, found a cte reference that could not be '
f'resolved: {cte_id}'
)
cte_model = manifest.nodes[cte_id]
if getattr(cte_model, 'compiled', False):
assert isinstance(cte_model, tuple(COMPILED_TYPES.values()))
return cast(NonSourceCompiledNode, cte_model)
elif cte_model.is_ephemeral_model:
# this must be some kind of parsed node that we can compile.
# we know it's not a parsed source definition
assert isinstance(cte_model, tuple(COMPILED_TYPES))
# update the node so
node = self.compile_node(cte_model, manifest, extra_context)
manifest.sync_update_node(node)
return node
else:
raise InternalException(
f'During compilation, found an uncompiled cte that '
f'was not an ephemeral model: {cte_id}'
)
def _inject_ctes_into_sql(self, sql: str, ctes: List[InjectedCTE]) -> str:
"""
`ctes` is a list of InjectedCTEs like:
@@ -260,26 +230,6 @@ class Compiler:
return str(parsed)
def _model_prepend_ctes(
self,
model: NonSourceCompiledNode,
prepended_ctes: List[InjectedCTE]
) -> NonSourceCompiledNode:
if model.compiled_sql is None:
raise RuntimeException(
'Cannot prepend ctes to an unparsed node', model
)
injected_sql = self._inject_ctes_into_sql(
model.compiled_sql,
prepended_ctes,
)
model.extra_ctes_injected = True
model.extra_ctes = prepended_ctes
model.injected_sql = injected_sql
model.validate(model.to_dict())
return model
def _get_dbt_test_name(self) -> str:
return 'dbt__CTE__INTERNAL_test'
@@ -287,7 +237,7 @@ class Compiler:
self,
model: NonSourceCompiledNode,
manifest: Manifest,
extra_context: Dict[str, Any],
extra_context: Optional[Dict[str, Any]],
) -> Tuple[NonSourceCompiledNode, List[InjectedCTE]]:
if model.extra_ctes_injected:
return (model, model.extra_ctes)
@@ -303,41 +253,73 @@ class Compiler:
dbt_test_name = self._get_dbt_test_name()
for cte in model.extra_ctes:
new_prep_ctes: List[InjectedCTE]
orig_compiled_sql: Optional[str]
if cte.id == dbt_test_name:
sql = cte.sql
orig_compiled_sql = sql
else:
cte_model = self._get_compiled_model(
manifest,
cte.id,
extra_context,
)
cte_model, new_prepended_ctes = self._recursively_prepend_ctes(
cte_model, manifest, extra_context
)
_extend_prepended_ctes(prepended_ctes, new_prepended_ctes)
cte_model = manifest.nodes[cte.id]
if getattr(cte_model, 'compiled', False):
assert isinstance(cte_model, tuple(
COMPILED_TYPES.values()))
cte_model = cast(NonSourceCompiledNode, cte_model)
orig_compiled_sql = cte_model.compiled_sql
cte_model, new_prep_ctes = self._recursively_prepend_ctes(
cte_model, manifest, extra_context
)
elif cte_model.is_ephemeral_model:
assert isinstance(cte_model, tuple(COMPILED_TYPES))
compiled_node = self._compile_node(
cte_model, manifest, extra_context)
orig_compiled_sql = compiled_node.compiled_sql
cte_model, new_prep_ctes = self._recursively_prepend_ctes(
compiled_node, manifest, extra_context
)
manifest.sync_update_node(cte_model)
else:
raise InternalException(
f'During compilation, found an uncompiled cte that '
f'was not an ephemeral model: {cte.id}'
)
_extend_prepended_ctes(prepended_ctes, new_prep_ctes)
new_cte_name = self.add_ephemeral_prefix(cte_model.name)
sql = f' {new_cte_name} as (\n{cte_model.compiled_sql}\n)'
sql = f' {new_cte_name} as (\n{orig_compiled_sql}\n)'
_add_prepended_cte(prepended_ctes, InjectedCTE(id=cte.id, sql=sql))
model = self._model_prepend_ctes(model, prepended_ctes)
if model.compiled_sql is None:
raise RuntimeException(
'Cannot inject ctes into an unparsed node', model
)
injected_sql = self._inject_ctes_into_sql(
model.compiled_sql,
prepended_ctes,
)
model.extra_ctes_injected = True
model.extra_ctes = prepended_ctes
model.compiled_sql = injected_sql
model.validate(model.to_dict())
manifest.update_node(model)
return model, prepended_ctes
def _insert_ctes(
def _add_extra_ctes(
self,
compiled_node: NonSourceCompiledNode,
manifest: Manifest,
extra_context: Dict[str, Any],
) -> NonSourceCompiledNode:
"""Insert the CTEs for the model."""
):
"""Add extra CTEs for the model."""
# for data tests, we need to insert a special CTE at the end of the
# list containing the test query, and then have the "real" query be a
# select count(*) from that model.
# the benefit of doing it this way is that _insert_ctes() can be
# the benefit of doing it this way is that _add_extra_ctes() can be
# rewritten for different adapters to handle databses that don't
# support CTEs, or at least don't have full support.
if isinstance(compiled_node, CompiledDataTestNode):
@@ -352,11 +334,7 @@ class Compiler:
compiled_node.extra_ctes.append(cte)
compiled_node.compiled_sql = f'\nselect count(*) from {name}'
injected_node, _ = self._recursively_prepend_ctes(
compiled_node, manifest, extra_context
)
return injected_node
# this is called by both 'compile_node' and '_recursively_prepend_ctes'
def _compile_node(
self,
node: ManifestNode,
@@ -374,7 +352,6 @@ class Compiler:
'compiled_sql': None,
'extra_ctes_injected': False,
'extra_ctes': [],
'injected_sql': None,
})
compiled_node = _compiled_type_for(node).from_dict(data)
@@ -390,11 +367,13 @@ class Compiler:
compiled_node.compiled = True
injected_node = self._insert_ctes(
# add extra ctes, such as for a test node or possibly
# adapter specific ctes
self._add_extra_ctes(
compiled_node, manifest, extra_context
)
return injected_node
return compiled_node
def write_graph_file(self, linker: Linker, manifest: Manifest):
filename = graph_file_name
@@ -454,18 +433,16 @@ class Compiler:
return node
logger.debug(f'Writing injected SQL for node "{node.unique_id}"')
if node.injected_sql is None:
# this should not really happen, but it'd be a shame to crash
# over it
if node.compiled_sql is None:
logger.error(
f'Compiled node "{node.unique_id}" had no injected_sql, '
f'Compiled node "{node.unique_id}" had no compiled_sql, '
'cannot write sql!'
)
else:
node.build_path = node.write_node(
self.config.target_path,
'compiled',
node.injected_sql
node.compiled_sql
)
return node
@@ -476,15 +453,19 @@ class Compiler:
extra_context: Optional[Dict[str, Any]] = None,
write: bool = True,
) -> NonSourceCompiledNode:
node = self._compile_node(node, manifest, extra_context)
compiled_node = self._compile_node(node, manifest, extra_context)
if write and _is_writable(node):
self._write_node(node)
return node
injected_node, _ = self._recursively_prepend_ctes(
compiled_node, manifest, extra_context
)
if write and _is_writable(injected_node):
self._write_node(injected_node)
return injected_node
def _is_writable(node):
if not node.injected_sql:
if not node.compiled_sql:
return False
if node.resource_type == NodeType.Snapshot:

View File

@@ -1217,7 +1217,7 @@ class ModelContext(ProviderContext):
@contextproperty
def sql(self) -> Optional[str]:
return getattr(self.model, 'injected_sql', None)
return getattr(self.model, 'compiled_sql', None)
@contextproperty
def database(self) -> str:

View File

@@ -42,7 +42,6 @@ class CompiledNode(ParsedNode, CompiledNodeMixin):
compiled_sql: Optional[str] = None
extra_ctes_injected: bool = False
extra_ctes: List[InjectedCTE] = field(default_factory=list)
injected_sql: Optional[str] = None
def set_cte(self, cte_id: str, sql: str):
"""This is the equivalent of what self.extra_ctes[cte_id] = sql would

View File

@@ -132,7 +132,7 @@ class RuntimeException(RuntimeError, Exception):
result.update({
'raw_sql': self.node.raw_sql,
# the node isn't always compiled, but if it is, include that!
'compiled_sql': getattr(self.node, 'injected_sql', None),
'compiled_sql': getattr(self.node, 'compiled_sql', None),
})
return result

View File

@@ -214,7 +214,7 @@
{% if not target_relation_exists %}
{% set build_sql = build_snapshot_table(strategy, model['injected_sql']) %}
{% set build_sql = build_snapshot_table(strategy, model['compiled_sql']) %}
{% set final_sql = create_table_as(False, target_relation, build_sql) %}
{% else %}

View File

@@ -106,7 +106,7 @@
{% macro snapshot_check_all_get_existing_columns(node, target_exists) -%}
{%- set query_columns = get_columns_in_query(node['injected_sql']) -%}
{%- set query_columns = get_columns_in_query(node['compiled_sql']) -%}
{%- if not target_exists -%}
{# no table yet -> return whatever the query does #}
{{ return([false, query_columns]) }}

View File

@@ -65,7 +65,7 @@ class RPCCompileRunner(GenericRPCRunner[RemoteCompileResult]):
def execute(self, compiled_node, manifest) -> RemoteCompileResult:
return RemoteCompileResult(
raw_sql=compiled_node.raw_sql,
compiled_sql=compiled_node.injected_sql,
compiled_sql=compiled_node.compiled_sql,
node=compiled_node,
timing=[], # this will get added later
logs=[],
@@ -88,7 +88,7 @@ class RPCCompileRunner(GenericRPCRunner[RemoteCompileResult]):
class RPCExecuteRunner(GenericRPCRunner[RemoteRunResult]):
def execute(self, compiled_node, manifest) -> RemoteRunResult:
_, execute_result = self.adapter.execute(
compiled_node.injected_sql, fetch=True
compiled_node.compiled_sql, fetch=True
)
table = ResultTable(
@@ -98,7 +98,7 @@ class RPCExecuteRunner(GenericRPCRunner[RemoteRunResult]):
return RemoteRunResult(
raw_sql=compiled_node.raw_sql,
compiled_sql=compiled_node.injected_sql,
compiled_sql=compiled_node.compiled_sql,
node=compiled_node,
table=table,
timing=[],

View File

@@ -255,7 +255,7 @@ class RunTask(CompileTask):
def get_hook_sql(self, adapter, hook, idx, num_hooks, extra_context):
compiler = adapter.get_compiler()
compiled = compiler.compile_node(hook, self.manifest, extra_context)
statement = compiled.injected_sql
statement = compiled.compiled_sql
hook_index = hook.index or num_hooks
hook_obj = get_hook(statement, index=hook_index)
return hook_obj.sql or ''

View File

@@ -42,7 +42,7 @@ class TestRunner(CompileRunner):
def execute_data_test(self, test: CompiledDataTestNode):
res, table = self.adapter.execute(
test.injected_sql, auto_begin=True, fetch=True
test.compiled_sql, auto_begin=True, fetch=True
)
num_rows = len(table.rows)
@@ -59,7 +59,7 @@ class TestRunner(CompileRunner):
def execute_schema_test(self, test: CompiledSchemaTestNode):
res, table = self.adapter.execute(
test.injected_sql,
test.compiled_sql,
auto_begin=True,
fetch=True,
)

View File

@@ -385,7 +385,7 @@ class BigQueryAdapter(BaseAdapter):
model_database = model.get('database')
model_schema = model.get('schema')
model_alias = model.get('alias')
model_sql = model.get('injected_sql')
model_sql = model.get('compiled_sql')
logger.debug("Model SQL ({}):\n{}".format(model_alias, model_sql))
self.connections.create_view(
@@ -505,7 +505,7 @@ class BigQueryAdapter(BaseAdapter):
decorator=None):
if sql_override is None:
sql_override = model.get('injected_sql')
sql_override = model.get('compiled_sql')
if flags.STRICT_MODE:
connection = self.connections.get_thread_connection()

View File

@@ -11,7 +11,7 @@
{{ log(table_start_time ~ ' | -> Running for day ' ~ date, info=True) }}
{% endif %}
{% set fixed_sql = model['injected_sql'] | replace('[DBT__PARTITION_DATE]', date) %}
{% set fixed_sql = model['compiled_sql'] | replace('[DBT__PARTITION_DATE]', date) %}
{% set _ = adapter.execute_model(model, 'table', fixed_sql, decorator=date) %}
{% endfor %}

View File

@@ -130,7 +130,7 @@ class TestCLIInvocationWithProfilesDir(ModelCopyingIntegrationTest):
# make sure the test runs against `custom_schema`
for test_result in res:
self.assertTrue(self.custom_schema, test_result.node.injected_sql)
self.assertTrue(self.custom_schema, test_result.node.compiled_sql)
class TestCLIInvocationWithProjectDir(ModelCopyingIntegrationTest):

View File

@@ -130,17 +130,6 @@ class TestDocsGenerate(DBTIntegrationTest):
def models(self):
return self.dir("models")
@property
def packages_config(self):
return {
'packages': [
{
'git': 'https://github.com/fishtown-analytics/dbt-integration-project',
'revision': 'dbt/0.17.0',
},
],
}
@property
def project_config(self):
return {
@@ -1114,7 +1103,6 @@ class TestDocsGenerate(DBTIntegrationTest):
'compiled_sql': ANY,
'extra_ctes_injected': True,
'extra_ctes': [],
'injected_sql': ANY,
'checksum': self._checksum_file(model_sql_path),
'unrendered_config': unrendered_model_config,
},
@@ -1188,7 +1176,6 @@ class TestDocsGenerate(DBTIntegrationTest):
'compiled_sql': ANY,
'extra_ctes_injected': True,
'extra_ctes': [],
'injected_sql': ANY,
'checksum': self._checksum_file(second_model_sql_path),
'unrendered_config': unrendered_second_config
},
@@ -1264,7 +1251,6 @@ class TestDocsGenerate(DBTIntegrationTest):
'compiled_sql': '',
'extra_ctes_injected': True,
'extra_ctes': [],
'injected_sql': '',
'checksum': self._checksum_file(seed_path),
'unrendered_config': unrendered_seed_config,
},
@@ -1301,7 +1287,6 @@ class TestDocsGenerate(DBTIntegrationTest):
'compiled_sql': AnyStringWith('count(*)'),
'extra_ctes_injected': True,
'extra_ctes': [],
'injected_sql': AnyStringWith('count(*)'),
'test_metadata': {
'namespace': None,
'name': 'not_null',
@@ -1346,7 +1331,6 @@ class TestDocsGenerate(DBTIntegrationTest):
'compiled_sql': AnyStringWith('select 0'),
'extra_ctes_injected': True,
'extra_ctes': [],
'injected_sql': AnyStringWith('select 0'),
'test_metadata': {
'namespace': 'test',
'name': 'nothing',
@@ -1390,7 +1374,6 @@ class TestDocsGenerate(DBTIntegrationTest):
'compiled_sql': AnyStringWith('count(*)'),
'extra_ctes_injected': True,
'extra_ctes': [],
'injected_sql': AnyStringWith('count(*)'),
'test_metadata': {
'namespace': None,
'name': 'unique',
@@ -1587,7 +1570,6 @@ class TestDocsGenerate(DBTIntegrationTest):
'compiled_sql': ANY,
'extra_ctes_injected': True,
'extra_ctes': [],
'injected_sql': ANY,
'checksum': self._checksum_file(ephemeral_copy_path),
'unrendered_config': self.unrendered_model_config(materialized='ephemeral'),
},
@@ -1645,7 +1627,6 @@ class TestDocsGenerate(DBTIntegrationTest):
'compiled_sql': ANY,
'extra_ctes_injected': True,
'extra_ctes': [ANY],
'injected_sql': ANY,
'checksum': self._checksum_file(ephemeral_summary_path),
'unrendered_config': self.unrendered_model_config(materialized='table'),
},
@@ -1702,7 +1683,6 @@ class TestDocsGenerate(DBTIntegrationTest):
'compiled_sql': ANY,
'extra_ctes_injected': True,
'extra_ctes': [],
'injected_sql': ANY,
'checksum': self._checksum_file(view_summary_path),
'unrendered_config': self.unrendered_model_config(materialized='view'),
},
@@ -1776,7 +1756,6 @@ class TestDocsGenerate(DBTIntegrationTest):
'compiled_sql': '',
'extra_ctes_injected': True,
'extra_ctes': [],
'injected_sql': '',
'checksum': self._checksum_file(seed_path),
'unrendered_config': self.unrendered_seed_config(),
},
@@ -2047,7 +2026,6 @@ class TestDocsGenerate(DBTIntegrationTest):
'compiled_sql': ANY,
'extra_ctes_injected': True,
'extra_ctes': [],
'injected_sql': ANY,
'checksum': self._checksum_file(clustered_sql_path),
'unrendered_config': self.unrendered_model_config(
cluster_by=['first_name'],
@@ -2129,7 +2107,6 @@ class TestDocsGenerate(DBTIntegrationTest):
'compiled_sql': ANY,
'extra_ctes_injected': True,
'extra_ctes': [],
'injected_sql': ANY,
'checksum': self._checksum_file(multi_clustered_sql_path),
'unrendered_config': self.unrendered_model_config(
cluster_by=['first_name', 'email'],
@@ -2210,7 +2187,6 @@ class TestDocsGenerate(DBTIntegrationTest):
'compiled_sql': ANY,
'extra_ctes_injected': True,
'extra_ctes': [],
'injected_sql': ANY,
'checksum': self._checksum_file(nested_view_sql_path),
'unrendered_config': self.unrendered_model_config(materialized='view'),
},
@@ -2246,7 +2222,6 @@ class TestDocsGenerate(DBTIntegrationTest):
'compiled_sql': ANY,
'extra_ctes_injected': True,
'extra_ctes': [],
'injected_sql': ANY,
'checksum': self._checksum_file(nested_table_sql_path),
'unrendered_config': self.unrendered_model_config(materialized='table'),
},
@@ -2323,7 +2298,6 @@ class TestDocsGenerate(DBTIntegrationTest):
'compiled_sql': '',
'extra_ctes_injected': True,
'extra_ctes': [],
'injected_sql': '',
'checksum': self._checksum_file(seed_path),
'unrendered_config': self.unrendered_seed_config(),
},
@@ -2459,7 +2433,6 @@ class TestDocsGenerate(DBTIntegrationTest):
'compiled_sql': ANY,
'extra_ctes_injected': True,
'extra_ctes': [],
'injected_sql': ANY,
'checksum': self._checksum_file(model_sql_path),
'unrendered_config': self.unrendered_model_config(bind=False, materialized='view'),
},
@@ -2536,7 +2509,6 @@ class TestDocsGenerate(DBTIntegrationTest):
'compiled_sql': ANY,
'extra_ctes_injected': True,
'extra_ctes': [],
'injected_sql': ANY,
'checksum': self._checksum_file(seed_path),
'unrendered_config': self.unrendered_seed_config(),
},
@@ -2710,7 +2682,6 @@ class TestDocsGenerate(DBTIntegrationTest):
'extra_ctes': [],
'extra_ctes_injected': True,
'fqn': ['test', 'model'],
'injected_sql': compiled_sql,
'meta': {},
'name': 'model',
'original_file_path': model_sql_path,
@@ -2799,7 +2770,6 @@ class TestDocsGenerate(DBTIntegrationTest):
'extra_ctes': [],
'extra_ctes_injected': True,
'fqn': ['test', 'second_model'],
'injected_sql': compiled_sql,
'meta': {},
'name': 'second_model',
'original_file_path': second_model_sql_path,
@@ -2883,7 +2853,6 @@ class TestDocsGenerate(DBTIntegrationTest):
'extra_ctes': [],
'extra_ctes_injected': True,
'fqn': ['test', 'seed'],
'injected_sql': '',
'meta': {},
'name': 'seed',
'original_file_path': seed_path,
@@ -2930,7 +2899,6 @@ class TestDocsGenerate(DBTIntegrationTest):
'extra_ctes': [],
'extra_ctes_injected': True,
'fqn': ['test', 'schema_test', 'not_null_model_id'],
'injected_sql': AnyStringWith('id is null'),
'meta': {},
'name': 'not_null_model_id',
'original_file_path': model_schema_yml_path,
@@ -2985,7 +2953,6 @@ class TestDocsGenerate(DBTIntegrationTest):
'extra_ctes': [],
'extra_ctes_injected': True,
'fqn': ['test', 'schema_test', 'test_nothing_model_'],
'injected_sql': AnyStringWith('select 0'),
'meta': {},
'name': 'test_nothing_model_',
'original_file_path': model_schema_yml_path,
@@ -3039,7 +3006,6 @@ class TestDocsGenerate(DBTIntegrationTest):
'extra_ctes': [],
'extra_ctes_injected': True,
'fqn': ['test', 'schema_test', 'unique_model_id'],
'injected_sql': AnyStringWith('count(*)'),
'meta': {},
'name': 'unique_model_id',
'original_file_path': model_schema_yml_path,
@@ -3129,7 +3095,7 @@ class TestDocsGenerate(DBTIntegrationTest):
},
},
'compiled': True,
'compiled_sql': ephemeral_compiled_sql,
'compiled_sql': ephemeral_injected_sql,
'config': self.rendered_model_config(materialized='table'),
'sources': [],
'depends_on': {
@@ -3146,7 +3112,6 @@ class TestDocsGenerate(DBTIntegrationTest):
],
'extra_ctes_injected': True,
'fqn': ['test', 'ephemeral_summary'],
'injected_sql': ephemeral_injected_sql,
'meta': {},
'name': 'ephemeral_summary',
'original_file_path': ephemeral_summary_path,
@@ -3220,7 +3185,6 @@ class TestDocsGenerate(DBTIntegrationTest):
'extra_ctes': [],
'extra_ctes_injected': True,
'fqn': ['test', 'view_summary'],
'injected_sql': view_compiled_sql,
'meta': {},
'name': 'view_summary',
'original_file_path': view_summary_path,
@@ -3308,7 +3272,6 @@ class TestDocsGenerate(DBTIntegrationTest):
'extra_ctes': [],
'extra_ctes_injected': True,
'fqn': ['test', 'seed'],
'injected_sql': '',
'meta': {},
'name': 'seed',
'original_file_path': seed_path,

View File

@@ -64,8 +64,8 @@ class TestDeferState(DBTIntegrationTest):
# with state it should work though
results = self.run_dbt(['run', '-m', 'view_model', '--state', 'state', '--defer', '--target', 'otherschema'])
assert self.other_schema not in results[0].node.injected_sql
assert self.unique_schema() in results[0].node.injected_sql
assert self.other_schema not in results[0].node.compiled_sql
assert self.unique_schema() in results[0].node.compiled_sql
with open('target/manifest.json') as fp:
data = json.load(fp)

View File

@@ -119,7 +119,6 @@ class CompilerTest(unittest.TestCase):
compiled=True,
extra_ctes_injected=False,
extra_ctes=[InjectedCTE(id='model.root.ephemeral', sql='select * from source_table')],
injected_sql='',
compiled_sql=(
'with cte as (select * from something_else) '
'select * from __dbt__CTE__ephemeral'),
@@ -147,7 +146,6 @@ class CompilerTest(unittest.TestCase):
compiled_sql='select * from source_table',
extra_ctes_injected=False,
extra_ctes=[],
injected_sql='',
checksum=FileHash.from_contents(''),
),
},
@@ -168,7 +166,7 @@ class CompilerTest(unittest.TestCase):
self.assertEqual(result, manifest.nodes['model.root.view'])
self.assertEqual(result.extra_ctes_injected, True)
self.assertEqualIgnoreWhitespace(
result.injected_sql,
result.compiled_sql,
('with __dbt__CTE__ephemeral as ('
'select * from source_table'
'), cte as (select * from something_else) '
@@ -204,7 +202,6 @@ class CompilerTest(unittest.TestCase):
compiled=True,
extra_ctes_injected=False,
extra_ctes=[],
injected_sql='',
compiled_sql=('with cte as (select * from something_else) '
'select * from source_table'),
checksum=FileHash.from_contents(''),
@@ -230,7 +227,6 @@ class CompilerTest(unittest.TestCase):
compiled=True,
extra_ctes_injected=False,
extra_ctes=[],
injected_sql='',
compiled_sql=('select * from source_table'),
checksum=FileHash.from_contents(''),
),
@@ -246,15 +242,14 @@ class CompilerTest(unittest.TestCase):
result, _ = compiler._recursively_prepend_ctes(
manifest.nodes['model.root.view'],
manifest,
{}
)
{})
self.assertEqual(
result,
manifest.nodes.get('model.root.view'))
self.assertTrue(result.extra_ctes_injected)
self.assertEqualIgnoreWhitespace(
result.injected_sql,
result.compiled_sql,
manifest.nodes.get('model.root.view').compiled_sql)
compiler = dbt.compilation.Compiler(self.config)
@@ -268,7 +263,7 @@ class CompilerTest(unittest.TestCase):
manifest.nodes.get('model.root.view_no_cte'))
self.assertTrue(result.extra_ctes_injected)
self.assertEqualIgnoreWhitespace(
result.injected_sql,
result.compiled_sql,
manifest.nodes.get('model.root.view_no_cte').compiled_sql)
def test__prepend_ctes(self):
@@ -298,7 +293,6 @@ class CompilerTest(unittest.TestCase):
compiled=True,
extra_ctes_injected=False,
extra_ctes=[InjectedCTE(id='model.root.ephemeral', sql='select * from source_table')],
injected_sql='',
compiled_sql='select * from __dbt__CTE__ephemeral',
checksum=FileHash.from_contents(''),
),
@@ -323,7 +317,6 @@ class CompilerTest(unittest.TestCase):
compiled=True,
extra_ctes_injected=False,
extra_ctes=[],
injected_sql='',
compiled_sql='select * from source_table',
checksum=FileHash.from_contents(''),
),
@@ -347,7 +340,7 @@ class CompilerTest(unittest.TestCase):
self.assertTrue(result.extra_ctes_injected)
self.assertEqualIgnoreWhitespace(
result.injected_sql,
result.compiled_sql,
('with __dbt__CTE__ephemeral as ('
'select * from source_table'
') '
@@ -397,7 +390,6 @@ class CompilerTest(unittest.TestCase):
raw_sql='select * from source_table',
compiled=True,
compiled_sql='select * from source_table',
injected_sql='select * from source_table',
extra_ctes_injected=True,
extra_ctes=[],
checksum=FileHash.from_contents(''),
@@ -426,7 +418,6 @@ class CompilerTest(unittest.TestCase):
compiled=True,
extra_ctes_injected=False,
extra_ctes=[InjectedCTE(id='model.root.ephemeral', sql='select * from source_table')],
injected_sql='',
compiled_sql='select * from __dbt__CTE__ephemeral',
checksum=FileHash.from_contents(''),
),
@@ -448,7 +439,6 @@ class CompilerTest(unittest.TestCase):
manifest,
{}
)
compile_node.assert_called_once_with(parsed_ephemeral, manifest, {})
self.assertEqual(result,
manifest.nodes.get('model.root.view'))
@@ -456,7 +446,7 @@ class CompilerTest(unittest.TestCase):
self.assertTrue(manifest.nodes['model.root.ephemeral'].compiled)
self.assertTrue(result.extra_ctes_injected)
self.assertEqualIgnoreWhitespace(
result.injected_sql,
result.compiled_sql,
('with __dbt__CTE__ephemeral as ('
'select * from source_table'
') '
@@ -491,7 +481,6 @@ class CompilerTest(unittest.TestCase):
compiled=True,
extra_ctes_injected=False,
extra_ctes=[InjectedCTE(id='model.root.ephemeral', sql=None)],
injected_sql=None,
compiled_sql='select * from __dbt__CTE__ephemeral',
checksum=FileHash.from_contents(''),
@@ -554,7 +543,7 @@ class CompilerTest(unittest.TestCase):
self.assertEqual(result, manifest.nodes['model.root.view'])
self.assertTrue(result.extra_ctes_injected)
self.assertEqualIgnoreWhitespace(
result.injected_sql,
result.compiled_sql,
('with __dbt__CTE__ephemeral_level_two as ('
'select * from source_table'
'), __dbt__CTE__ephemeral as ('

View File

@@ -74,10 +74,9 @@ def basic_compiled_model():
config=NodeConfig(),
meta={},
compiled=True,
compiled_sql='select * from whatever',
extra_ctes=[InjectedCTE('whatever', 'select * from other')],
extra_ctes_injected=True,
injected_sql='with whatever as (select * from other) select * from whatever',
compiled_sql='with whatever as (select * from other) select * from whatever',
checksum=FileHash.from_contents(''),
unrendered_config={}
)
@@ -183,10 +182,9 @@ def basic_compiled_dict():
'columns': {},
'meta': {},
'compiled': True,
'compiled_sql': 'select * from whatever',
'extra_ctes': [{'id': 'whatever', 'sql': 'select * from other'}],
'extra_ctes_injected': True,
'injected_sql': 'with whatever as (select * from other) select * from whatever',
'compiled_sql': 'with whatever as (select * from other) select * from whatever',
'checksum': {'name': 'sha256', 'checksum': 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855'},
'unrendered_config': {}
}
@@ -375,10 +373,9 @@ def basic_compiled_schema_test_node():
config=TestConfig(severity='warn'),
meta={},
compiled=True,
compiled_sql='select * from whatever',
extra_ctes=[InjectedCTE('whatever', 'select * from other')],
extra_ctes_injected=True,
injected_sql='with whatever as (select * from other) select * from whatever',
compiled_sql='with whatever as (select * from other) select * from whatever',
column_name='id',
test_metadata=TestMetadata(namespace=None, name='foo', kwargs={}),
checksum=FileHash.from_contents(''),
@@ -474,10 +471,9 @@ def basic_compiled_schema_test_dict():
'columns': {},
'meta': {},
'compiled': True,
'compiled_sql': 'select * from whatever',
'extra_ctes': [{'id': 'whatever', 'sql': 'select * from other'}],
'extra_ctes_injected': True,
'injected_sql': 'with whatever as (select * from other) select * from whatever',
'compiled_sql': 'with whatever as (select * from other) select * from whatever',
'column_name': 'id',
'test_metadata': {
'name': 'foo',

View File

@@ -38,7 +38,6 @@ REQUIRED_PARSED_NODE_KEYS = frozenset({
REQUIRED_COMPILED_NODE_KEYS = frozenset(REQUIRED_PARSED_NODE_KEYS | {
'compiled', 'extra_ctes_injected', 'extra_ctes', 'compiled_sql',
'injected_sql',
})
@@ -482,7 +481,6 @@ class MixedManifestTest(unittest.TestCase):
compiled=True,
compiled_sql='also does not matter',
extra_ctes_injected=True,
injected_sql=None,
extra_ctes=[],
checksum=FileHash.empty(),
),
@@ -508,7 +506,6 @@ class MixedManifestTest(unittest.TestCase):
compiled=True,
compiled_sql='also does not matter',
extra_ctes_injected=True,
injected_sql='and this also does not matter',
extra_ctes=[],
checksum=FileHash.empty(),
),