Remove injected_sql, use compiled_sql instead

CHANGELOG.md

@@ -9,9 +9,15 @@
 - Added schema and dbt versions to JSON artifacts ([#2670](https://github.com/fishtown-analytics/dbt/issues/2670), [#2767](https://github.com/fishtown-analytics/dbt/pull/2767))
 - Added ability to snapshot hard-deleted records (opt-in with `invalidate_hard_deletes` config option). ([#249](https://github.com/fishtown-analytics/dbt/issues/249), [#2749](https://github.com/fishtown-analytics/dbt/pull/2749))
 - Improved error messages for YAML selectors ([#2700](https://github.com/fishtown-analytics/dbt/issues/2700), [#2781](https://github.com/fishtown-analytics/dbt/pull/2781))
+- Save manifest at the same time we save the run_results at the end of a run ([#2765](https://github.com/fishtown-analytics/dbt/issues/2765), [#2799](https://github.com/fishtown-analytics/dbt/pull/2799))
+
+### Under the hood
+- Added strategy-specific validation to improve the relevancy of compilation errors for the `timestamp` and `check` snapshot strategies. (([#2787](https://github.com/fishtown-analytics/dbt/issues/2787), [#2791](https://github.com/fishtown-analytics/dbt/pull/2791))
+- Changed rpc test timeouts to avoid locally run test failures ([#2803](https://github.com/fishtown-analytics/dbt/issues/2803),[#2804](https://github.com/fishtown-analytics/dbt/pull/2804))
 
 Contributors:
 - [@joelluijmes](https://github.com/joelluijmes) ([#2749](https://github.com/fishtown-analytics/dbt/pull/2749))
+- [@kingfink](https://github.com/kingfink) ([#2791](https://github.com/fishtown-analytics/dbt/pull/2791))
 
 ## dbt 0.18.1 (Release TBD)
 
@@ -30,7 +36,6 @@ Contributors:
 - [@tpilewicz](https://github.com/tpilewicz) ([#2723](https://github.com/fishtown-analytics/dbt/pull/2723))
 - [@heisencoder](https://github.com/heisencoder) ([#2739](https://github.com/fishtown-analytics/dbt/issues/2739))
 
-
 ## dbt 0.18.0 (September 03, 2020)
 
 ### Under the hood

core/dbt/compilation.py

@@ -169,36 +169,6 @@ class Compiler:
         relation_cls = adapter.Relation
         return relation_cls.add_ephemeral_prefix(name)
 
-    def _get_compiled_model(
-        self,
-        manifest: Manifest,
-        cte_id: str,
-        extra_context: Dict[str, Any],
-    ) -> NonSourceCompiledNode:
-
-        if cte_id not in manifest.nodes:
-            raise InternalException(
-                f'During compilation, found a cte reference that could not be '
-                f'resolved: {cte_id}'
-            )
-        cte_model = manifest.nodes[cte_id]
-        if getattr(cte_model, 'compiled', False):
-            assert isinstance(cte_model, tuple(COMPILED_TYPES.values()))
-            return cast(NonSourceCompiledNode, cte_model)
-        elif cte_model.is_ephemeral_model:
-            # this must be some kind of parsed node that we can compile.
-            # we know it's not a parsed source definition
-            assert isinstance(cte_model, tuple(COMPILED_TYPES))
-            # update the node so
-            node = self.compile_node(cte_model, manifest, extra_context)
-            manifest.sync_update_node(node)
-            return node
-        else:
-            raise InternalException(
-                f'During compilation, found an uncompiled cte that '
-                f'was not an ephemeral model: {cte_id}'
-            )
-
     def _inject_ctes_into_sql(self, sql: str, ctes: List[InjectedCTE]) -> str:
         """
         `ctes` is a list of InjectedCTEs like:
@@ -260,26 +230,6 @@ class Compiler:
 
         return str(parsed)
 
-    def _model_prepend_ctes(
-        self,
-        model: NonSourceCompiledNode,
-        prepended_ctes: List[InjectedCTE]
-    ) -> NonSourceCompiledNode:
-        if model.compiled_sql is None:
-            raise RuntimeException(
-                'Cannot prepend ctes to an unparsed node', model
-            )
-        injected_sql = self._inject_ctes_into_sql(
-            model.compiled_sql,
-            prepended_ctes,
-        )
-
-        model.extra_ctes_injected = True
-        model.extra_ctes = prepended_ctes
-        model.injected_sql = injected_sql
-        model.validate(model.to_dict())
-        return model
-
     def _get_dbt_test_name(self) -> str:
         return 'dbt__CTE__INTERNAL_test'
 
@@ -288,9 +238,10 @@ class Compiler:
         model: NonSourceCompiledNode,
         manifest: Manifest,
         extra_context: Dict[str, Any],
-    ) -> Tuple[NonSourceCompiledNode, List[InjectedCTE]]:
+    ) -> Tuple[NonSourceCompiledNode, List[InjectedCTE], str]:
+
         if model.extra_ctes_injected:
-            return (model, model.extra_ctes)
+            return (model, model.extra_ctes, '')
 
         if flags.STRICT_MODE:
             if not isinstance(model, tuple(COMPILED_TYPES.values())):
@@ -303,28 +254,84 @@ class Compiler:
         dbt_test_name = self._get_dbt_test_name()
 
         for cte in model.extra_ctes:
+            new_prepended_ctes: List[InjectedCTE]
+            orig_compiled_sql: str
             if cte.id == dbt_test_name:
                 sql = cte.sql
             else:
-                cte_model = self._get_compiled_model(
-                    manifest,
-                    cte.id,
-                    extra_context,
-                )
-                cte_model, new_prepended_ctes = self._recursively_prepend_ctes(
-                    cte_model, manifest, extra_context
-                )
+                cte_model = manifest.nodes[cte.id]
+                if getattr(cte_model, 'compiled', False):
+                    assert isinstance(cte_model, tuple(COMPILED_TYPES.values()))
+                    cte_model = cast(NonSourceCompiledNode, cte_model)
+                    cte_model, new_prepended_ctes, orig_compiled_sql = self._recursively_prepend_ctes(
+                        cte_model, manifest, extra_context
+                    )
+                elif cte_model.is_ephemeral_model:
+                    assert isinstance(cte_model, tuple(COMPILED_TYPES))
+                    data = cte_model.to_dict()
+                    data.update({
+                        'compiled': False,
+                        'compiled_sql': None,
+                        'extra_ctes_injected': False,
+                        'extra_ctes': [],
+                    })
+                    compiled_node = _compiled_type_for(cte_model).from_dict(data)
+                    context = self._create_node_context(
+                        compiled_node, manifest, extra_context
+                    )
+                    compiled_node.compiled_sql = jinja.get_rendered(
+                        cte_model.raw_sql,
+                        context,
+                        cte_model,
+                    )
+
+                    compiled_node.compiled = True
+                    if isinstance(compiled_node, CompiledDataTestNode):
+                        name = self._get_dbt_test_name()
+                        cte = InjectedCTE(
+                            id=name,
+                            sql=f' {name} as (\n{compiled_node.compiled_sql}\n)'
+                        )
+                        compiled_node.extra_ctes.append(cte)
+                        compiled_node.compiled_sql = f'\nselect count(*) from {name}'
+
+                    injected_node, new_prepended_ctes, orig_compiled_sql = self._recursively_prepend_ctes(
+                        compiled_node, manifest, extra_context
+                    )
+                    node = injected_node
+                    manifest.sync_update_node(node)
+                    cte_model = node
+                else:
+                    raise InternalException(
+                        f'During compilation, found an uncompiled cte that '
+                        f'was not an ephemeral model: {cte.id}'
+                    )
+
+
                 _extend_prepended_ctes(prepended_ctes, new_prepended_ctes)
 
                 new_cte_name = self.add_ephemeral_prefix(cte_model.name)
-                sql = f' {new_cte_name} as (\n{cte_model.compiled_sql}\n)'
+                sql = f' {new_cte_name} as (\n{orig_compiled_sql}\n)'
+
             _add_prepended_cte(prepended_ctes, InjectedCTE(id=cte.id, sql=sql))
 
-        model = self._model_prepend_ctes(model, prepended_ctes)
+        if model.compiled_sql is None:
+            raise RuntimeException(
+                'Cannot prepend ctes to an unparsed node', model
+            )
+        injected_sql = self._inject_ctes_into_sql(
+            model.compiled_sql,
+            prepended_ctes,
+        )
+        model.extra_ctes_injected = True
+        model.extra_ctes = prepended_ctes
+        orig_compiled_sql = model.compiled_sql
+        model.compiled_sql = injected_sql
+        model.validate(model.to_dict())
 
         manifest.update_node(model)
 
-        return model, prepended_ctes
+        return model, prepended_ctes, orig_compiled_sql
 
     def _insert_ctes(
         self,
@@ -352,7 +359,7 @@ class Compiler:
             compiled_node.extra_ctes.append(cte)
             compiled_node.compiled_sql = f'\nselect count(*) from {name}'
 
-        injected_node, _ = self._recursively_prepend_ctes(
+        injected_node, _, _ = self._recursively_prepend_ctes(
             compiled_node, manifest, extra_context
         )
         return injected_node
@@ -374,7 +381,6 @@ class Compiler:
             'compiled_sql': None,
             'extra_ctes_injected': False,
             'extra_ctes': [],
-            'injected_sql': None,
         })
         compiled_node = _compiled_type_for(node).from_dict(data)
 
@@ -454,18 +460,16 @@ class Compiler:
             return node
         logger.debug(f'Writing injected SQL for node "{node.unique_id}"')
 
-        if node.injected_sql is None:
-            # this should not really happen, but it'd be a shame to crash
-            # over it
+        if node.compiled_sql is None:
             logger.error(
-                f'Compiled node "{node.unique_id}" had no injected_sql, '
+                f'Compiled node "{node.unique_id}" had no compiled_sql, '
                 'cannot write sql!'
             )
         else:
             node.build_path = node.write_node(
                 self.config.target_path,
                 'compiled',
-                node.injected_sql
+                node.compiled_sql
             )
         return node
 
@@ -484,7 +488,7 @@ class Compiler:
 
 
 def _is_writable(node):
-    if not node.injected_sql:
+    if not node.compiled_sql:
         return False
 
     if node.resource_type == NodeType.Snapshot:

core/dbt/context/providers.py

@@ -1217,7 +1217,7 @@ class ModelContext(ProviderContext):
 
     @contextproperty
     def sql(self) -> Optional[str]:
-        return getattr(self.model, 'injected_sql', None)
+        return getattr(self.model, 'compiled_sql', None)
 
     @contextproperty
     def database(self) -> str:

core/dbt/contracts/graph/compiled.py

@@ -42,7 +42,6 @@ class CompiledNode(ParsedNode, CompiledNodeMixin):
     compiled_sql: Optional[str] = None
     extra_ctes_injected: bool = False
     extra_ctes: List[InjectedCTE] = field(default_factory=list)
-    injected_sql: Optional[str] = None
 
     def set_cte(self, cte_id: str, sql: str):
         """This is the equivalent of what self.extra_ctes[cte_id] = sql would

core/dbt/contracts/graph/model_config.py

@@ -481,12 +481,27 @@ class SnapshotWrapper(JsonSchemaMixin):
 
     @classmethod
     def validate(cls, data: Any):
-        schema = _validate_schema(cls)
+        config = data.get('config', {})
+
+        if config.get('strategy') == 'check':
+            schema = _validate_schema(CheckSnapshotConfig)
+            to_validate = config
+
+        elif config.get('strategy') == 'timestamp':
+            schema = _validate_schema(TimestampSnapshotConfig)
+            to_validate = config
+
+        else:
+            schema = _validate_schema(cls)
+            to_validate = data
+
         validator = jsonschema.Draft7Validator(schema)
+
         error = jsonschema.exceptions.best_match(
-            validator.iter_errors(data),
+            validator.iter_errors(to_validate),
             key=_relevance_without_strategy,
         )
+
         if error is not None:
             raise ValidationError.create_from(error) from error
 

core/dbt/exceptions.py

@@ -132,7 +132,7 @@ class RuntimeException(RuntimeError, Exception):
         result.update({
             'raw_sql': self.node.raw_sql,
             # the node isn't always compiled, but if it is, include that!
-            'compiled_sql': getattr(self.node, 'injected_sql', None),
+            'compiled_sql': getattr(self.node, 'compiled_sql', None),
         })
         return result
 

core/dbt/include/global_project/macros/materializations/snapshot/snapshot.sql

@@ -216,7 +216,7 @@
 
   {% if not target_relation_exists %}
 
-      {% set build_sql = build_snapshot_table(strategy, model['injected_sql']) %}
+      {% set build_sql = build_snapshot_table(strategy, model['compiled_sql']) %}
       {% set final_sql = create_table_as(False, target_relation, build_sql) %}
 
   {% else %}

core/dbt/include/global_project/macros/materializations/snapshot/strategies.sql

@@ -106,7 +106,7 @@
 
 
 {% macro snapshot_check_all_get_existing_columns(node, target_exists) -%}
-    {%- set query_columns = get_columns_in_query(node['injected_sql']) -%}
+    {%- set query_columns = get_columns_in_query(node['compiled_sql']) -%}
     {%- if not target_exists -%}
         {# no table yet -> return whatever the query does #}
         {{ return([false, query_columns]) }}

core/dbt/rpc/node_runners.py

@@ -65,7 +65,7 @@ class RPCCompileRunner(GenericRPCRunner[RemoteCompileResult]):
     def execute(self, compiled_node, manifest) -> RemoteCompileResult:
         return RemoteCompileResult(
             raw_sql=compiled_node.raw_sql,
-            compiled_sql=compiled_node.injected_sql,
+            compiled_sql=compiled_node.compiled_sql,
             node=compiled_node,
             timing=[],  # this will get added later
             logs=[],
@@ -88,7 +88,7 @@ class RPCCompileRunner(GenericRPCRunner[RemoteCompileResult]):
 class RPCExecuteRunner(GenericRPCRunner[RemoteRunResult]):
     def execute(self, compiled_node, manifest) -> RemoteRunResult:
         _, execute_result = self.adapter.execute(
-            compiled_node.injected_sql, fetch=True
+            compiled_node.compiled_sql, fetch=True
         )
 
         table = ResultTable(
@@ -98,7 +98,7 @@ class RPCExecuteRunner(GenericRPCRunner[RemoteRunResult]):
 
         return RemoteRunResult(
             raw_sql=compiled_node.raw_sql,
-            compiled_sql=compiled_node.injected_sql,
+            compiled_sql=compiled_node.compiled_sql,
             node=compiled_node,
             table=table,
             timing=[],

core/dbt/task/run.py

@@ -255,7 +255,7 @@ class RunTask(CompileTask):
     def get_hook_sql(self, adapter, hook, idx, num_hooks, extra_context):
         compiler = adapter.get_compiler()
         compiled = compiler.compile_node(hook, self.manifest, extra_context)
-        statement = compiled.injected_sql
+        statement = compiled.compiled_sql
         hook_index = hook.index or num_hooks
         hook_obj = get_hook(statement, index=hook_index)
         return hook_obj.sql or ''

core/dbt/task/runnable.py

@@ -425,6 +425,7 @@ class GraphRunnableTask(ManifestTask):
         result = self.execute_with_hooks(selected_uids)
 
         if flags.WRITE_JSON:
+            self.write_manifest()
             self.write_result(result)
 
         self.task_end_messages(result.results)

core/dbt/task/test.py

@@ -42,7 +42,7 @@ class TestRunner(CompileRunner):
 
     def execute_data_test(self, test: CompiledDataTestNode):
         res, table = self.adapter.execute(
-            test.injected_sql, auto_begin=True, fetch=True
+            test.compiled_sql, auto_begin=True, fetch=True
         )
 
         num_rows = len(table.rows)
@@ -59,7 +59,7 @@ class TestRunner(CompileRunner):
 
     def execute_schema_test(self, test: CompiledSchemaTestNode):
         res, table = self.adapter.execute(
-            test.injected_sql,
+            test.compiled_sql,
             auto_begin=True,
             fetch=True,
         )

plugins/bigquery/dbt/adapters/bigquery/impl.py

@@ -385,7 +385,7 @@ class BigQueryAdapter(BaseAdapter):
         model_database = model.get('database')
         model_schema = model.get('schema')
         model_alias = model.get('alias')
-        model_sql = model.get('injected_sql')
+        model_sql = model.get('compiled_sql')
 
         logger.debug("Model SQL ({}):\n{}".format(model_alias, model_sql))
         self.connections.create_view(
@@ -505,7 +505,7 @@ class BigQueryAdapter(BaseAdapter):
                       decorator=None):
 
         if sql_override is None:
-            sql_override = model.get('injected_sql')
+            sql_override = model.get('compiled_sql')
 
         if flags.STRICT_MODE:
             connection = self.connections.get_thread_connection()

plugins/bigquery/dbt/include/bigquery/macros/materializations/table.sql

@@ -11,7 +11,7 @@
         {{ log(table_start_time ~ ' | -> Running for day ' ~ date, info=True) }}
     {% endif %}
 
-    {% set fixed_sql = model['injected_sql'] | replace('[DBT__PARTITION_DATE]', date) %}
+    {% set fixed_sql = model['compiled_sql'] | replace('[DBT__PARTITION_DATE]', date) %}
     {% set _ = adapter.execute_model(model, 'table', fixed_sql, decorator=date) %}
   {% endfor %}
 

test/integration/015_cli_invocation_tests/test_cli_invocation.py

@@ -130,7 +130,7 @@ class TestCLIInvocationWithProfilesDir(ModelCopyingIntegrationTest):
 
         # make sure the test runs against `custom_schema`
         for test_result in res:
-            self.assertTrue(self.custom_schema, test_result.node.injected_sql)
+            self.assertTrue(self.custom_schema, test_result.node.compiled_sql)
 
 
 class TestCLIInvocationWithProjectDir(ModelCopyingIntegrationTest):

test/integration/029_docs_generate_tests/test_docs_generate.py

@@ -130,17 +130,6 @@ class TestDocsGenerate(DBTIntegrationTest):
     def models(self):
         return self.dir("models")
 
-    @property
-    def packages_config(self):
-        return {
-            'packages': [
-                {
-                    'git': 'https://github.com/fishtown-analytics/dbt-integration-project',
-                    'revision': 'dbt/0.17.0',
-                },
-            ],
-        }
-
     @property
     def project_config(self):
         return {
@@ -1114,7 +1103,6 @@ class TestDocsGenerate(DBTIntegrationTest):
                     'compiled_sql': ANY,
                     'extra_ctes_injected': True,
                     'extra_ctes': [],
-                    'injected_sql': ANY,
                     'checksum': self._checksum_file(model_sql_path),
                     'unrendered_config': unrendered_model_config,
                 },
@@ -1188,7 +1176,6 @@ class TestDocsGenerate(DBTIntegrationTest):
                     'compiled_sql': ANY,
                     'extra_ctes_injected': True,
                     'extra_ctes': [],
-                    'injected_sql': ANY,
                     'checksum': self._checksum_file(second_model_sql_path),
                     'unrendered_config': unrendered_second_config
                 },
@@ -1264,7 +1251,6 @@ class TestDocsGenerate(DBTIntegrationTest):
                     'compiled_sql': '',
                     'extra_ctes_injected': True,
                     'extra_ctes': [],
-                    'injected_sql': '',
                     'checksum': self._checksum_file(seed_path),
                     'unrendered_config': unrendered_seed_config,
                 },
@@ -1301,7 +1287,6 @@ class TestDocsGenerate(DBTIntegrationTest):
                     'compiled_sql': AnyStringWith('count(*)'),
                     'extra_ctes_injected': True,
                     'extra_ctes': [],
-                    'injected_sql': AnyStringWith('count(*)'),
                     'test_metadata': {
                         'namespace': None,
                         'name': 'not_null',
@@ -1346,7 +1331,6 @@ class TestDocsGenerate(DBTIntegrationTest):
                     'compiled_sql': AnyStringWith('select 0'),
                     'extra_ctes_injected': True,
                     'extra_ctes': [],
-                    'injected_sql': AnyStringWith('select 0'),
                     'test_metadata': {
                         'namespace': 'test',
                         'name': 'nothing',
@@ -1390,7 +1374,6 @@ class TestDocsGenerate(DBTIntegrationTest):
                     'compiled_sql': AnyStringWith('count(*)'),
                     'extra_ctes_injected': True,
                     'extra_ctes': [],
-                    'injected_sql': AnyStringWith('count(*)'),
                     'test_metadata': {
                         'namespace': None,
                         'name': 'unique',
@@ -1587,7 +1570,6 @@ class TestDocsGenerate(DBTIntegrationTest):
                     'compiled_sql': ANY,
                     'extra_ctes_injected': True,
                     'extra_ctes': [],
-                    'injected_sql': ANY,
                     'checksum': self._checksum_file(ephemeral_copy_path),
                     'unrendered_config': self.unrendered_model_config(materialized='ephemeral'),
                 },
@@ -1645,7 +1627,6 @@ class TestDocsGenerate(DBTIntegrationTest):
                     'compiled_sql': ANY,
                     'extra_ctes_injected': True,
                     'extra_ctes': [ANY],
-                    'injected_sql': ANY,
                     'checksum': self._checksum_file(ephemeral_summary_path),
                     'unrendered_config': self.unrendered_model_config(materialized='table'),
                 },
@@ -1702,7 +1683,6 @@ class TestDocsGenerate(DBTIntegrationTest):
                     'compiled_sql': ANY,
                     'extra_ctes_injected': True,
                     'extra_ctes': [],
-                    'injected_sql': ANY,
                     'checksum': self._checksum_file(view_summary_path),
                     'unrendered_config': self.unrendered_model_config(materialized='view'),
                 },
@@ -1776,7 +1756,6 @@ class TestDocsGenerate(DBTIntegrationTest):
                     'compiled_sql': '',
                     'extra_ctes_injected': True,
                     'extra_ctes': [],
-                    'injected_sql': '',
                     'checksum': self._checksum_file(seed_path),
                     'unrendered_config': self.unrendered_seed_config(),
                 },
@@ -2047,7 +2026,6 @@ class TestDocsGenerate(DBTIntegrationTest):
                     'compiled_sql': ANY,
                     'extra_ctes_injected': True,
                     'extra_ctes': [],
-                    'injected_sql': ANY,
                     'checksum': self._checksum_file(clustered_sql_path),
                     'unrendered_config': self.unrendered_model_config(
                         cluster_by=['first_name'],
@@ -2129,7 +2107,6 @@ class TestDocsGenerate(DBTIntegrationTest):
                     'compiled_sql': ANY,
                     'extra_ctes_injected': True,
                     'extra_ctes': [],
-                    'injected_sql': ANY,
                     'checksum': self._checksum_file(multi_clustered_sql_path),
                     'unrendered_config': self.unrendered_model_config(
                         cluster_by=['first_name', 'email'],
@@ -2210,7 +2187,6 @@ class TestDocsGenerate(DBTIntegrationTest):
                     'compiled_sql': ANY,
                     'extra_ctes_injected': True,
                     'extra_ctes': [],
-                    'injected_sql': ANY,
                     'checksum': self._checksum_file(nested_view_sql_path),
                     'unrendered_config': self.unrendered_model_config(materialized='view'),
                 },
@@ -2246,7 +2222,6 @@ class TestDocsGenerate(DBTIntegrationTest):
                     'compiled_sql': ANY,
                     'extra_ctes_injected': True,
                     'extra_ctes': [],
-                    'injected_sql': ANY,
                     'checksum': self._checksum_file(nested_table_sql_path),
                     'unrendered_config': self.unrendered_model_config(materialized='table'),
                 },
@@ -2323,7 +2298,6 @@ class TestDocsGenerate(DBTIntegrationTest):
                     'compiled_sql': '',
                     'extra_ctes_injected': True,
                     'extra_ctes': [],
-                    'injected_sql': '',
                     'checksum': self._checksum_file(seed_path),
                     'unrendered_config': self.unrendered_seed_config(),
                 },
@@ -2459,7 +2433,6 @@ class TestDocsGenerate(DBTIntegrationTest):
                     'compiled_sql': ANY,
                     'extra_ctes_injected': True,
                     'extra_ctes': [],
-                    'injected_sql': ANY,
                     'checksum': self._checksum_file(model_sql_path),
                     'unrendered_config': self.unrendered_model_config(bind=False, materialized='view'),
                 },
@@ -2536,7 +2509,6 @@ class TestDocsGenerate(DBTIntegrationTest):
                     'compiled_sql': ANY,
                     'extra_ctes_injected': True,
                     'extra_ctes': [],
-                    'injected_sql': ANY,
                     'checksum': self._checksum_file(seed_path),
                     'unrendered_config': self.unrendered_seed_config(),
                 },
@@ -2710,7 +2682,6 @@ class TestDocsGenerate(DBTIntegrationTest):
                     'extra_ctes': [],
                     'extra_ctes_injected': True,
                     'fqn': ['test', 'model'],
-                    'injected_sql': compiled_sql,
                     'meta': {},
                     'name': 'model',
                     'original_file_path': model_sql_path,
@@ -2799,7 +2770,6 @@ class TestDocsGenerate(DBTIntegrationTest):
                     'extra_ctes': [],
                     'extra_ctes_injected': True,
                     'fqn': ['test', 'second_model'],
-                    'injected_sql': compiled_sql,
                     'meta': {},
                     'name': 'second_model',
                     'original_file_path': second_model_sql_path,
@@ -2883,7 +2853,6 @@ class TestDocsGenerate(DBTIntegrationTest):
                     'extra_ctes': [],
                     'extra_ctes_injected': True,
                     'fqn': ['test', 'seed'],
-                    'injected_sql': '',
                     'meta': {},
                     'name': 'seed',
                     'original_file_path': seed_path,
@@ -2930,7 +2899,6 @@ class TestDocsGenerate(DBTIntegrationTest):
                     'extra_ctes': [],
                     'extra_ctes_injected': True,
                     'fqn': ['test', 'schema_test', 'not_null_model_id'],
-                    'injected_sql': AnyStringWith('id is null'),
                     'meta': {},
                     'name': 'not_null_model_id',
                     'original_file_path': model_schema_yml_path,
@@ -2985,7 +2953,6 @@ class TestDocsGenerate(DBTIntegrationTest):
                     'extra_ctes': [],
                     'extra_ctes_injected': True,
                     'fqn': ['test', 'schema_test', 'test_nothing_model_'],
-                    'injected_sql': AnyStringWith('select 0'),
                     'meta': {},
                     'name': 'test_nothing_model_',
                     'original_file_path': model_schema_yml_path,
@@ -3039,7 +3006,6 @@ class TestDocsGenerate(DBTIntegrationTest):
                     'extra_ctes': [],
                     'extra_ctes_injected': True,
                     'fqn': ['test', 'schema_test', 'unique_model_id'],
-                    'injected_sql': AnyStringWith('count(*)'),
                     'meta': {},
                     'name': 'unique_model_id',
                     'original_file_path': model_schema_yml_path,
@@ -3129,7 +3095,7 @@ class TestDocsGenerate(DBTIntegrationTest):
                         },
                     },
                     'compiled': True,
-                    'compiled_sql': ephemeral_compiled_sql,
+                    'compiled_sql': ephemeral_injected_sql,
                     'config': self.rendered_model_config(materialized='table'),
                     'sources': [],
                     'depends_on': {
@@ -3146,7 +3112,6 @@ class TestDocsGenerate(DBTIntegrationTest):
                     ],
                     'extra_ctes_injected': True,
                     'fqn': ['test', 'ephemeral_summary'],
-                    'injected_sql': ephemeral_injected_sql,
                     'meta': {},
                     'name': 'ephemeral_summary',
                     'original_file_path': ephemeral_summary_path,
@@ -3220,7 +3185,6 @@ class TestDocsGenerate(DBTIntegrationTest):
                     'extra_ctes': [],
                     'extra_ctes_injected': True,
                     'fqn': ['test', 'view_summary'],
-                    'injected_sql': view_compiled_sql,
                     'meta': {},
                     'name': 'view_summary',
                     'original_file_path': view_summary_path,
@@ -3308,7 +3272,6 @@ class TestDocsGenerate(DBTIntegrationTest):
                     'extra_ctes': [],
                     'extra_ctes_injected': True,
                     'fqn': ['test', 'seed'],
-                    'injected_sql': '',
                     'meta': {},
                     'name': 'seed',
                     'original_file_path': seed_path,

test/integration/062_defer_state_test/test_defer_state.py

@@ -64,8 +64,8 @@ class TestDeferState(DBTIntegrationTest):
 
         # with state it should work though
         results = self.run_dbt(['run', '-m', 'view_model', '--state', 'state', '--defer', '--target', 'otherschema'])
-        assert self.other_schema not in results[0].node.injected_sql
-        assert self.unique_schema() in results[0].node.injected_sql
+        assert self.other_schema not in results[0].node.compiled_sql
+        assert self.unique_schema() in results[0].node.compiled_sql
 
         with open('target/manifest.json') as fp:
             data = json.load(fp)

test/integration/100_rpc_test/test_rpc.py

@@ -66,7 +66,7 @@ class ServerProcess(dbt.flags.MP_CONTEXT.Process):
 
     def start(self):
         super().start()
-        for _ in range(60):
+        for _ in range(180):
             if self.is_up():
                 break
             time.sleep(0.5)
@@ -113,7 +113,7 @@ class HasRPCServer(DBTIntegrationTest):
         os.environ['DBT_TEST_SCHEMA_NAME_VARIABLE'] = 'test_run_schema'
         if self.should_seed:
             self.run_dbt_with_vars(['seed'], strict=False)
-        port = random.randint(20000, 65535)
+        port = random.randint(49152, 61000)
         self._server = self.ServerProcess(
             cli_vars='{{test_run_schema: {}}}'.format(self.unique_schema()),
             profiles_dir=self.test_root_dir,
@@ -178,7 +178,7 @@ class HasRPCServer(DBTIntegrationTest):
     def url(self):
         return 'http://localhost:{}/jsonrpc'.format(self._server.port)
 
-    def poll_for_result(self, request_token, request_id=1, timeout=60, state='success', logs=None):
+    def poll_for_result(self, request_token, request_id=1, timeout=180, state='success', logs=None):
         start = time.time()
         kwargs = {
             'request_token': request_token,
@@ -204,7 +204,7 @@ class HasRPCServer(DBTIntegrationTest):
                     .format(delta, state, result)
                 )
 
-    def async_query(self, _method, _sql=None, _test_request_id=1, _poll_timeout=60, macros=None, **kwargs):
+    def async_query(self, _method, _sql=None, _test_request_id=1, _poll_timeout=180, macros=None, **kwargs):
         response = self.query(_method, _sql, _test_request_id, macros, **kwargs).json()
         result = self.assertIsResult(response, _test_request_id)
         self.assertIn('request_token', result)
@@ -346,7 +346,7 @@ class HasRPCServer(DBTIntegrationTest):
         return request_token, request_id
 
     def wait_for_state(
-        self, state, timestamp, timeout=25, raise_on_timeout=True
+        self, state, timestamp, timeout=180, raise_on_timeout=True
     ):
         started = time.time()
         time.sleep(0.5)
@@ -994,7 +994,7 @@ class TestRPCTaskManagement(HasRPCServer):
         self.assertIsResult(done_query)
         sleepers = []
 
-        sleepers.append(self.get_sleep_query(duration=60, request_id=1000))
+        sleepers.append(self.get_sleep_query(duration=180, request_id=1000))
         self.assertRunning(sleepers)
 
         self.run_command_with_id('seed', 20)
@@ -1135,7 +1135,7 @@ class TestRPCServerDeps(HasRPCServer):
         status = self._check_start_predeps()
 
         # do a dbt deps, wait for the result
-        self.assertIsResult(self.async_query('deps', _poll_timeout=120).json())
+        self.assertIsResult(self.async_query('deps', _poll_timeout=180).json())
 
         self._check_deps_ok(status)
 
@@ -1145,6 +1145,6 @@ class TestRPCServerDeps(HasRPCServer):
         status = self._check_start_predeps()
 
         # do a dbt deps, wait for the result
-        self.assertIsResult(self.async_query('cli_args', cli='deps', _poll_timeout=120).json())
+        self.assertIsResult(self.async_query('cli_args', cli='deps', _poll_timeout=180).json())
 
         self._check_deps_ok(status)

test/unit/test_compiler.py

@@ -119,7 +119,6 @@ class CompilerTest(unittest.TestCase):
                     compiled=True,
                     extra_ctes_injected=False,
                     extra_ctes=[InjectedCTE(id='model.root.ephemeral', sql='select * from source_table')],
-                    injected_sql='',
                     compiled_sql=(
                         'with cte as (select * from something_else) '
                         'select * from __dbt__CTE__ephemeral'),
@@ -147,7 +146,6 @@ class CompilerTest(unittest.TestCase):
                     compiled_sql='select * from source_table',
                     extra_ctes_injected=False,
                     extra_ctes=[],
-                    injected_sql='',
                     checksum=FileHash.from_contents(''),
                 ),
             },
@@ -159,7 +157,7 @@ class CompilerTest(unittest.TestCase):
         )
 
         compiler = dbt.compilation.Compiler(self.config)
-        result, _ = compiler._recursively_prepend_ctes(
+        result, _, _ = compiler._recursively_prepend_ctes(
             manifest.nodes['model.root.view'],
             manifest,
             {}
@@ -168,7 +166,7 @@ class CompilerTest(unittest.TestCase):
         self.assertEqual(result, manifest.nodes['model.root.view'])
         self.assertEqual(result.extra_ctes_injected, True)
         self.assertEqualIgnoreWhitespace(
-            result.injected_sql,
+            result.compiled_sql,
             ('with __dbt__CTE__ephemeral as ('
              'select * from source_table'
              '), cte as (select * from something_else) '
@@ -204,7 +202,6 @@ class CompilerTest(unittest.TestCase):
                     compiled=True,
                     extra_ctes_injected=False,
                     extra_ctes=[],
-                    injected_sql='',
                     compiled_sql=('with cte as (select * from something_else) '
                                   'select * from source_table'),
                     checksum=FileHash.from_contents(''),
@@ -230,7 +227,6 @@ class CompilerTest(unittest.TestCase):
                     compiled=True,
                     extra_ctes_injected=False,
                     extra_ctes=[],
-                    injected_sql='',
                     compiled_sql=('select * from source_table'),
                     checksum=FileHash.from_contents(''),
                 ),
@@ -243,7 +239,7 @@ class CompilerTest(unittest.TestCase):
         )
 
         compiler = dbt.compilation.Compiler(self.config)
-        result, _ = compiler._recursively_prepend_ctes(
+        result, _, _ = compiler._recursively_prepend_ctes(
             manifest.nodes['model.root.view'],
             manifest,
             {}
@@ -254,11 +250,11 @@ class CompilerTest(unittest.TestCase):
             manifest.nodes.get('model.root.view'))
         self.assertTrue(result.extra_ctes_injected)
         self.assertEqualIgnoreWhitespace(
-            result.injected_sql,
+            result.compiled_sql,
             manifest.nodes.get('model.root.view').compiled_sql)
 
         compiler = dbt.compilation.Compiler(self.config)
-        result, _ = compiler._recursively_prepend_ctes(
+        result, _, _ = compiler._recursively_prepend_ctes(
             manifest.nodes.get('model.root.view_no_cte'),
             manifest,
             {})
@@ -268,7 +264,7 @@ class CompilerTest(unittest.TestCase):
             manifest.nodes.get('model.root.view_no_cte'))
         self.assertTrue(result.extra_ctes_injected)
         self.assertEqualIgnoreWhitespace(
-            result.injected_sql,
+            result.compiled_sql,
             manifest.nodes.get('model.root.view_no_cte').compiled_sql)
 
     def test__prepend_ctes(self):
@@ -298,7 +294,6 @@ class CompilerTest(unittest.TestCase):
                     compiled=True,
                     extra_ctes_injected=False,
                     extra_ctes=[InjectedCTE(id='model.root.ephemeral', sql='select * from source_table')],
-                    injected_sql='',
                     compiled_sql='select * from __dbt__CTE__ephemeral',
                     checksum=FileHash.from_contents(''),
                 ),
@@ -323,7 +318,6 @@ class CompilerTest(unittest.TestCase):
                     compiled=True,
                     extra_ctes_injected=False,
                     extra_ctes=[],
-                    injected_sql='',
                     compiled_sql='select * from source_table',
                     checksum=FileHash.from_contents(''),
                 ),
@@ -336,7 +330,7 @@ class CompilerTest(unittest.TestCase):
         )
 
         compiler = dbt.compilation.Compiler(self.config)
-        result, _ = compiler._recursively_prepend_ctes(
+        result, _, _ = compiler._recursively_prepend_ctes(
             manifest.nodes['model.root.view'],
             manifest,
             {}
@@ -347,7 +341,7 @@ class CompilerTest(unittest.TestCase):
 
         self.assertTrue(result.extra_ctes_injected)
         self.assertEqualIgnoreWhitespace(
-            result.injected_sql,
+            result.compiled_sql,
             ('with __dbt__CTE__ephemeral as ('
              'select * from source_table'
              ') '
@@ -397,7 +391,6 @@ class CompilerTest(unittest.TestCase):
             raw_sql='select * from source_table',
             compiled=True,
             compiled_sql='select * from source_table',
-            injected_sql='select * from source_table',
             extra_ctes_injected=True,
             extra_ctes=[],
             checksum=FileHash.from_contents(''),
@@ -426,7 +419,6 @@ class CompilerTest(unittest.TestCase):
                     compiled=True,
                     extra_ctes_injected=False,
                     extra_ctes=[InjectedCTE(id='model.root.ephemeral', sql='select * from source_table')],
-                    injected_sql='',
                     compiled_sql='select * from __dbt__CTE__ephemeral',
                     checksum=FileHash.from_contents(''),
                 ),
@@ -443,12 +435,11 @@ class CompilerTest(unittest.TestCase):
         with patch.object(compiler, 'compile_node') as compile_node:
             compile_node.return_value = compiled_ephemeral
 
-            result, _ = compiler._recursively_prepend_ctes(
+            result, _, _ = compiler._recursively_prepend_ctes(
                 manifest.nodes['model.root.view'],
                 manifest,
                 {}
             )
-            compile_node.assert_called_once_with(parsed_ephemeral, manifest, {})
 
         self.assertEqual(result,
                          manifest.nodes.get('model.root.view'))
@@ -456,7 +447,7 @@ class CompilerTest(unittest.TestCase):
         self.assertTrue(manifest.nodes['model.root.ephemeral'].compiled)
         self.assertTrue(result.extra_ctes_injected)
         self.assertEqualIgnoreWhitespace(
-            result.injected_sql,
+            result.compiled_sql,
             ('with __dbt__CTE__ephemeral as ('
              'select * from source_table'
              ') '
@@ -491,7 +482,6 @@ class CompilerTest(unittest.TestCase):
                     compiled=True,
                     extra_ctes_injected=False,
                     extra_ctes=[InjectedCTE(id='model.root.ephemeral', sql=None)],
-                    injected_sql=None,
                     compiled_sql='select * from __dbt__CTE__ephemeral',
                     checksum=FileHash.from_contents(''),
 
@@ -545,7 +535,7 @@ class CompilerTest(unittest.TestCase):
         )
 
         compiler = dbt.compilation.Compiler(self.config)
-        result, _ = compiler._recursively_prepend_ctes(
+        result, _, _ = compiler._recursively_prepend_ctes(
             manifest.nodes['model.root.view'],
             manifest,
             {}
@@ -554,7 +544,7 @@ class CompilerTest(unittest.TestCase):
         self.assertEqual(result, manifest.nodes['model.root.view'])
         self.assertTrue(result.extra_ctes_injected)
         self.assertEqualIgnoreWhitespace(
-            result.injected_sql,
+            result.compiled_sql,
             ('with __dbt__CTE__ephemeral_level_two as ('
              'select * from source_table'
              '), __dbt__CTE__ephemeral as ('

test/unit/test_contracts_graph_compiled.py

@@ -74,10 +74,9 @@ def basic_compiled_model():
         config=NodeConfig(),
         meta={},
         compiled=True,
-        compiled_sql='select * from whatever',
         extra_ctes=[InjectedCTE('whatever', 'select * from other')],
         extra_ctes_injected=True,
-        injected_sql='with whatever as (select * from other) select * from whatever',
+        compiled_sql='with whatever as (select * from other) select * from whatever',
         checksum=FileHash.from_contents(''),
         unrendered_config={}
     )
@@ -183,10 +182,9 @@ def basic_compiled_dict():
         'columns': {},
         'meta': {},
         'compiled': True,
-        'compiled_sql': 'select * from whatever',
         'extra_ctes': [{'id': 'whatever', 'sql': 'select * from other'}],
         'extra_ctes_injected': True,
-        'injected_sql': 'with whatever as (select * from other) select * from whatever',
+        'compiled_sql': 'with whatever as (select * from other) select * from whatever',
         'checksum': {'name': 'sha256', 'checksum': 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855'},
         'unrendered_config': {}
     }
@@ -375,10 +373,9 @@ def basic_compiled_schema_test_node():
         config=TestConfig(severity='warn'),
         meta={},
         compiled=True,
-        compiled_sql='select * from whatever',
         extra_ctes=[InjectedCTE('whatever', 'select * from other')],
         extra_ctes_injected=True,
-        injected_sql='with whatever as (select * from other) select * from whatever',
+        compiled_sql='with whatever as (select * from other) select * from whatever',
         column_name='id',
         test_metadata=TestMetadata(namespace=None, name='foo', kwargs={}),
         checksum=FileHash.from_contents(''),
@@ -474,10 +471,9 @@ def basic_compiled_schema_test_dict():
         'columns': {},
         'meta': {},
         'compiled': True,
-        'compiled_sql': 'select * from whatever',
         'extra_ctes': [{'id': 'whatever', 'sql': 'select * from other'}],
         'extra_ctes_injected': True,
-        'injected_sql': 'with whatever as (select * from other) select * from whatever',
+        'compiled_sql': 'with whatever as (select * from other) select * from whatever',
         'column_name': 'id',
         'test_metadata': {
             'name': 'foo',

test/unit/test_contracts_graph_parsed.py

@@ -1348,7 +1348,8 @@ def test_invalid_check_wrong_strategy(basic_check_snapshot_config_dict):
 def test_invalid_missing_check_cols(basic_check_snapshot_config_dict):
     wrong_fields = basic_check_snapshot_config_dict
     del wrong_fields['check_cols']
-    assert_fails_validation(wrong_fields, CheckSnapshotConfig)
+    with pytest.raises(ValidationError, match=r"'check_cols' is a required property"):
+        CheckSnapshotConfig.from_dict(wrong_fields)
 
 
 def test_invalid_check_value(basic_check_snapshot_config_dict):

test/unit/test_manifest.py

@@ -38,7 +38,6 @@ REQUIRED_PARSED_NODE_KEYS = frozenset({
 
 REQUIRED_COMPILED_NODE_KEYS = frozenset(REQUIRED_PARSED_NODE_KEYS | {
     'compiled', 'extra_ctes_injected', 'extra_ctes', 'compiled_sql',
-    'injected_sql',
 })
 
 
@@ -482,7 +481,6 @@ class MixedManifestTest(unittest.TestCase):
                 compiled=True,
                 compiled_sql='also does not matter',
                 extra_ctes_injected=True,
-                injected_sql=None,
                 extra_ctes=[],
                 checksum=FileHash.empty(),
             ),
@@ -508,7 +506,6 @@ class MixedManifestTest(unittest.TestCase):
                 compiled=True,
                 compiled_sql='also does not matter',
                 extra_ctes_injected=True,
-                injected_sql='and this also does not matter',
                 extra_ctes=[],
                 checksum=FileHash.empty(),
             ),