What should docs generate --defer really do?

core/dbt/cli/main.py

@@ -101,6 +101,7 @@ def cli(ctx, **kwargs):
 @cli.command("build")
 @click.pass_context
 @p.defer
+@p.favor_defer_state
 @p.exclude
 @p.fail_fast
 @p.full_refresh
@@ -169,6 +170,7 @@ def docs(ctx, **kwargs):
 @click.pass_context
 @p.compile_docs
 @p.defer
+@p.favor_defer_state
 @p.exclude
 @p.models
 @p.profile
@@ -232,6 +234,7 @@ def docs_serve(ctx, **kwargs):
 @cli.command("compile")
 @click.pass_context
 @p.defer
+@p.favor_defer_state
 @p.exclude
 @p.full_refresh
 @p.models
@@ -401,6 +404,7 @@ def parse(ctx, **kwargs):
 @cli.command("run")
 @click.pass_context
 @p.defer
+@p.favor_defer_state
 @p.exclude
 @p.fail_fast
 @p.full_refresh
@@ -502,6 +506,7 @@ def seed(ctx, **kwargs):
 @cli.command("snapshot")
 @click.pass_context
 @p.defer
+@p.favor_defer_state
 @p.exclude
 @p.models
 @p.profile
@@ -581,6 +586,7 @@ cli.commands["source"].add_command(snapshot_freshness, "snapshot-freshness")  #
 @cli.command("test")
 @click.pass_context
 @p.defer
+@p.favor_defer_state
 @p.exclude
 @p.fail_fast
 @p.indirect_selection

core/dbt/cli/params.py

@@ -75,6 +75,13 @@ defer = click.option(
     help="If set, defer to the state variable for resolving unselected nodes.",
 )
 
+favor_defer_state = click.option(
+    "--favor-state/--no-favor-state",
+    envvar="DBT_FAVOR_STATE",
+    help="If favor state, defer to the state variable for resolving unselected nodes, even if node exist as a database object in the current environment. Otherwise expect standard defer behavior",
+)
+
+
 enable_legacy_logger = click.option(
     "--enable-legacy-logger/--no-enable-legacy-logger",
     envvar="DBT_ENABLE_LEGACY_LOGGER",

core/dbt/docs/build/html/index.html

@@ -66,6 +66,11 @@
 <p>Type: boolean</p>
 <p>If set, defer to the state variable for resolving unselected nodes.</p>
 </section>
+<section id="build|favor_state">
+<h4>favor_state<a class="headerlink" href="#build|favor_state" title="Permalink to this heading">¶</a></h4>
+<p>Type: boolean</p>
+<p>If favor state, defer to the state variable for resolving unselected nodes, even if node exist as a database object in the current environment. Otherwise expect standard defer behavior</p>
+</section>
 <section id="build|exclude">
 <h4>exclude<a class="headerlink" href="#build|exclude" title="Permalink to this heading">¶</a></h4>
 <p>Type: unknown</p>
@@ -83,7 +88,7 @@
 </section>
 <section id="build|indirect_selection">
 <h4>indirect_selection<a class="headerlink" href="#build|indirect_selection" title="Permalink to this heading">¶</a></h4>
-<p>Type: choice: [‘eager’, ‘cautious’]</p>
+<p>Type: choice: [‘eager’, ‘cautious’, ‘buildable’]</p>
 <p>Select all tests that are adjacent to selected resources, even if they those resources have been explicitly selected.</p>
 </section>
 <section id="build|profile">
@@ -188,6 +193,11 @@
 <p>Type: boolean</p>
 <p>If set, defer to the state variable for resolving unselected nodes.</p>
 </section>
+<section id="compile|favor_state">
+<h4>favor_state<a class="headerlink" href="#compile|favor_state" title="Permalink to this heading">¶</a></h4>
+<p>Type: boolean</p>
+<p>If favor state, defer to the state variable for resolving unselected nodes, even if node exist as a database object in the current environment. Otherwise expect standard defer behavior</p>
+</section>
 <section id="compile|exclude">
 <h4>exclude<a class="headerlink" href="#compile|exclude" title="Permalink to this heading">¶</a></h4>
 <p>Type: unknown</p>
@@ -349,7 +359,7 @@
 <section id="init|skip_profile_setup">
 <h4>skip_profile_setup<a class="headerlink" href="#init|skip_profile_setup" title="Permalink to this heading">¶</a></h4>
 <p>Type: boolean</p>
-<p>Skip interative profile setup.</p>
+<p>Skip interactive profile setup.</p>
 </section>
 <section id="init|target">
 <h4>target<a class="headerlink" href="#init|target" title="Permalink to this heading">¶</a></h4>
@@ -369,7 +379,7 @@
 </section>
 <section id="list|indirect_selection">
 <h4>indirect_selection<a class="headerlink" href="#list|indirect_selection" title="Permalink to this heading">¶</a></h4>
-<p>Type: choice: [‘eager’, ‘cautious’]</p>
+<p>Type: choice: [‘eager’, ‘cautious’, ‘buildable’]</p>
 <p>Select all tests that are adjacent to selected resources, even if they those resources have been explicitly selected.</p>
 </section>
 <section id="list|models">
@@ -440,7 +450,7 @@
 </section>
 <section id="list|indirect_selection">
 <h4>indirect_selection<a class="headerlink" href="#list|indirect_selection" title="Permalink to this heading">¶</a></h4>
-<p>Type: choice: [‘eager’, ‘cautious’]</p>
+<p>Type: choice: [‘eager’, ‘cautious’, ‘buildable’]</p>
 <p>Select all tests that are adjacent to selected resources, even if they those resources have been explicitly selected.</p>
 </section>
 <section id="list|models">
@@ -560,6 +570,11 @@
 <p>Type: boolean</p>
 <p>If set, defer to the state variable for resolving unselected nodes.</p>
 </section>
+<section id="run|favor_state">
+<h4>favor_state<a class="headerlink" href="#run|favor_state" title="Permalink to this heading">¶</a></h4>
+<p>Type: boolean</p>
+<p>If favor state, defer to the state variable for resolving unselected nodes, even if node exist as a database object in the current environment. Otherwise expect standard defer behavior</p>
+</section>
 <section id="run|exclude">
 <h4>exclude<a class="headerlink" href="#run|exclude" title="Permalink to this heading">¶</a></h4>
 <p>Type: unknown</p>
@@ -752,6 +767,11 @@
 <p>Type: boolean</p>
 <p>If set, defer to the state variable for resolving unselected nodes.</p>
 </section>
+<section id="snapshot|favor_state">
+<h4>favor_state<a class="headerlink" href="#snapshot|favor_state" title="Permalink to this heading">¶</a></h4>
+<p>Type: boolean</p>
+<p>If favor state, defer to the state variable for resolving unselected nodes, even if node exist as a database object in the current environment. Otherwise expect standard defer behavior</p>
+</section>
 <section id="snapshot|exclude">
 <h4>exclude<a class="headerlink" href="#snapshot|exclude" title="Permalink to this heading">¶</a></h4>
 <p>Type: unknown</p>
@@ -814,6 +834,11 @@
 <p>Type: boolean</p>
 <p>If set, defer to the state variable for resolving unselected nodes.</p>
 </section>
+<section id="test|favor_state">
+<h4>favor_state<a class="headerlink" href="#test|favor_state" title="Permalink to this heading">¶</a></h4>
+<p>Type: boolean</p>
+<p>If favor state, defer to the state variable for resolving unselected nodes, even if node exist as a database object in the current environment. Otherwise expect standard defer behavior</p>
+</section>
 <section id="test|exclude">
 <h4>exclude<a class="headerlink" href="#test|exclude" title="Permalink to this heading">¶</a></h4>
 <p>Type: unknown</p>
@@ -826,7 +851,7 @@
 </section>
 <section id="test|indirect_selection">
 <h4>indirect_selection<a class="headerlink" href="#test|indirect_selection" title="Permalink to this heading">¶</a></h4>
-<p>Type: choice: [‘eager’, ‘cautious’]</p>
+<p>Type: choice: [‘eager’, ‘cautious’, ‘buildable’]</p>
 <p>Select all tests that are adjacent to selected resources, even if they those resources have been explicitly selected.</p>
 </section>
 <section id="test|models">

core/dbt/task/generate.py

@@ -1,7 +1,7 @@
 import os
 import shutil
 from datetime import datetime
-from typing import Dict, List, Any, Optional, Tuple, Set
+from typing import Dict, List, Any, Optional, Tuple, Set, AbstractSet
 
 from dbt.dataclass_schema import ValidationError
 
@@ -229,6 +229,13 @@ class GenerateTask(CompileTask):
 
         adapter = get_adapter(self.config)
         with adapter.connection_named("generate_catalog"):
+            if self.args.defer:
+                # Look up catalog entries only on the basis of whether they exist in the
+                # target schema, ignoring selection logic passed to `docs generate` -> `compile`
+                # Unfortunately, this requires running caching queries again,
+                # even though we may have just run them during CompileTask.run()
+                self.populate_adapter_cache(adapter)
+                self.defer_to_manifest(adapter, selected_uids=set())
             fire_event(BuildingCatalog())
             catalog_table, exceptions = adapter.get_catalog(self.manifest)
 

tests/functional/defer_state/test_defer_state.py

@@ -6,6 +6,7 @@ from copy import deepcopy
 import pytest
 
 from dbt.tests.util import run_dbt, write_file, rm_file
+from dbt.cli.main import dbtUsageException
 
 from dbt.exceptions import DbtRuntimeError
 
@@ -98,7 +99,7 @@ class BaseDeferState:
 class TestDeferStateUnsupportedCommands(BaseDeferState):
     def test_unsupported_commands(self, project):
         # make sure these commands don"t work with --defer
-        with pytest.raises(SystemExit):
+        with pytest.raises(dbtUsageException):
             run_dbt(["seed", "--defer"])
 
     def test_no_state(self, project):
@@ -149,7 +150,7 @@ class TestRunDeferState(BaseDeferState):
 
         # defer test, it succeeds
         results = run_dbt(
-            ["test", "-m", "view_model+", "--state", "state", "--defer", "--target", "otherschema"]
+            ["test", "-s", "view_model+", "--state", "state", "--defer", "--target", "otherschema"]
         )
 
         # defer docs generate with state, catalog refers schema from the happy times
@@ -157,7 +158,7 @@ class TestRunDeferState(BaseDeferState):
             [
                 "docs",
                 "generate",
-                "-m",
+                "-s",
                 "view_model+",
                 "--state",
                 "state",
@@ -171,7 +172,7 @@ class TestRunDeferState(BaseDeferState):
 
         # with state it should work though
         results = run_dbt(
-            ["run", "-m", "view_model", "--state", "state", "--defer", "--target", "otherschema"]
+            ["run", "-s", "view_model", "--state", "state", "--defer", "--target", "otherschema"]
         )
         assert other_schema not in results[0].node.compiled_code
         assert unique_schema in results[0].node.compiled_code
@@ -192,12 +193,12 @@ class TestRunDeferStateChangedModel(BaseDeferState):
 
         # the sql here is just wrong, so it should fail
         run_dbt(
-            ["run", "-m", "view_model", "--state", "state", "--defer", "--target", "otherschema"],
+            ["run", "-s", "view_model", "--state", "state", "--defer", "--target", "otherschema"],
             expect_pass=False,
         )
         # but this should work since we just use the old happy model
         run_dbt(
-            ["run", "-m", "table_model", "--state", "state", "--defer", "--target", "otherschema"],
+            ["run", "-s", "table_model", "--state", "state", "--defer", "--target", "otherschema"],
             expect_pass=True,
         )
 
@@ -206,7 +207,7 @@ class TestRunDeferStateChangedModel(BaseDeferState):
         # this should fail because the table model refs a broken ephemeral
         # model, which it should see
         run_dbt(
-            ["run", "-m", "table_model", "--state", "state", "--defer", "--target", "otherschema"],
+            ["run", "-s", "table_model", "--state", "state", "--defer", "--target", "otherschema"],
             expect_pass=False,
         )
 
@@ -246,7 +247,7 @@ class TestDeferStateDeletedUpstream(BaseDeferState):
         # ephemeral_model is now gone. previously this caused a
         # keyerror (dbt#2875), now it should pass
         run_dbt(
-            ["run", "-m", "view_model", "--state", "state", "--defer", "--target", "otherschema"],
+            ["run", "-s", "view_model", "--state", "state", "--defer", "--target", "otherschema"],
             expect_pass=True,
         )
 
@@ -258,7 +259,7 @@ class TestDeferStateDeletedUpstream(BaseDeferState):
         run_dbt(
             [
                 "run",
-                "-m",
+                "-s",
                 "view_model",
                 "--state",
                 "state",