work

* added test that fails

* added new exception

* add partial error checking - needs more specifics

* move contract check under modelnode

* try adding only enforced constraints

* add checks for enforced constraints

* changelog

* add materialization logic

* clean up tests, tweak materializations

* PR feedback

* more PR feedback

* change to tuple

.bumpversion.cfg

@@ -1,13 +1,19 @@
 [bumpversion]
-current_version = 1.5.0a1
-parse = (?P<major>\d+)
-	\.(?P<minor>\d+)
-	\.(?P<patch>\d+)
-	((?P<prekind>a|b|rc)
-	(?P<pre>\d+)  # pre-release version num
+current_version = 1.6.0a1
+parse = (?P<major>[\d]+) # major version number
+	\.(?P<minor>[\d]+) # minor version number
+	\.(?P<patch>[\d]+) # patch version number
+	(?P<prerelease>  # optional pre-release - ex: a1, b2, rc25
+	(?P<prekind>a|b|rc) # pre-release type
+	(?P<num>[\d]+) # pre-release version number
 	)?
+	( # optional nightly release indicator
+	\.(?P<nightly>dev[0-9]+) # ex: .dev02142023
+	)? # expected matches: `1.15.0`, `1.5.0a11`, `1.5.0a1.dev123`, `1.5.0.dev123457`, expected failures: `1`, `1.5`, `1.5.2-a1`, `text1.5.0`
 serialize =
-	{major}.{minor}.{patch}{prekind}{pre}
+	{major}.{minor}.{patch}{prekind}{num}.{nightly}
+	{major}.{minor}.{patch}.{nightly}
+	{major}.{minor}.{patch}{prekind}{num}
 	{major}.{minor}.{patch}
 commit = False
 tag = False
@@ -21,9 +27,11 @@ values =
 	rc
 	final
 
-[bumpversion:part:pre]
+[bumpversion:part:num]
 first_value = 1
 
+[bumpversion:part:nightly]
+
 [bumpversion:file:core/setup.py]
 
 [bumpversion:file:core/dbt/version.py]

.changes/1.6.0-a1.md

@@ -0,0 +1 @@
+## dbt-core 1.6.0-a1 - April 17, 2023

.changes/unreleased/Features-20230321-213338.yaml

@@ -0,0 +1,6 @@
+kind: Features
+body: Skip catalog generation
+time: 2023-03-21T21:33:38.513443Z
+custom:
+  Author: AndyBys
+  Issue: "6980"

.changes/unreleased/Features-20230420-124756.yaml

@@ -0,0 +1,6 @@
+kind: Features
+body: Publication artifacts and cross-project ref
+time: 2023-04-20T12:47:56.92683-04:00
+custom:
+  Author: gshank
+  Issue: "7227"

.changes/unreleased/Features-20230425-142522.yaml

@@ -0,0 +1,6 @@
+kind: Features
+body: Add graph structure summaries to target path output
+time: 2023-04-25T14:25:22.269051-04:00
+custom:
+  Author: peterallenwebb
+  Issue: "7357"

.changes/unreleased/Features-20230429-155057.yaml

@@ -0,0 +1,7 @@
+kind: Features
+body: Allow duplicate manifest node (models, seeds, analyses, snapshots) names across
+  packages
+time: 2023-04-29T15:50:57.757832-04:00
+custom:
+  Author: MichelleArk
+  Issue: "7446"

.changes/unreleased/Features-20230503-101100.yaml

@@ -0,0 +1,6 @@
+kind: Features
+body: Detect breaking changes to enforced constraints
+time: 2023-05-03T10:11:00.617936-05:00
+custom:
+  Author: emmyoop
+  Issue: "7065"

.changes/unreleased/Features-20230509-094147.yaml

@@ -0,0 +1,6 @@
+kind: Features
+body: Check for project dependency cycles
+time: 2023-05-09T09:41:47.2-04:00
+custom:
+  Author: gshank
+  Issue: "7468"

.changes/unreleased/Fixes-20230413-133157.yaml

@@ -0,0 +1,6 @@
+kind: Fixes
+body: Persist timing info in run results for failed nodes
+time: 2023-04-13T13:31:57.938847-05:00
+custom:
+  Author: stu-k
+  Issue: "5476"

.changes/unreleased/Fixes-20230414-163642.yaml

@@ -0,0 +1,6 @@
+kind: Fixes
+body: fix typo in unpacking statically parsed ref
+time: 2023-04-14T16:36:42.279838-04:00
+custom:
+  Author: MichelleArk
+  Issue: "7364"

.changes/unreleased/Fixes-20230418-135257.yaml

@@ -0,0 +1,6 @@
+kind: Fixes
+body: safe version attribute access in _check_resource_uniqueness
+time: 2023-04-18T13:52:57.367108-04:00
+custom:
+  Author: MichelleArk
+  Issue: "7375"

.changes/unreleased/Fixes-20230419-142150.yaml

@@ -0,0 +1,6 @@
+kind: Fixes
+body: Fix dbt command missing target-path param
+time: 2023-04-19T14:21:50.959786-07:00
+custom:
+  Author: ChenyuLInx
+  Issue: "\t7411"

.changes/unreleased/Fixes-20230419-220910.yaml

@@ -0,0 +1,6 @@
+kind: Fixes
+body: Fix v0 ref resolution
+time: 2023-04-19T22:09:10.155137-04:00
+custom:
+  Author: MichelleArk
+  Issue: "7408"

.changes/unreleased/Fixes-20230420-104254.yaml

@@ -0,0 +1,6 @@
+kind: Fixes
+body: Add --target-path to dbt snapshot command.
+time: 2023-04-20T10:42:54.17972-04:00
+custom:
+  Author: dwreeves
+  Issue: "7418"

.changes/unreleased/Fixes-20230421-172428.yaml

@@ -0,0 +1,6 @@
+kind: Fixes
+body: dbt build selection of tests' descendants
+time: 2023-04-21T17:24:28.335866975+02:00
+custom:
+  Author: b-luu
+  Issue: "7289"

.changes/unreleased/Fixes-20230424-161843.yaml

@@ -0,0 +1,6 @@
+kind: Fixes
+body: fix groupable node partial parsing, raise DbtReferenceError at runtime for safety
+time: 2023-04-24T16:18:43.130637-04:00
+custom:
+  Author: MichelleArk
+  Issue: "7437"

.changes/unreleased/Fixes-20230424-164649.yaml

@@ -0,0 +1,6 @@
+kind: Fixes
+body: Fix partial parsing of latest_version changes for downstream references
+time: 2023-04-24T16:46:49.721231-04:00
+custom:
+  Author: MichelleArk
+  Issue: "7369"

.changes/unreleased/Fixes-20230424-173404.yaml

@@ -0,0 +1,6 @@
+kind: Fixes
+body: Use "add_node" to update depends_on.nodes
+time: 2023-04-24T17:34:04.37479-04:00
+custom:
+  Author: gshank
+  Issue: "7453"

.changes/unreleased/Fixes-20230427-230714.yaml

@@ -0,0 +1,6 @@
+kind: Fixes
+body: 'Fix var precedence in configs: root vars override package vars'
+time: 2023-04-27T23:07:14.992529-04:00
+custom:
+  Author: MichelleArk
+  Issue: "6705"

.changes/unreleased/Fixes-20230505-132545.yaml

@@ -0,0 +1,6 @@
+kind: Fixes
+body: Fix inverted `--print/--no-print` flag
+time: 2023-05-05T13:25:45.949997-06:00
+custom:
+  Author: dbeatty10 thomasgjerdekog
+  Issue: "7517"

.changes/unreleased/Fixes-20230506-173315.yaml

@@ -0,0 +1,6 @@
+kind: Fixes
+body: Back-compat for previous return type of 'collect_freshness' macro
+time: 2023-05-06T17:33:15.104953+02:00
+custom:
+  Author: jtcohen6
+  Issue: "7489"

.changes/unreleased/Fixes-20230508-042707.yaml

@@ -0,0 +1,6 @@
+kind: Fixes
+body: print model version in dbt show if specified
+time: 2023-05-08T04:27:07.9965-07:00
+custom:
+  Author: aranke
+  Issue: "7407"

.changes/unreleased/Fixes-20230508-044922.yaml

@@ -0,0 +1,6 @@
+kind: Fixes
+body: enable dbt show for seeds
+time: 2023-05-08T04:49:22.82093-07:00
+custom:
+  Author: aranke
+  Issue: "7273"

.changes/unreleased/Fixes-20230508-060926.yaml

@@ -0,0 +1,6 @@
+kind: Fixes
+body: push down limit filtering to adapter
+time: 2023-05-08T06:09:26.455524-07:00
+custom:
+  Author: aranke
+  Issue: "7390"

.changes/unreleased/Fixes-20230508-093732.yaml

@@ -0,0 +1,6 @@
+kind: Fixes
+body: '`run_results.json` is now written after every node completes.'
+time: 2023-05-08T09:37:32.809356-05:00
+custom:
+  Author: iknox-fa
+  Issue: "7302"

.changes/unreleased/Fixes-20230508-142518.yaml

@@ -0,0 +1,6 @@
+kind: Fixes
+body: Do not rewrite manifest.json during 'docs serve' command
+time: 2023-05-08T14:25:18.376379-04:00
+custom:
+  Author: jtcohen6
+  Issue: "7553"

.changes/unreleased/Fixes-20230509-102932.yaml

@@ -0,0 +1,6 @@
+kind: Fixes
+body: inject sql header in query for show
+time: 2023-05-09T10:29:32.12947-07:00
+custom:
+  Author: aranke
+  Issue: "7413"

.changes/unreleased/Fixes-20230509-123159.yaml

@@ -0,0 +1,6 @@
+kind: Fixes
+body: Pin protobuf to greater than 4.0.0
+time: 2023-05-09T12:31:59.019718-04:00
+custom:
+  Author: gshank
+  Issue: "7565"

.changes/unreleased/Fixes-20230509-153530.yaml

@@ -0,0 +1,6 @@
+kind: Fixes
+body: Pin urllib3 to ~=1.0
+time: 2023-05-09T15:35:30.504674-04:00
+custom:
+  Author: mikealfare
+  Issue: "7573"

.changes/unreleased/Fixes-20230509-165007.yaml

@@ -0,0 +1,6 @@
+kind: Fixes
+body: Throw error for duplicated versioned and unversioned models
+time: 2023-05-09T16:50:07.530882-04:00
+custom:
+  Author: gshank
+  Issue: "7487"

.changes/unreleased/Under the Hood-20230113-132513.yaml

@@ -1,6 +0,0 @@
-kind: Under the Hood
-body: Fix use of ConnectionReused logging event
-time: 2023-01-13T13:25:13.023168-05:00
-custom:
-  Author: gshank
-  Issue: "6168"

.changes/unreleased/Under the Hood-20230117-111737.yaml

@@ -1,6 +0,0 @@
-kind: Under the Hood
-body: Update deprecated github action command
-time: 2023-01-17T11:17:37.046095-06:00
-custom:
-  Author: davidbloss
-  Issue: "6153"

.changes/unreleased/Under the Hood-20230417-114501.yaml

@@ -0,0 +1,6 @@
+kind: Under the Hood
+body: Update docs link in ContractBreakingChangeError message
+time: 2023-04-17T11:45:01.005104+02:00
+custom:
+  Author: jtcohen6
+  Issue: "7366"

.changes/unreleased/Under the Hood-20230417-122721.yaml

@@ -0,0 +1,6 @@
+kind: Under the Hood
+body: Reduce memory footprint of cached statement results.
+time: 2023-04-17T12:27:21.972268-05:00
+custom:
+  Author: iknox-fa
+  Issue: "7281"

.changes/unreleased/Under the Hood-20230418-121236.yaml

@@ -0,0 +1,7 @@
+kind: Under the Hood
+body: 'Remove noisy parsing events: GenericTestFileParse, MacroFileParse, Note events
+  for static model parsing'
+time: 2023-04-18T12:12:36.928665+02:00
+custom:
+  Author: jtcohen6
+  Issue: "6671"

.changes/unreleased/Under the Hood-20230418-122323.yaml

@@ -0,0 +1,6 @@
+kind: Under the Hood
+body: Update --help text for cache-related parameters
+time: 2023-04-18T12:23:23.276693+02:00
+custom:
+  Author: jtcohen6
+  Issue: "7381"

.changes/unreleased/Under the Hood-20230424-135300.yaml

@@ -0,0 +1,8 @@
+kind: Under the Hood
+body: 'Small UX improvements to model versions: Support defining latest_version in
+  unsuffixed file by default. Notify on unpinned ref when a prerelease version is
+  available. '
+time: 2023-04-24T13:53:00.484916+02:00
+custom:
+  Author: jtcohen6
+  Issue: "7443"

.changie.yaml

@@ -4,6 +4,7 @@ headerPath: header.tpl.md
 versionHeaderPath: ""
 changelogPath: CHANGELOG.md
 versionExt: md
+envPrefix: "CHANGIE_"
 versionFormat: '## dbt-core {{.Version}} - {{.Time.Format "January 02, 2006"}}'
 kindFormat: '### {{.Kind}}'
 changeFormat: |-
@@ -87,32 +88,44 @@ custom:
 
 footerFormat: |
   {{- $contributorDict := dict }}
-  {{- /* any names added to this list should be all lowercase for later matching purposes */}}
-  {{- $core_team := list "michelleark" "peterallenwebb" "emmyoop" "nathaniel-may" "gshank" "leahwicz" "chenyulinx" "stu-k" "iknox-fa" "versusfacit" "mcknight-42" "jtcohen6" "aranke" "dependabot[bot]" "snyk-bot" "colin-rogers-dbt" }}
+  {{- /* ensure all names in this list are all lowercase for later matching purposes */}}
+  {{- $core_team := splitList " " .Env.CORE_TEAM }}
+  {{- /* ensure we always skip snyk and dependabot in addition to the core team */}}
+  {{- $maintainers := list "dependabot[bot]" "snyk-bot"}}
+  {{- range $team_member := $core_team }}
+    {{- $team_member_lower := lower $team_member }}
+    {{- $maintainers = append $maintainers $team_member_lower }}
+  {{- end }}
   {{- range $change := .Changes }}
     {{- $authorList := splitList " " $change.Custom.Author }}
     {{- /* loop through all authors for a single changelog */}}
     {{- range $author := $authorList }}
       {{- $authorLower := lower $author }}
       {{- /* we only want to include non-core team contributors */}}
-      {{- if not (has $authorLower $core_team)}}
+      {{- if not (has $authorLower $maintainers)}}
         {{- $changeList := splitList " " $change.Custom.Author }}
-          {{- /* Docs kind link back to dbt-docs instead of dbt-core issues */}}
+          {{- $IssueList := list }}
           {{- $changeLink := $change.Kind }}
           {{- if or (eq $change.Kind "Dependencies") (eq $change.Kind "Security") }}
-            {{- $changeLink = "[#nbr](https://github.com/dbt-labs/dbt-core/pull/nbr)" | replace "nbr" $change.Custom.PR }}
-          {{- else if eq $change.Kind "Docs"}}
-            {{- $changeLink = "[dbt-docs/#nbr](https://github.com/dbt-labs/dbt-docs/issues/nbr)" | replace "nbr" $change.Custom.Issue }}
+            {{- $changes := splitList " " $change.Custom.PR }}
+            {{- range $issueNbr := $changes }}
+              {{- $changeLink := "[#nbr](https://github.com/dbt-labs/dbt-core/pull/nbr)" | replace "nbr" $issueNbr }}
+              {{- $IssueList = append $IssueList $changeLink  }}
+            {{- end -}}
           {{- else }}
-            {{- $changeLink = "[#nbr](https://github.com/dbt-labs/dbt-core/issues/nbr)" | replace "nbr" $change.Custom.Issue }}
+            {{- $changes := splitList " " $change.Custom.Issue }}
+            {{- range $issueNbr := $changes }}
+              {{- $changeLink := "[#nbr](https://github.com/dbt-labs/dbt-core/issues/nbr)" | replace "nbr" $issueNbr }}
+              {{- $IssueList = append $IssueList $changeLink  }}
+            {{- end -}}
           {{- end }}
           {{- /* check if this contributor has other changes associated with them already */}}
           {{- if hasKey $contributorDict $author }}
             {{- $contributionList := get $contributorDict $author }}
-            {{- $contributionList = append $contributionList $changeLink  }}
+            {{- $contributionList = concat $contributionList $IssueList  }}
             {{- $contributorDict := set $contributorDict $author $contributionList }}
           {{- else }}
-            {{- $contributionList := list $changeLink }}
+            {{- $contributionList := $IssueList }}
             {{- $contributorDict := set $contributorDict $author $contributionList }}
           {{- end }}
         {{- end}}

.gitattributes

@@ -1,2 +1,6 @@
 core/dbt/include/index.html binary
 tests/functional/artifacts/data/state/*/manifest.json binary
+core/dbt/docs/build/html/searchindex.js binary
+core/dbt/docs/build/html/index.html binary
+performance/runner/Cargo.lock binary
+core/dbt/events/types_pb2.py binary

.github/CODEOWNERS

@@ -13,8 +13,23 @@
 # the core team as a whole will be assigned
 *       @dbt-labs/core
 
-# Changes to GitHub configurations including Actions
-/.github/ @leahwicz
+### OSS Tooling Guild
+
+/.github/                       @dbt-labs/guild-oss-tooling
+.bumpversion.cfg                @dbt-labs/guild-oss-tooling
+
+/.changes/                      @dbt-labs/guild-oss-tooling
+.changie.yaml                   @dbt-labs/guild-oss-tooling
+
+pre-commit-config.yaml          @dbt-labs/guild-oss-tooling
+pytest.ini                      @dbt-labs/guild-oss-tooling
+tox.ini                         @dbt-labs/guild-oss-tooling
+
+pyproject.toml                  @dbt-labs/guild-oss-tooling
+requirements.txt                @dbt-labs/guild-oss-tooling
+dev_requirements.txt            @dbt-labs/guild-oss-tooling
+/core/setup.py                  @dbt-labs/guild-oss-tooling
+/core/MANIFEST.in               @dbt-labs/guild-oss-tooling
 
 ### LANGUAGE
 
@@ -60,6 +75,7 @@
 
 # Postgres plugin
 /plugins/                       @dbt-labs/core-adapters
+/plugins/postgres/setup.py      @dbt-labs/core-adapters @dbt-labs/guild-oss-tooling
 
 # Functional tests for adapter plugins
 /tests/adapter                  @dbt-labs/core-adapters
@@ -71,5 +87,9 @@
 # Perf regression testing framework
 # This excludes the test project files itself since those aren't specific
 # framework changes (excluded by not setting an owner next to it- no owner)
-/performance @nathaniel-may
+/performance                    @nathaniel-may
 /performance/projects
+
+### ARTIFACTS
+
+/schemas/dbt                    @dbt-labs/cloud-artifacts

.github/workflows/cut-release-branch.yml

@@ -0,0 +1,41 @@
+# **what?**
+# Cuts a new `*.latest` branch
+# Also cleans up all files in `.changes/unreleased` and `.changes/previous verion on
+# `main` and bumps `main` to the input version.
+
+# **why?**
+# Generally reduces the workload of engineers and reduces error.  Allow automation.
+
+# **when?**
+# This will run when called manually.
+
+name: Cut new release branch
+
+on:
+  workflow_dispatch:
+    inputs:
+      version_to_bump_main:
+        description: 'The alpha version main should bump to (ex. 1.6.0a1)'
+        required: true
+      new_branch_name:
+        description: 'The full name of the new branch (ex. 1.5.latest)'
+        required: true
+
+defaults:
+  run:
+    shell: bash
+
+permissions:
+  contents: write
+
+jobs:
+  cut_branch:
+    name: "Cut branch and clean up main for dbt-core"
+    uses: dbt-labs/actions/.github/workflows/cut-release-branch.yml@main
+    with:
+      version_to_bump_main: ${{ inputs.version_to_bump_main }}
+      new_branch_name: ${{ inputs.new_branch_name }}
+      PR_title: "Cleanup main after cutting new ${{ inputs.new_branch_name }} branch"
+      PR_body: "All adapter PRs will fail CI until the dbt-core PR has been merged due to release version conflicts."
+    secrets:
+      FISHTOWN_BOT_PAT: ${{ secrets.FISHTOWN_BOT_PAT }}

.github/workflows/generate-cli-api-docs.yml

@@ -1,165 +0,0 @@
-# **what?**
-# On push, if anything in core/dbt/docs or core/dbt/cli has been
-# created or modified, regenerate the CLI API docs using sphinx.
-
-# **why?**
-# We watch for changes in core/dbt/cli because the CLI API docs rely on click
-# and all supporting flags/params to be generated. We watch for changes in
-# core/dbt/docs since any changes to sphinx configuration or any of the
-# .rst files there could result in a differently build final index.html file.
-
-# **when?**
-# Whenever a change has been pushed to a branch, and only if there is a diff
-# between the PR branch and main's core/dbt/cli and or core/dbt/docs dirs.
-
-# TODO: add bot comment to PR informing contributor that the docs have been committed
-# TODO: figure out why github action triggered pushes cause github to fail to report
-#       the status of jobs
-
-name: Generate CLI API docs
-
-on:
-  pull_request:
-
-permissions:
-  contents: write
-  pull-requests: write
-
-env:
-  CLI_DIR: ${{ github.workspace }}/core/dbt/cli
-  DOCS_DIR: ${{ github.workspace }}/core/dbt/docs
-  DOCS_BUILD_DIR: ${{ github.workspace }}/core/dbt/docs/build
-
-jobs:
-  check_gen:
-    name: check if generation needed
-    runs-on: ubuntu-latest
-    if: ${{ github.event.pull_request.head.repo.fork == false }}
-    outputs:
-      cli_dir_changed: ${{ steps.check_cli.outputs.cli_dir_changed }}
-      docs_dir_changed: ${{ steps.check_docs.outputs.docs_dir_changed }}
-
-    steps:
-    - name: "[DEBUG] print variables"
-      run: |
-        echo "env.CLI_DIR:        ${{ env.CLI_DIR }}"
-        echo "env.DOCS_BUILD_DIR: ${{ env.DOCS_BUILD_DIR }}"
-        echo "env.DOCS_DIR:       ${{ env.DOCS_DIR }}"
-
-    - name: git checkout
-      uses: actions/checkout@v3
-      with:
-        fetch-depth: 0
-        ref: ${{ github.head_ref }}
-
-    - name: set shas
-      id: set_shas
-      run: |
-        THIS_SHA=$(git rev-parse @)
-        LAST_SHA=$(git rev-parse @~1)
-
-        echo "this sha: $THIS_SHA"
-        echo "last sha: $LAST_SHA"
-
-        echo "this_sha=$THIS_SHA" >> $GITHUB_OUTPUT
-        echo "last_sha=$LAST_SHA" >> $GITHUB_OUTPUT
-
-    - name: check for changes in core/dbt/cli
-      id: check_cli
-      run: |
-        CLI_DIR_CHANGES=$(git diff \
-          ${{ steps.set_shas.outputs.last_sha }} \
-          ${{ steps.set_shas.outputs.this_sha }} \
-          -- ${{ env.CLI_DIR }})
-
-        if [ -n "$CLI_DIR_CHANGES" ]; then
-          echo "changes found"
-          echo $CLI_DIR_CHANGES
-          echo "cli_dir_changed=true" >> $GITHUB_OUTPUT
-          exit 0
-        fi
-        echo "cli_dir_changed=false" >> $GITHUB_OUTPUT
-        echo "no changes found"
-
-    - name: check for changes in core/dbt/docs
-      id: check_docs
-      if: steps.check_cli.outputs.cli_dir_changed == 'false'
-      run: |
-        DOCS_DIR_CHANGES=$(git diff --name-only \
-          ${{ steps.set_shas.outputs.last_sha }} \
-          ${{ steps.set_shas.outputs.this_sha }} \
-          -- ${{ env.DOCS_DIR }} ':!${{ env.DOCS_BUILD_DIR }}')
-
-        DOCS_BUILD_DIR_CHANGES=$(git diff --name-only \
-          ${{ steps.set_shas.outputs.last_sha }} \
-          ${{ steps.set_shas.outputs.this_sha }} \
-          -- ${{ env.DOCS_BUILD_DIR }})
-
-        if [ -n "$DOCS_DIR_CHANGES" ] && [ -z "$DOCS_BUILD_DIR_CHANGES" ]; then
-          echo "changes found"
-          echo $DOCS_DIR_CHANGES
-          echo "docs_dir_changed=true" >> $GITHUB_OUTPUT
-          exit 0
-        fi
-        echo "docs_dir_changed=false" >> $GITHUB_OUTPUT
-        echo "no changes found"
-
-  gen_docs:
-    name: generate docs
-    runs-on: ubuntu-latest
-    needs: [check_gen]
-    if: |
-      needs.check_gen.outputs.cli_dir_changed == 'true'
-      || needs.check_gen.outputs.docs_dir_changed == 'true'
-
-    steps:
-      - name: "[DEBUG] print variables"
-        run: |
-          echo "env.DOCS_DIR:    ${{ env.DOCS_DIR }}"
-          echo "github head_ref: ${{ github.head_ref }}"
-
-      - name: git checkout
-        uses: actions/checkout@v3
-        with:
-          ref: ${{ github.head_ref }}
-
-      - name: install python
-        uses: actions/setup-python@v4.3.0
-        with:
-          python-version: 3.8
-
-      - name: install dev requirements
-        run: |
-          python3 -m venv env
-          source env/bin/activate
-          python -m pip install --upgrade pip
-          pip install -r requirements.txt -r dev-requirements.txt
-
-      - name: generate docs
-        run: |
-          source env/bin/activate
-          cd ${{ env.DOCS_DIR }}
-
-          echo "cleaning existing docs"
-          make clean
-
-          echo "creating docs"
-          make html
-
-      - name: debug
-        run: |
-          echo ">>>>> status"
-          git status
-          echo ">>>>> remotes"
-          git remote -v
-          echo ">>>>> branch"
-          git branch -v
-          echo ">>>>> log"
-          git log --pretty=oneline | head -5
-
-      - name: commit docs
-        run: |
-          git config user.name 'Github Build Bot'
-          git config user.email 'buildbot@fishtownanalytics.com'
-          git commit -am "Add generated CLI API docs"
-          git push -u origin ${{ github.head_ref }}

.github/workflows/main.yml

@@ -42,7 +42,7 @@ jobs:
 
     steps:
       - name: Check out the repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Set up Python
         uses: actions/setup-python@v4.3.0
@@ -53,12 +53,8 @@ jobs:
         run: |
           python -m pip install --user --upgrade pip
           python -m pip --version
-          python -m pip install pre-commit
-          pre-commit --version
-          python -m pip install mypy==0.942
+          make dev
           mypy --version
-          python -m pip install -r requirements.txt
-          python -m pip install -r dev-requirements.txt
           dbt --version
 
       - name: Run pre-commit hooks
@@ -81,7 +77,7 @@ jobs:
 
     steps:
       - name: Check out the repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v4.3.0
@@ -105,7 +101,7 @@ jobs:
           CURRENT_DATE=$(date +'%Y-%m-%dT%H_%M_%S') # no colons allowed for artifacts
           echo "date=$CURRENT_DATE" >> $GITHUB_OUTPUT
 
-      - uses: actions/upload-artifact@v2
+      - uses: actions/upload-artifact@v3
         if: always()
         with:
           name: unit_results_${{ matrix.python-version }}-${{ steps.date.outputs.date }}.csv
@@ -138,7 +134,7 @@ jobs:
 
     steps:
       - name: Check out the repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v4.3.0
@@ -174,13 +170,13 @@ jobs:
           CURRENT_DATE=$(date +'%Y-%m-%dT%H_%M_%S') # no colons allowed for artifacts
           echo "date=$CURRENT_DATE" >> $GITHUB_OUTPUT
 
-      - uses: actions/upload-artifact@v2
+      - uses: actions/upload-artifact@v3
         if: always()
         with:
           name: logs_${{ matrix.python-version }}_${{ matrix.os }}_${{ steps.date.outputs.date }}
           path: ./logs
 
-      - uses: actions/upload-artifact@v2
+      - uses: actions/upload-artifact@v3
         if: always()
         with:
           name: integration_results_${{ matrix.python-version }}_${{ matrix.os }}_${{ steps.date.outputs.date }}.csv
@@ -193,7 +189,7 @@ jobs:
 
     steps:
       - name: Check out the repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Set up Python
         uses: actions/setup-python@v4.3.0

.github/workflows/model_performance.yml

@@ -0,0 +1,265 @@
+# **what?**
+# This workflow models the performance characteristics of a point in time in dbt.
+# It runs specific dbt commands on committed projects multiple times to create and
+# commit information about the distribution to the current branch. For more information
+# see the readme in the performance module at /performance/README.md.
+#
+# **why?**
+# When developing new features, we can take quick performance samples and compare
+# them against the commited baseline measurements produced by this workflow to detect
+# some performance regressions at development time before they reach users.
+#
+# **when?**
+# This is only run once directly after each release (for non-prereleases). If for some
+# reason the results of a run are not satisfactory, it can also be triggered manually.
+
+name: Model Performance Characteristics
+
+on:
+  # runs after non-prereleases are published.
+  release:
+    types: [released]
+  # run manually from the actions tab
+  workflow_dispatch:
+    inputs:
+      release_id:
+        description: 'dbt version to model (must be non-prerelease in Pypi)'
+        type: string
+        required: true
+
+env:
+  RUNNER_CACHE_PATH: performance/runner/target/release/runner
+
+# both jobs need to write
+permissions:
+  contents: write
+  pull-requests: write
+
+jobs:
+  set-variables:
+    name: Setting Variables
+    runs-on: ubuntu-latest
+    outputs:
+      cache_key: ${{ steps.variables.outputs.cache_key }}
+      release_id: ${{ steps.semver.outputs.base-version }}
+      release_branch: ${{ steps.variables.outputs.release_branch }}
+    steps:
+
+      # explicitly checkout the performance runner from main regardless of which
+      # version we are modeling.
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          ref: main
+
+      - name: Parse version into parts
+        id: semver
+        uses: dbt-labs/actions/parse-semver@v1
+        with:
+          version: ${{ github.event.inputs.release_id || github.event.release.tag_name }}
+
+      # collect all the variables that need to be used in subsequent jobs
+      - name: Set variables
+        id: variables
+        run: |
+          # create a cache key that will be used in the next job. without this the
+          # next job would have to checkout from main and hash the files itself.
+          echo "cache_key=${{ runner.os }}-${{ hashFiles('performance/runner/Cargo.toml')}}-${{ hashFiles('performance/runner/src/*') }}" >> $GITHUB_OUTPUT
+
+          branch_name="${{steps.semver.outputs.major}}.${{steps.semver.outputs.minor}}.latest"
+          echo "release_branch=$branch_name" >> $GITHUB_OUTPUT
+          echo "release branch is inferred to be ${branch_name}"
+
+  latest-runner:
+    name: Build or Fetch Runner
+    runs-on: ubuntu-latest
+    needs: [set-variables]
+    env:
+      RUSTFLAGS: "-D warnings"
+    steps:
+      - name: '[DEBUG] print variables'
+        run: |
+          echo "all variables defined in set-variables"
+          echo "cache_key:      ${{ needs.set-variables.outputs.cache_key }}"
+          echo "release_id:     ${{ needs.set-variables.outputs.release_id }}"
+          echo "release_branch: ${{ needs.set-variables.outputs.release_branch }}"
+
+      # explicitly checkout the performance runner from main regardless of which
+      # version we are modeling.
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          ref: main
+
+      # attempts to access a previously cached runner
+      - uses: actions/cache@v3
+        id: cache
+        with:
+          path: ${{ env.RUNNER_CACHE_PATH }}
+          key: ${{ needs.set-variables.outputs.cache_key }}
+
+      - name: Fetch Rust Toolchain
+        if: steps.cache.outputs.cache-hit != 'true'
+        uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: stable
+          override: true
+
+      - name: Add fmt
+        if: steps.cache.outputs.cache-hit != 'true'
+        run: rustup component add rustfmt
+
+      - name: Cargo fmt
+        if: steps.cache.outputs.cache-hit != 'true'
+        uses: actions-rs/cargo@v1
+        with:
+          command: fmt
+          args: --manifest-path performance/runner/Cargo.toml --all -- --check
+
+      - name: Test
+        if: steps.cache.outputs.cache-hit != 'true'
+        uses: actions-rs/cargo@v1
+        with:
+          command: test
+          args: --manifest-path performance/runner/Cargo.toml
+
+      - name: Build (optimized)
+        if: steps.cache.outputs.cache-hit != 'true'
+        uses: actions-rs/cargo@v1
+        with:
+          command: build
+          args: --release --manifest-path performance/runner/Cargo.toml
+      # the cache action automatically caches this binary at the end of the job
+
+  model:
+    # depends on `latest-runner` as a separate job so that failures in this job do not prevent
+    # a successfully tested and built binary from being cached.
+    needs: [set-variables, latest-runner]
+    name: Model a release
+    runs-on: ubuntu-latest
+    steps:
+
+      - name: '[DEBUG] print variables'
+        run: |
+          echo "all variables defined in set-variables"
+          echo "cache_key:      ${{ needs.set-variables.outputs.cache_key }}"
+          echo "release_id:     ${{ needs.set-variables.outputs.release_id }}"
+          echo "release_branch: ${{ needs.set-variables.outputs.release_branch }}"
+
+      - name: Setup Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.8"
+
+      - name: Install dbt
+        run: pip install dbt-postgres==${{ needs.set-variables.outputs.release_id }}
+
+      - name: Install Hyperfine
+        run: wget https://github.com/sharkdp/hyperfine/releases/download/v1.11.0/hyperfine_1.11.0_amd64.deb && sudo dpkg -i hyperfine_1.11.0_amd64.deb
+
+      # explicitly checkout main to get the latest project definitions
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          ref: main
+
+      # this was built in the previous job so it will be there.
+      - name: Fetch Runner
+        uses: actions/cache@v3
+        id: cache
+        with:
+          path: ${{ env.RUNNER_CACHE_PATH }}
+          key: ${{ needs.set-variables.outputs.cache_key }}
+
+      - name: Move Runner
+        run: mv performance/runner/target/release/runner performance/app
+
+      - name: Change Runner Permissions
+        run: chmod +x ./performance/app
+
+      - name: '[DEBUG] ls baseline directory before run'
+        run: ls -R performance/baselines/
+
+      # `${{ github.workspace }}` is used to pass the absolute path
+      - name: Create directories
+        run: |
+          mkdir ${{ github.workspace }}/performance/tmp/
+          mkdir -p performance/baselines/${{ needs.set-variables.outputs.release_id }}/
+
+      # Run modeling with taking 20 samples
+      - name: Run Measurement
+        run: |
+          performance/app model -v ${{ needs.set-variables.outputs.release_id }} -b ${{ github.workspace }}/performance/baselines/ -p ${{ github.workspace }}/performance/projects/ -t ${{ github.workspace }}/performance/tmp/ -n 20
+
+      - name: '[DEBUG] ls baseline directory after run'
+        run: ls -R performance/baselines/
+
+      - uses: actions/upload-artifact@v3
+        with:
+          name: baseline
+          path: performance/baselines/${{ needs.set-variables.outputs.release_id }}/
+
+  create-pr:
+    name: Open PR for ${{ matrix.base-branch }}
+
+    # depends on `model` as a separate job so that the baseline can be committed to more than one branch
+    # i.e. release branch and main
+    needs: [set-variables, latest-runner, model]
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        include:
+          - base-branch: refs/heads/main
+            target-branch: performance-bot/main_${{ needs.set-variables.outputs.release_id }}_${{GITHUB.RUN_ID}}
+          - base-branch: refs/heads/${{ needs.set-variables.outputs.release_branch }}
+            target-branch: performance-bot/release_${{ needs.set-variables.outputs.release_id }}_${{GITHUB.RUN_ID}}
+
+    steps:
+      - name: '[DEBUG] print variables'
+        run: |
+          echo "all variables defined in set-variables"
+          echo "cache_key:      ${{ needs.set-variables.outputs.cache_key }}"
+          echo "release_id:     ${{ needs.set-variables.outputs.release_id }}"
+          echo "release_branch: ${{ needs.set-variables.outputs.release_branch }}"
+
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ matrix.base-branch }}
+
+      - name: Create PR branch
+        run: |
+          git checkout -b ${{ matrix.target-branch }}
+          git push origin ${{ matrix.target-branch }}
+          git branch --set-upstream-to=origin/${{ matrix.target-branch }} ${{ matrix.target-branch }}
+
+      - uses: actions/download-artifact@v3
+        with:
+          name: baseline
+          path: performance/baselines/${{ needs.set-variables.outputs.release_id }}
+
+      - name: '[DEBUG] ls baselines after artifact download'
+        run: ls -R performance/baselines/
+
+      - name: Commit baseline
+        uses: EndBug/add-and-commit@v9
+        with:
+          add: 'performance/baselines/*'
+          author_name: 'Github Build Bot'
+          author_email: 'buildbot@fishtownanalytics.com'
+          message: 'adding performance baseline for ${{ needs.set-variables.outputs.release_id }}'
+          push: 'origin origin/${{ matrix.target-branch }}'
+
+      - name: Create Pull Request
+        uses: peter-evans/create-pull-request@v5
+        with:
+          author: 'Github Build Bot <buildbot@fishtownanalytics.com>'
+          base: ${{ matrix.base-branch }}
+          branch: '${{ matrix.target-branch }}'
+          title: 'Adding performance modeling for ${{needs.set-variables.outputs.release_id}} to ${{ matrix.base-branch }}'
+          body: 'Committing perf results for tracking for the ${{needs.set-variables.outputs.release_id}}'
+          labels: |
+            Skip Changelog
+            Performance

.github/workflows/nightly-release.yml

@@ -0,0 +1,109 @@
+# **what?**
+# Nightly releases to GitHub and PyPI. This workflow produces the following outcome:
+# - generate and validate data for night release (commit SHA, version number, release branch);
+# - pass data to release workflow;
+# - night release will be pushed to GitHub as a draft release;
+# - night build will be pushed to test PyPI;
+#
+# **why?**
+# Ensure an automated and tested release process for nightly builds
+#
+# **when?**
+# This workflow runs on schedule or can be run manually on demand.
+
+name: Nightly Test Release to GitHub and PyPI
+
+on:
+  workflow_dispatch: # for manual triggering
+  schedule:
+    - cron: 0 9 * * *
+
+permissions:
+  contents: write # this is the permission that allows creating a new release
+
+defaults:
+  run:
+    shell: bash
+
+env:
+  RELEASE_BRANCH: "main"
+
+jobs:
+  aggregate-release-data:
+    runs-on: ubuntu-latest
+
+    outputs:
+      commit_sha: ${{ steps.resolve-commit-sha.outputs.release_commit }}
+      version_number: ${{ steps.nightly-release-version.outputs.number }}
+      release_branch: ${{ steps.release-branch.outputs.name }}
+
+    steps:
+      - name: "Checkout ${{ github.repository }} Branch ${{ env.RELEASE_BRANCH }}"
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ env.RELEASE_BRANCH }}
+
+      - name: "Resolve Commit To Release"
+        id: resolve-commit-sha
+        run: |
+          commit_sha=$(git rev-parse HEAD)
+          echo "release_commit=$commit_sha" >> $GITHUB_OUTPUT
+
+      - name: "Get Current Version Number"
+        id: version-number-sources
+        run: |
+          current_version=`awk -F"current_version = " '{print $2}' .bumpversion.cfg | tr '\n' ' '`
+          echo "current_version=$current_version" >> $GITHUB_OUTPUT
+
+      - name: "Audit Version And Parse Into Parts"
+        id: semver
+        uses: dbt-labs/actions/parse-semver@v1.1.0
+        with:
+          version: ${{ steps.version-number-sources.outputs.current_version }}
+
+      - name: "Get Current Date"
+        id: current-date
+        run: echo "date=$(date +'%m%d%Y')" >> $GITHUB_OUTPUT
+
+      - name: "Generate Nightly Release Version Number"
+        id: nightly-release-version
+        run: |
+          number="${{ steps.semver.outputs.version }}.dev${{ steps.current-date.outputs.date }}"
+          echo "number=$number" >> $GITHUB_OUTPUT
+
+      - name: "Audit Nightly Release Version And Parse Into Parts"
+        uses: dbt-labs/actions/parse-semver@v1.1.0
+        with:
+          version: ${{ steps.nightly-release-version.outputs.number }}
+
+      - name: "Set Release Branch"
+        id: release-branch
+        run: |
+          echo "name=${{ env.RELEASE_BRANCH }}" >> $GITHUB_OUTPUT
+
+  log-outputs-aggregate-release-data:
+    runs-on: ubuntu-latest
+    needs: [aggregate-release-data]
+
+    steps:
+      - name: "[DEBUG] Log Outputs"
+        run: |
+          echo commit_sha    : ${{ needs.aggregate-release-data.outputs.commit_sha }}
+          echo version_number: ${{ needs.aggregate-release-data.outputs.version_number }}
+          echo release_branch: ${{ needs.aggregate-release-data.outputs.release_branch }}
+
+  release-github-pypi:
+    needs: [aggregate-release-data]
+
+    uses: ./.github/workflows/release.yml
+    with:
+      sha: ${{ needs.aggregate-release-data.outputs.commit_sha }}
+      target_branch: ${{ needs.aggregate-release-data.outputs.release_branch }}
+      version_number: ${{ needs.aggregate-release-data.outputs.version_number }}
+      build_script_path: "scripts/build-dist.sh"
+      env_setup_script_path: "scripts/env-setup.sh"
+      s3_bucket_name: "core-team-artifacts"
+      package_test_command: "dbt --version"
+      test_run: true
+      nightly_release: true
+    secrets: inherit

.github/workflows/release-branch-tests.yml

@@ -28,7 +28,33 @@ on:
 permissions: read-all
 
 jobs:
+  fetch-latest-branches:
+    runs-on: ubuntu-latest
+
+    outputs:
+      latest-branches: ${{ steps.get-latest-branches.outputs.repo-branches }}
+
+    steps:
+      - name: "Fetch dbt-core Latest Branches"
+        uses: dbt-labs/actions/fetch-repo-branches@v1.1.1
+        id: get-latest-branches
+        with:
+          repo_name: ${{ github.event.repository.name }}
+          organization: "dbt-labs"
+          pat: ${{ secrets.GITHUB_TOKEN }}
+          fetch_protected_branches_only: true
+          regex: "^1.[0-9]+.latest$"
+          perform_match_method: "match"
+          retries: 3
+
+      - name: "[ANNOTATION] ${{ github.event.repository.name }} - branches to test"
+        run: |
+          title="${{ github.event.repository.name }} - branches to test"
+          message="The workflow will run tests for the following branches of the ${{ github.event.repository.name }} repo: ${{ steps.get-latest-branches.outputs.repo-branches }}"
+          echo "::notice $title::$message"
+
   kick-off-ci:
+    needs: [fetch-latest-branches]
     name: Kick-off CI
     runs-on: ubuntu-latest
 
@@ -39,7 +65,9 @@ jobs:
       max-parallel: 1
       fail-fast: false
       matrix:
-        branch: [1.0.latest, 1.1.latest, 1.2.latest, 1.3.latest, main]
+        branch: ${{ fromJSON(needs.fetch-latest-branches.outputs.latest-branches) }}
+        include:
+          - branch: 'main'
 
     steps:
     - name: Call CI workflow for ${{ matrix.branch }} branch

.github/workflows/release.yml

@@ -1,24 +1,110 @@
 # **what?**
-# Take the given commit, run unit tests specifically on that sha, build and
-# package it, and then release to GitHub and PyPi with that specific build
-
+# Release workflow provides the following steps:
+# - checkout the given commit;
+# - validate version in sources and changelog file for given version;
+# - bump the version and generate a changelog if needed;
+# - merge all changes to the target branch if needed;
+# - run unit and integration tests against given commit;
+# - build and package that SHA;
+# - release it to GitHub and PyPI with that specific build;
+#
 # **why?**
 # Ensure an automated and tested release process
-
+#
 # **when?**
-# This will only run manually with a given sha and version
+# This workflow can be run manually on demand or can be called by other workflows
 
-name: Release to GitHub and PyPi
+name: Release to GitHub and PyPI
 
 on:
   workflow_dispatch:
     inputs:
       sha:
-       description: 'The last commit sha in the release'
-       required: true
+        description: "The last commit sha in the release"
+        type: string
+        required: true
+      target_branch:
+        description: "The branch to release from"
+        type: string
+        required: true
       version_number:
-       description: 'The release version number (i.e. 1.0.0b1)'
-       required: true
+        description: "The release version number (i.e. 1.0.0b1)"
+        type: string
+        required: true
+      build_script_path:
+        description: "Build script path"
+        type: string
+        default: "scripts/build-dist.sh"
+        required: true
+      env_setup_script_path:
+        description: "Environment setup script path"
+        type: string
+        default: "scripts/env-setup.sh"
+        required: false
+      s3_bucket_name:
+        description: "AWS S3 bucket name"
+        type: string
+        default: "core-team-artifacts"
+        required: true
+      package_test_command:
+        description: "Package test command"
+        type: string
+        default: "dbt --version"
+        required: true
+      test_run:
+        description: "Test run (Publish release as draft)"
+        type: boolean
+        default: true
+        required: false
+      nightly_release:
+        description: "Nightly release to dev environment"
+        type: boolean
+        default: false
+        required: false
+  workflow_call:
+    inputs:
+      sha:
+        description: "The last commit sha in the release"
+        type: string
+        required: true
+      target_branch:
+        description: "The branch to release from"
+        type: string
+        required: true
+      version_number:
+        description: "The release version number (i.e. 1.0.0b1)"
+        type: string
+        required: true
+      build_script_path:
+        description: "Build script path"
+        type: string
+        default: "scripts/build-dist.sh"
+        required: true
+      env_setup_script_path:
+        description: "Environment setup script path"
+        type: string
+        default: "scripts/env-setup.sh"
+        required: false
+      s3_bucket_name:
+        description: "AWS S3 bucket name"
+        type: string
+        default: "core-team-artifacts"
+        required: true
+      package_test_command:
+        description: "Package test command"
+        type: string
+        default: "dbt --version"
+        required: true
+      test_run:
+        description: "Test run (Publish release as draft)"
+        type: boolean
+        default: true
+        required: false
+      nightly_release:
+        description: "Nightly release to dev environment"
+        type: boolean
+        default: false
+        required: false
 
 permissions:
   contents: write # this is the permission that allows creating a new release
@@ -28,175 +114,116 @@ defaults:
     shell: bash
 
 jobs:
-  unit:
-    name: Unit test
-
+  log-inputs:
+    name: Log Inputs
     runs-on: ubuntu-latest
-
-    env:
-      TOXENV: "unit"
-
     steps:
-      - name: Check out the repository
-        uses: actions/checkout@v2
-        with:
-          persist-credentials: false
-          ref: ${{ github.event.inputs.sha }}
-
-      - name: Set up Python
-        uses: actions/setup-python@v2
-        with:
-          python-version: 3.8
-
-      - name: Install python dependencies
+      - name: "[DEBUG] Print Variables"
         run: |
-          pip install --user --upgrade pip
-          pip install tox
-          pip --version
-          tox --version
+          echo The last commit sha in the release: ${{ inputs.sha }}
+          echo The branch to release from:         ${{ inputs.target_branch }}
+          echo The release version number:         ${{ inputs.version_number }}
+          echo Build script path:                  ${{ inputs.build_script_path }}
+          echo Environment setup script path:      ${{ inputs.env_setup_script_path }}
+          echo AWS S3 bucket name:                 ${{ inputs.s3_bucket_name }}
+          echo Package test command:               ${{ inputs.package_test_command }}
+          echo Test run:                           ${{ inputs.test_run }}
+          echo Nightly release:                    ${{ inputs.nightly_release }}
 
-      - name: Run tox
-        run: tox
+  bump-version-generate-changelog:
+    name: Bump package version, Generate changelog
 
-  build:
-    name: build packages
+    uses: dbt-labs/dbt-release/.github/workflows/release-prep.yml@main
+
+    with:
+      sha: ${{ inputs.sha }}
+      version_number: ${{ inputs.version_number }}
+      target_branch: ${{ inputs.target_branch }}
+      env_setup_script_path: ${{ inputs.env_setup_script_path }}
+      test_run: ${{ inputs.test_run }}
+      nightly_release: ${{ inputs.nightly_release }}
+
+    secrets: inherit
+
+  log-outputs-bump-version-generate-changelog:
+    name: "[Log output] Bump package version, Generate changelog"
+    if: ${{ !failure() && !cancelled() }}
+
+    needs: [bump-version-generate-changelog]
 
     runs-on: ubuntu-latest
 
     steps:
-      - name: Check out the repository
-        uses: actions/checkout@v2
-        with:
-          persist-credentials: false
-          ref: ${{ github.event.inputs.sha }}
-
-      - name: Set up Python
-        uses: actions/setup-python@v2
-        with:
-          python-version: 3.8
-
-      - name: Install python dependencies
+      - name: Print variables
         run: |
-          pip install --user --upgrade pip
-          pip install --upgrade setuptools wheel twine check-wheel-contents
-          pip --version
+          echo Final SHA     : ${{ needs.bump-version-generate-changelog.outputs.final_sha }}
+          echo Changelog path: ${{ needs.bump-version-generate-changelog.outputs.changelog_path }}
 
-      - name: Build distributions
-        run: ./scripts/build-dist.sh
+  build-test-package:
+    name: Build, Test, Package
+    if: ${{ !failure() && !cancelled() }}
+    needs: [bump-version-generate-changelog]
 
-      - name: Show distributions
-        run: ls -lh dist/
+    uses: dbt-labs/dbt-release/.github/workflows/build.yml@main
 
-      - name: Check distribution descriptions
-        run: |
-          twine check dist/*
+    with:
+      sha: ${{ needs.bump-version-generate-changelog.outputs.final_sha }}
+      version_number: ${{ inputs.version_number }}
+      changelog_path: ${{ needs.bump-version-generate-changelog.outputs.changelog_path }}
+      build_script_path: ${{ inputs.build_script_path }}
+      s3_bucket_name: ${{ inputs.s3_bucket_name }}
+      package_test_command: ${{ inputs.package_test_command }}
+      test_run: ${{ inputs.test_run }}
+      nightly_release: ${{ inputs.nightly_release }}
 
-      - name: Check wheel contents
-        run: |
-          check-wheel-contents dist/*.whl --ignore W007,W008
-
-      - uses: actions/upload-artifact@v2
-        with:
-          name: dist
-          path: |
-            dist/
-            !dist/dbt-${{github.event.inputs.version_number}}.tar.gz
-
-  test-build:
-    name: verify packages
-
-    needs: [build, unit]
-
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: Set up Python
-        uses: actions/setup-python@v2
-        with:
-          python-version: 3.8
-
-      - name: Install python dependencies
-        run: |
-          pip install --user --upgrade pip
-          pip install --upgrade wheel
-          pip --version
-
-      - uses: actions/download-artifact@v2
-        with:
-          name: dist
-          path: dist/
-
-      - name: Show distributions
-        run: ls -lh dist/
-
-      - name: Install wheel distributions
-        run: |
-          find ./dist/*.whl -maxdepth 1 -type f | xargs pip install --force-reinstall --find-links=dist/
-
-      - name: Check wheel distributions
-        run: |
-          dbt --version
-
-      - name: Install source distributions
-        run: |
-          find ./dist/*.gz -maxdepth 1 -type f | xargs pip install --force-reinstall --find-links=dist/
-
-      - name: Check source distributions
-        run: |
-          dbt --version
+    secrets:
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
 
   github-release:
     name: GitHub Release
+    if: ${{ !failure() && !cancelled() }}
 
-    needs: test-build
+    needs: [bump-version-generate-changelog, build-test-package]
 
-    runs-on: ubuntu-latest
+    uses: dbt-labs/dbt-release/.github/workflows/github-release.yml@main
 
-    steps:
-      - uses: actions/download-artifact@v2
-        with:
-          name: dist
-          path: '.'
-
-      # Need to set an output variable because env variables can't be taken as input
-      # This is needed for the next step with releasing to GitHub
-      - name: Find release type
-        id: release_type
-        env:
-          IS_PRERELEASE: ${{ contains(github.event.inputs.version_number, 'rc') ||  contains(github.event.inputs.version_number, 'b') }}
-        run: |
-          echo "isPrerelease=$IS_PRERELEASE" >> $GITHUB_OUTPUT
-
-      - name: Creating GitHub Release
-        uses: softprops/action-gh-release@v1
-        with:
-          name: dbt-core v${{github.event.inputs.version_number}}
-          tag_name: v${{github.event.inputs.version_number}}
-          prerelease: ${{ steps.release_type.outputs.isPrerelease }}
-          target_commitish: ${{github.event.inputs.sha}}
-          body: |
-            [Release notes](https://github.com/dbt-labs/dbt-core/blob/main/CHANGELOG.md)
-          files: |
-            dbt_postgres-${{github.event.inputs.version_number}}-py3-none-any.whl
-            dbt_core-${{github.event.inputs.version_number}}-py3-none-any.whl
-            dbt-postgres-${{github.event.inputs.version_number}}.tar.gz
-            dbt-core-${{github.event.inputs.version_number}}.tar.gz
+    with:
+      sha: ${{ needs.bump-version-generate-changelog.outputs.final_sha }}
+      version_number: ${{ inputs.version_number }}
+      changelog_path: ${{ needs.bump-version-generate-changelog.outputs.changelog_path }}
+      test_run: ${{ inputs.test_run }}
 
   pypi-release:
-    name: Pypi release
+    name: PyPI Release
 
-    runs-on: ubuntu-latest
+    needs: [github-release]
 
-    needs: github-release
+    uses: dbt-labs/dbt-release/.github/workflows/pypi-release.yml@main
 
-    environment: PypiProd
-    steps:
-      - uses: actions/download-artifact@v2
-        with:
-          name: dist
-          path: 'dist'
+    with:
+      version_number: ${{ inputs.version_number }}
+      test_run: ${{ inputs.test_run }}
 
-      - name: Publish distribution to PyPI
-        uses: pypa/gh-action-pypi-publish@v1.4.2
-        with:
-          password: ${{ secrets.PYPI_API_TOKEN }}
+    secrets:
+      PYPI_API_TOKEN: ${{ secrets.PYPI_API_TOKEN }}
+      TEST_PYPI_API_TOKEN: ${{ secrets.TEST_PYPI_API_TOKEN }}
+
+  slack-notification:
+    name: Slack Notification
+    if: ${{ failure() && (!inputs.test_run || inputs.nightly_release) }}
+
+    needs:
+      [
+        bump-version-generate-changelog,
+        build-test-package,
+        github-release,
+        pypi-release,
+      ]
+
+    uses: dbt-labs/dbt-release/.github/workflows/slack-post-notification.yml@main
+    with:
+      status: "failure"
+
+    secrets:
+      SLACK_WEBHOOK_URL: ${{ secrets.SLACK_DEV_CORE_ALERTS }}

.github/workflows/structured-logging-schema-check.yml

@@ -30,6 +30,8 @@ jobs:
       LOG_DIR: "/home/runner/work/dbt-core/dbt-core/logs"
       # tells integration tests to output into json format
       DBT_LOG_FORMAT: "json"
+      # tell eventmgr to convert logging events into bytes
+      DBT_TEST_BINARY_SERIALIZATION: "true"
       # Additional test users
       DBT_TEST_USER_1: dbt_test_user_1
       DBT_TEST_USER_2: dbt_test_user_2

.github/workflows/version-bump.yml

@@ -20,106 +20,9 @@ on:
        description: 'The version number to bump to (ex. 1.2.0, 1.3.0b1)'
        required: true
 
-permissions:
-  contents: write
-  pull-requests: write
-
 jobs:
-  bump:
-    runs-on: ubuntu-latest
-    steps:
-      - name: "[DEBUG] Print Variables"
-        run: |
-          echo "all variables defined as inputs"
-          echo The version_number: ${{ github.event.inputs.version_number }}
-
-      - name: Check out the repository
-        uses: actions/checkout@v2
-
-      - uses: actions/setup-python@v2
-        with:
-          python-version: "3.8"
-
-      - name: Install python dependencies
-        run: |
-          python3 -m venv env
-          source env/bin/activate
-          pip install --upgrade pip
-
-      - name: Add Homebrew to PATH
-        run: |
-          echo "/home/linuxbrew/.linuxbrew/bin:/home/linuxbrew/.linuxbrew/sbin" >> $GITHUB_PATH
-
-      - name: Install Homebrew packages
-        run: |
-          brew install pre-commit
-          brew tap miniscruff/changie https://github.com/miniscruff/changie
-          brew install changie
-
-      - name: Audit Version and Parse Into Parts
-        id: semver
-        uses: dbt-labs/actions/parse-semver@v1
-        with:
-          version: ${{ github.event.inputs.version_number }}
-
-      - name: Set branch value
-        id: variables
-        run: |
-          echo "BRANCH_NAME=prep-release/${{ github.event.inputs.version_number }}_$GITHUB_RUN_ID" >> $GITHUB_OUTPUT
-
-      - name: Create PR branch
-        run: |
-          git checkout -b ${{ steps.variables.outputs.BRANCH_NAME }}
-          git push origin ${{ steps.variables.outputs.BRANCH_NAME }}
-          git branch --set-upstream-to=origin/${{ steps.variables.outputs.BRANCH_NAME }} ${{ steps.variables.outputs.BRANCH_NAME }}
-
-      - name: Bump version
-        run: |
-          source env/bin/activate
-          pip install -r dev-requirements.txt
-          env/bin/bumpversion --allow-dirty --new-version ${{ github.event.inputs.version_number }} major
-          git status
-
-      - name: Run changie
-        run: |
-          if [[ ${{ steps.semver.outputs.is-pre-release }} -eq 1 ]]
-          then
-            changie batch ${{ steps.semver.outputs.base-version }}  --move-dir '${{ steps.semver.outputs.base-version }}' --prerelease '${{ steps.semver.outputs.pre-release }}'
-          else
-            changie batch ${{ steps.semver.outputs.base-version }}  --include '${{ steps.semver.outputs.base-version }}' --remove-prereleases
-          fi
-          changie merge
-          git status
-
-      # this step will fail on whitespace errors but also correct them
-      - name: Remove trailing whitespace
-        continue-on-error: true
-        run: |
-          pre-commit run trailing-whitespace --files .bumpversion.cfg CHANGELOG.md .changes/*
-          git status
-
-      # this step will fail on newline errors but also correct them
-      - name: Removing extra newlines
-        continue-on-error: true
-        run: |
-          pre-commit run end-of-file-fixer --files .bumpversion.cfg CHANGELOG.md .changes/*
-          git status
-
-      - name: Commit version bump to branch
-        uses: EndBug/add-and-commit@v7
-        with:
-          author_name: 'Github Build Bot'
-          author_email: 'buildbot@fishtownanalytics.com'
-          message: 'Bumping version to ${{ github.event.inputs.version_number }} and generate CHANGELOG'
-          branch: '${{ steps.variables.outputs.BRANCH_NAME }}'
-          push: 'origin origin/${{ steps.variables.outputs.BRANCH_NAME }}'
-
-      - name: Create Pull Request
-        uses: peter-evans/create-pull-request@v3
-        with:
-          author: 'Github Build Bot <buildbot@fishtownanalytics.com>'
-          base: ${{github.ref}}
-          title: 'Bumping version to ${{ github.event.inputs.version_number }} and generate changelog'
-          branch: '${{ steps.variables.outputs.BRANCH_NAME }}'
-          labels: |
-            Skip Changelog
+  version_bump_and_changie:
+    uses: dbt-labs/actions/.github/workflows/version-bump.yml@main
+    with:
+      version_number: ${{ inputs.version_number }}
+    secrets: inherit  # ok since what we are calling is internally maintained

.gitignore

@@ -11,6 +11,7 @@ __pycache__/
 env*/
 dbt_env/
 build/
+!tests/functional/build
 !core/dbt/docs/build
 develop-eggs/
 dist/
@@ -51,6 +52,7 @@ coverage.xml
 *,cover
 .hypothesis/
 test.env
+makefile.test.env
 *.pytest_cache/
 
 

.pre-commit-config.yaml

@@ -1,8 +1,7 @@
 # Configuration for pre-commit hooks (see https://pre-commit.com/).
 # Eventually the hooks described here will be run as tests before merging each PR.
 
-# TODO: remove global exclusion of tests when testing overhaul is complete
-exclude: ^(test/|core/dbt/docs/build/)
+exclude: ^(core/dbt/docs/build/|core/dbt/events/types_pb2.py)
 
 # Force all unspecified python hooks to run python 3.8
 default_language_version:
@@ -38,7 +37,7 @@ repos:
     alias: flake8-check
     stages: [manual]
 - repo: https://github.com/pre-commit/mirrors-mypy
-  rev: v0.942
+  rev: v0.981
   hooks:
   - id: mypy
     # N.B.: Mypy is... a bit fragile.

CHANGELOG.md

@@ -5,11 +5,15 @@
 - "Breaking changes" listed under a version may require action from end users or external maintainers when upgrading to that version.
 - Do not edit this file directly. This file is auto-generated using [changie](https://github.com/miniscruff/changie). For details on how to document a change, see [the contributing guide](https://github.com/dbt-labs/dbt-core/blob/main/CONTRIBUTING.md#adding-changelog-entry)
 
+## dbt-core 1.6.0-a1 - April 17, 2023
+
+
+
 ## Previous Releases
 
 For information on prior major and minor releases, see their changelogs:
 
-
+* [1.5](https://github.com/dbt-labs/dbt-core/blob/1.5.latest/CHANGELOG.md)
 * [1.4](https://github.com/dbt-labs/dbt-core/blob/1.4.latest/CHANGELOG.md)
 * [1.3](https://github.com/dbt-labs/dbt-core/blob/1.3.latest/CHANGELOG.md)
 * [1.2](https://github.com/dbt-labs/dbt-core/blob/1.2.latest/CHANGELOG.md)

Dockerfile.test

@@ -3,7 +3,7 @@
 #  See `/docker` for a generic and production-ready docker file
 ##
 
-FROM ubuntu:22.04
+FROM ubuntu:23.04
 
 ENV DEBIAN_FRONTEND noninteractive
 

Makefile

@@ -6,29 +6,42 @@ ifeq ($(USE_DOCKER),true)
 	DOCKER_CMD := docker-compose run --rm test
 endif
 
-LOGS_DIR := ./logs
+#
+# To override CI_flags, create a file at this repo's root dir named `makefile.test.env`. Fill it
+# with any ENV_VAR overrides required by your test environment, e.g.
+#    DBT_TEST_USER_1=user
+#    LOG_DIR="dir with a space in it"
+#
+# Warn: Restrict each line to one variable only.
+#
+ifeq (./makefile.test.env,$(wildcard ./makefile.test.env))
+	include ./makefile.test.env
+endif
 
-# Optional flag to invoke tests using our CI env.
-# But we always want these active for structured
-# log testing.
 CI_FLAGS =\
-	DBT_TEST_USER_1=dbt_test_user_1\
-	DBT_TEST_USER_2=dbt_test_user_2\
-	DBT_TEST_USER_3=dbt_test_user_3\
-	RUSTFLAGS="-D warnings"\
-	LOG_DIR=./logs\
-	DBT_LOG_FORMAT=json
+	DBT_TEST_USER_1=$(if $(DBT_TEST_USER_1),$(DBT_TEST_USER_1),dbt_test_user_1)\
+	DBT_TEST_USER_2=$(if $(DBT_TEST_USER_2),$(DBT_TEST_USER_2),dbt_test_user_2)\
+	DBT_TEST_USER_3=$(if $(DBT_TEST_USER_3),$(DBT_TEST_USER_3),dbt_test_user_3)\
+	RUSTFLAGS=$(if $(RUSTFLAGS),$(RUSTFLAGS),"-D warnings")\
+	LOG_DIR=$(if $(LOG_DIR),$(LOG_DIR),./logs)\
+	DBT_LOG_FORMAT=$(if $(DBT_LOG_FORMAT),$(DBT_LOG_FORMAT),json)
+
 
 .PHONY: dev_req
 dev_req: ## Installs dbt-* packages in develop mode along with only development dependencies.
 	@\
-	pip install -r dev-requirements.txt -r editable-requirements.txt
+	pip install -r dev-requirements.txt
+	pip install -r editable-requirements.txt
 
 .PHONY: dev
 dev: dev_req ## Installs dbt-* packages in develop mode along with development dependencies and pre-commit.
 	@\
 	pre-commit install
 
+.PHONY: proto_types
+proto_types:  ## generates google protobuf python file from types.proto
+	protoc -I=./core/dbt/events --python_out=./core/dbt/events ./core/dbt/events/types.proto
+
 .PHONY: mypy
 mypy: .env ## Runs mypy against staged changes for static type checking.
 	@\
@@ -66,7 +79,7 @@ test: .env ## Runs unit tests with py and code checks against staged changes.
 .PHONY: integration
 integration: .env ## Runs postgres integration tests with py-integration
 	@\
-	$(if $(USE_CI_FLAGS), $(CI_FLAGS)) $(DOCKER_CMD) tox -e py-integration -- -nauto
+	$(CI_FLAGS) $(DOCKER_CMD) tox -e py-integration -- -nauto
 
 .PHONY: integration-fail-fast
 integration-fail-fast: .env ## Runs postgres integration tests with py-integration in "fail fast" mode.
@@ -76,9 +89,9 @@ integration-fail-fast: .env ## Runs postgres integration tests with py-integrati
 .PHONY: interop
 interop: clean
 	@\
-	mkdir $(LOGS_DIR) && \
+	mkdir $(LOG_DIR) && \
 	$(CI_FLAGS) $(DOCKER_CMD) tox -e py-integration -- -nauto && \
-	LOG_DIR=$(LOGS_DIR) cargo run --manifest-path test/interop/log_parsing/Cargo.toml
+	LOG_DIR=$(LOG_DIR) cargo run --manifest-path test/interop/log_parsing/Cargo.toml
 
 .PHONY: setup-db
 setup-db: ## Setup Postgres database with docker-compose for system testing.

README.md

@@ -21,7 +21,7 @@ These select statements, or "models", form a dbt project. Models frequently buil
 
 ## Getting started
 
-- [Install dbt](https://docs.getdbt.com/docs/installation)
+- [Install dbt](https://docs.getdbt.com/docs/get-started/installation)
 - Read the [introduction](https://docs.getdbt.com/docs/introduction/) and [viewpoint](https://docs.getdbt.com/docs/about/viewpoint/)
 
 ## Join the dbt Community

core/dbt/adapters/base/__init__.py

@@ -1,14 +1,19 @@
 # these are all just exports, #noqa them so flake8 will be happy
 
 # TODO: Should we still include this in the `adapters` namespace?
-from dbt.contracts.connection import Credentials  # noqa
-from dbt.adapters.base.meta import available  # noqa
-from dbt.adapters.base.connections import BaseConnectionManager  # noqa
-from dbt.adapters.base.relation import (  # noqa
+from dbt.contracts.connection import Credentials  # noqa: F401
+from dbt.adapters.base.meta import available  # noqa: F401
+from dbt.adapters.base.connections import BaseConnectionManager  # noqa: F401
+from dbt.adapters.base.relation import (  # noqa: F401
     BaseRelation,
     RelationType,
     SchemaSearchMap,
 )
-from dbt.adapters.base.column import Column  # noqa
-from dbt.adapters.base.impl import AdapterConfig, BaseAdapter, PythonJobHelper  # noqa
-from dbt.adapters.base.plugin import AdapterPlugin  # noqa
+from dbt.adapters.base.column import Column  # noqa: F401
+from dbt.adapters.base.impl import (  # noqa: F401
+    AdapterConfig,
+    BaseAdapter,
+    PythonJobHelper,
+    ConstraintSupport,
+)
+from dbt.adapters.base.plugin import AdapterPlugin  # noqa: F401

core/dbt/adapters/base/column.py

@@ -60,6 +60,7 @@ class Column:
             "float",
             "double precision",
             "float8",
+            "double",
         ]
 
     def is_integer(self) -> bool:

core/dbt/adapters/base/impl.py

@@ -2,46 +2,48 @@ import abc
 from concurrent.futures import as_completed, Future
 from contextlib import contextmanager
 from datetime import datetime
+from enum import Enum
 import time
 from itertools import chain
 from typing import (
-    Optional,
-    Tuple,
-    Callable,
-    Iterable,
-    Type,
-    Dict,
     Any,
+    Callable,
+    Dict,
+    Iterable,
+    Iterator,
     List,
     Mapping,
-    Iterator,
+    Optional,
     Set,
+    Tuple,
+    Type,
+    Union,
 )
 
+from dbt.contracts.graph.nodes import ColumnLevelConstraint, ConstraintType, ModelLevelConstraint
+
 import agate
 import pytz
 
 from dbt.exceptions import (
     DbtInternalError,
+    DbtRuntimeError,
+    DbtValidationError,
     MacroArgTypeError,
     MacroResultError,
-    QuoteConfigTypeError,
     NotImplementedError,
     NullRelationCacheAttemptedError,
     NullRelationDropAttemptedError,
+    QuoteConfigTypeError,
     RelationReturnedMultipleResultsError,
     RenameToNoneAttemptedError,
-    DbtRuntimeError,
     SnapshotTargetIncompleteError,
     SnapshotTargetNotSnapshotTableError,
-    UnexpectedNullError,
     UnexpectedNonTimestampError,
+    UnexpectedNullError,
 )
 
-from dbt.adapters.protocol import (
-    AdapterConfig,
-    ConnectionManagerProtocol,
-)
+from dbt.adapters.protocol import AdapterConfig, ConnectionManagerProtocol
 from dbt.clients.agate_helper import empty_table, merge_tables, table_from_rows
 from dbt.clients.jinja import MacroGenerator
 from dbt.contracts.graph.manifest import Manifest, MacroManifest
@@ -53,8 +55,10 @@ from dbt.events.types import (
     CodeExecution,
     CodeExecutionStatus,
     CatalogGenerationError,
+    ConstraintNotSupported,
+    ConstraintNotEnforced,
 )
-from dbt.utils import filter_null_values, executor, cast_to_str
+from dbt.utils import filter_null_values, executor, cast_to_str, AttrDict
 
 from dbt.adapters.base.connections import Connection, AdapterResponse
 from dbt.adapters.base.meta import AdapterMeta, available
@@ -66,13 +70,19 @@ from dbt.adapters.base.relation import (
 )
 from dbt.adapters.base import Column as BaseColumn
 from dbt.adapters.base import Credentials
-from dbt.adapters.cache import RelationsCache, _make_ref_key_msg
-
+from dbt.adapters.cache import RelationsCache, _make_ref_key_dict
+from dbt import deprecations
 
 GET_CATALOG_MACRO_NAME = "get_catalog"
 FRESHNESS_MACRO_NAME = "collect_freshness"
 
 
+class ConstraintSupport(str, Enum):
+    ENFORCED = "enforced"
+    NOT_ENFORCED = "not_enforced"
+    NOT_SUPPORTED = "not_supported"
+
+
 def _expect_row_value(key: str, row: agate.Row):
     if key not in row.keys():
         raise DbtInternalError(
@@ -177,6 +187,7 @@ class BaseAdapter(metaclass=AdapterMeta):
         - truncate_relation
         - rename_relation
         - get_columns_in_relation
+        - get_column_schema_from_query
         - expand_column_types
         - list_relations_without_caching
         - is_cancelable
@@ -203,6 +214,14 @@ class BaseAdapter(metaclass=AdapterMeta):
     # for use in materializations
     AdapterSpecificConfigs: Type[AdapterConfig] = AdapterConfig
 
+    CONSTRAINT_SUPPORT = {
+        ConstraintType.check: ConstraintSupport.NOT_SUPPORTED,
+        ConstraintType.not_null: ConstraintSupport.ENFORCED,
+        ConstraintType.unique: ConstraintSupport.NOT_ENFORCED,
+        ConstraintType.primary_key: ConstraintSupport.NOT_ENFORCED,
+        ConstraintType.foreign_key: ConstraintSupport.ENFORCED,
+    }
+
     def __init__(self, config):
         self.config = config
         self.cache = RelationsCache()
@@ -255,7 +274,7 @@ class BaseAdapter(metaclass=AdapterMeta):
 
     @available.parse(lambda *a, **k: ("", empty_table()))
     def execute(
-        self, sql: str, auto_begin: bool = False, fetch: bool = False
+        self, sql: str, auto_begin: bool = False, fetch: bool = False, limit: Optional[int] = None
     ) -> Tuple[AdapterResponse, agate.Table]:
         """Execute the given SQL. This is a thin wrapper around
         ConnectionManager.execute.
@@ -264,10 +283,24 @@ class BaseAdapter(metaclass=AdapterMeta):
         :param bool auto_begin: If set, and dbt is not currently inside a
             transaction, automatically begin one.
         :param bool fetch: If set, fetch results.
+        :param Optional[int] limit: If set, only fetch n number of rows
         :return: A tuple of the query status and results (empty if fetch=False).
         :rtype: Tuple[AdapterResponse, agate.Table]
         """
-        return self.connections.execute(sql=sql, auto_begin=auto_begin, fetch=fetch)
+        return self.connections.execute(sql=sql, auto_begin=auto_begin, fetch=fetch, limit=limit)
+
+    @available.parse(lambda *a, **k: [])
+    def get_column_schema_from_query(self, sql: str) -> List[BaseColumn]:
+        """Get a list of the Columns with names and data types from the given sql."""
+        _, cursor = self.connections.add_select_query(sql)
+        columns = [
+            self.Column.create(
+                column_name, self.connections.data_type_code_to_name(column_type_code)
+            )
+            # https://peps.python.org/pep-0249/#description
+            for column_name, column_type_code, *_ in cursor.description
+        ]
+        return columns
 
     @available.parse(lambda *a, **k: ("", empty_table()))
     def get_partitions_metadata(self, table: str) -> Tuple[agate.Table]:
@@ -704,11 +737,23 @@ class BaseAdapter(metaclass=AdapterMeta):
         # we can't build the relations cache because we don't have a
         # manifest so we can't run any operations.
         relations = self.list_relations_without_caching(schema_relation)
+
+        # if the cache is already populated, add this schema in
+        # otherwise, skip updating the cache and just ignore
+        if self.cache:
+            for relation in relations:
+                self.cache.add(relation)
+            if not relations:
+                # it's possible that there were no relations in some schemas. We want
+                # to insert the schemas we query into the cache's `.schemas` attribute
+                # so we can check it later
+                self.cache.update_schemas([(database, schema)])
+
         fire_event(
             ListRelations(
                 database=cast_to_str(database),
                 schema=schema,
-                relations=[_make_ref_key_msg(x) for x in relations],
+                relations=[_make_ref_key_dict(x) for x in relations],
             )
         )
 
@@ -943,7 +988,7 @@ class BaseAdapter(metaclass=AdapterMeta):
         context_override: Optional[Dict[str, Any]] = None,
         kwargs: Dict[str, Any] = None,
         text_only_columns: Optional[Iterable[str]] = None,
-    ) -> agate.Table:
+    ) -> AttrDict:
         """Look macro_name up in the manifest and execute its results.
 
         :param macro_name: The name of the macro to execute.
@@ -1028,7 +1073,7 @@ class BaseAdapter(metaclass=AdapterMeta):
             manifest=manifest,
         )
 
-        results = self._catalog_filter_table(table, manifest)
+        results = self._catalog_filter_table(table, manifest)  # type: ignore[arg-type]
         return results
 
     def get_catalog(self, manifest: Manifest) -> Tuple[agate.Table, List[Exception]]:
@@ -1060,7 +1105,7 @@ class BaseAdapter(metaclass=AdapterMeta):
         loaded_at_field: str,
         filter: Optional[str],
         manifest: Optional[Manifest] = None,
-    ) -> Dict[str, Any]:
+    ) -> Tuple[Optional[AdapterResponse], Dict[str, Any]]:
         """Calculate the freshness of sources in dbt, and return it"""
         kwargs: Dict[str, Any] = {
             "source": source,
@@ -1069,7 +1114,19 @@ class BaseAdapter(metaclass=AdapterMeta):
         }
 
         # run the macro
-        table = self.execute_macro(FRESHNESS_MACRO_NAME, kwargs=kwargs, manifest=manifest)
+        # in older versions of dbt-core, the 'collect_freshness' macro returned the table of results directly
+        # starting in v1.5, by default, we return both the table and the adapter response (metadata about the query)
+        result: Union[
+            AttrDict,  # current: contains AdapterResponse + agate.Table
+            agate.Table,  # previous: just table
+        ]
+        result = self.execute_macro(FRESHNESS_MACRO_NAME, kwargs=kwargs, manifest=manifest)
+        if isinstance(result, agate.Table):
+            deprecations.warn("collect-freshness-return-signature")
+            adapter_response = None
+            table = result
+        else:
+            adapter_response, table = result.response, result.table  # type: ignore[attr-defined]
         # now we have a 1-row table of the maximum `loaded_at_field` value and
         # the current time according to the db.
         if len(table) != 1 or len(table[0]) != 2:
@@ -1083,11 +1140,12 @@ class BaseAdapter(metaclass=AdapterMeta):
 
         snapshotted_at = _utc(table[0][1], source, loaded_at_field)
         age = (snapshotted_at - max_loaded_at).total_seconds()
-        return {
+        freshness = {
             "max_loaded_at": max_loaded_at,
             "snapshotted_at": snapshotted_at,
             "age": age,
         }
+        return adapter_response, freshness
 
     def pre_model_hook(self, config: Mapping[str, Any]) -> Any:
         """A hook for running some operation before the model materialization
@@ -1249,6 +1307,110 @@ class BaseAdapter(metaclass=AdapterMeta):
         # This returns a callable macro
         return model_context[macro_name]
 
+    @classmethod
+    def _parse_column_constraint(cls, raw_constraint: Dict[str, Any]) -> ColumnLevelConstraint:
+        try:
+            ColumnLevelConstraint.validate(raw_constraint)
+            return ColumnLevelConstraint.from_dict(raw_constraint)
+        except Exception:
+            raise DbtValidationError(f"Could not parse constraint: {raw_constraint}")
+
+    @classmethod
+    def render_column_constraint(cls, constraint: ColumnLevelConstraint) -> Optional[str]:
+        """Render the given constraint as DDL text. Should be overriden by adapters which need custom constraint
+        rendering."""
+        if constraint.type == ConstraintType.check and constraint.expression:
+            return f"check {constraint.expression}"
+        elif constraint.type == ConstraintType.not_null:
+            return "not null"
+        elif constraint.type == ConstraintType.unique:
+            return "unique"
+        elif constraint.type == ConstraintType.primary_key:
+            return "primary key"
+        elif constraint.type == ConstraintType.foreign_key:
+            return "foreign key"
+        elif constraint.type == ConstraintType.custom and constraint.expression:
+            return constraint.expression
+        else:
+            return None
+
+    @available
+    @classmethod
+    def render_raw_columns_constraints(cls, raw_columns: Dict[str, Dict[str, Any]]) -> List:
+        rendered_column_constraints = []
+
+        for v in raw_columns.values():
+            rendered_column_constraint = [f"{v['name']} {v['data_type']}"]
+            for con in v.get("constraints", None):
+                constraint = cls._parse_column_constraint(con)
+                c = cls.process_parsed_constraint(constraint, cls.render_column_constraint)
+                if c is not None:
+                    rendered_column_constraint.append(c)
+            rendered_column_constraints.append(" ".join(rendered_column_constraint))
+
+        return rendered_column_constraints
+
+    @classmethod
+    def process_parsed_constraint(
+        cls, parsed_constraint: Union[ColumnLevelConstraint, ModelLevelConstraint], render_func
+    ) -> Optional[str]:
+        if (
+            parsed_constraint.warn_unsupported
+            and cls.CONSTRAINT_SUPPORT[parsed_constraint.type] == ConstraintSupport.NOT_SUPPORTED
+        ):
+            warn_or_error(
+                ConstraintNotSupported(constraint=parsed_constraint.type.value, adapter=cls.type())
+            )
+        if (
+            parsed_constraint.warn_unenforced
+            and cls.CONSTRAINT_SUPPORT[parsed_constraint.type] == ConstraintSupport.NOT_ENFORCED
+        ):
+            warn_or_error(
+                ConstraintNotEnforced(constraint=parsed_constraint.type.value, adapter=cls.type())
+            )
+        if cls.CONSTRAINT_SUPPORT[parsed_constraint.type] != ConstraintSupport.NOT_SUPPORTED:
+            return render_func(parsed_constraint)
+
+        return None
+
+    @classmethod
+    def _parse_model_constraint(cls, raw_constraint: Dict[str, Any]) -> ModelLevelConstraint:
+        try:
+            ModelLevelConstraint.validate(raw_constraint)
+            c = ModelLevelConstraint.from_dict(raw_constraint)
+            return c
+        except Exception:
+            raise DbtValidationError(f"Could not parse constraint: {raw_constraint}")
+
+    @available
+    @classmethod
+    def render_raw_model_constraints(cls, raw_constraints: List[Dict[str, Any]]) -> List[str]:
+        return [c for c in map(cls.render_raw_model_constraint, raw_constraints) if c is not None]
+
+    @classmethod
+    def render_raw_model_constraint(cls, raw_constraint: Dict[str, Any]) -> Optional[str]:
+        constraint = cls._parse_model_constraint(raw_constraint)
+        return cls.process_parsed_constraint(constraint, cls.render_model_constraint)
+
+    @classmethod
+    def render_model_constraint(cls, constraint: ModelLevelConstraint) -> Optional[str]:
+        """Render the given constraint as DDL text. Should be overriden by adapters which need custom constraint
+        rendering."""
+        constraint_prefix = f"constraint {constraint.name} " if constraint.name else ""
+        column_list = ", ".join(constraint.columns)
+        if constraint.type == ConstraintType.check and constraint.expression:
+            return f"{constraint_prefix}check {constraint.expression}"
+        elif constraint.type == ConstraintType.unique:
+            return f"{constraint_prefix}unique ({column_list})"
+        elif constraint.type == ConstraintType.primary_key:
+            return f"{constraint_prefix}primary key ({column_list})"
+        elif constraint.type == ConstraintType.foreign_key:
+            return f"{constraint_prefix}foreign key ({column_list})"
+        elif constraint.type == ConstraintType.custom and constraint.expression:
+            return f"{constraint_prefix}{constraint.expression}"
+        else:
+            return None
+
 
 COLUMNS_EQUAL_SQL = """
 with diff_count as (

core/dbt/adapters/base/plugin.py

@@ -7,9 +7,9 @@ from dbt.adapters.protocol import AdapterProtocol
 
 def project_name_from_path(include_path: str) -> str:
     # avoid an import cycle
-    from dbt.config.project import Project
+    from dbt.config.project import PartialProject
 
-    partial = Project.partial_load(include_path)
+    partial = PartialProject.from_project_root(include_path)
     if partial.project_name is None:
         raise CompilationError(f"Invalid project at {include_path}: name not set!")
     return partial.project_name

core/dbt/adapters/base/relation.py

@@ -227,7 +227,7 @@ class BaseRelation(FakeAPIObject, Hashable):
     def create_from_node(
         cls: Type[Self],
         config: HasQuoting,
-        node: ManifestNode,
+        node,
         quote_policy: Optional[Dict[str, bool]] = None,
         **kwargs: Any,
     ) -> Self:

core/dbt/adapters/cache.py

@@ -4,8 +4,7 @@ from typing import Any, Dict, Iterable, List, Optional, Set, Tuple
 
 from dbt.adapters.reference_keys import (
     _make_ref_key,
-    _make_ref_key_msg,
-    _make_msg_from_ref_key,
+    _make_ref_key_dict,
     _ReferenceKey,
 )
 from dbt.exceptions import (
@@ -17,7 +16,7 @@ from dbt.exceptions import (
 )
 from dbt.events.functions import fire_event, fire_event_if
 from dbt.events.types import CacheAction, CacheDumpGraph
-import dbt.flags as flags
+from dbt.flags import get_flags
 from dbt.utils import lowercase
 
 
@@ -230,7 +229,7 @@ class RelationsCache:
         # self.relations or any cache entry's referenced_by during iteration
         # it's a runtime error!
         with self.lock:
-            return {dot_separated(k): v.dump_graph_entry() for k, v in self.relations.items()}
+            return {dot_separated(k): str(v.dump_graph_entry()) for k, v in self.relations.items()}
 
     def _setdefault(self, relation: _CachedRelation):
         """Add a relation to the cache, or return it if it already exists.
@@ -290,8 +289,8 @@ class RelationsCache:
             # a link - we will never drop the referenced relation during a run.
             fire_event(
                 CacheAction(
-                    ref_key=_make_msg_from_ref_key(ref_key),
-                    ref_key_2=_make_msg_from_ref_key(dep_key),
+                    ref_key=ref_key._asdict(),
+                    ref_key_2=dep_key._asdict(),
                 )
             )
             return
@@ -306,8 +305,8 @@ class RelationsCache:
         fire_event(
             CacheAction(
                 action="add_link",
-                ref_key=_make_msg_from_ref_key(dep_key),
-                ref_key_2=_make_msg_from_ref_key(ref_key),
+                ref_key=dep_key._asdict(),
+                ref_key_2=ref_key._asdict(),
             )
         )
         with self.lock:
@@ -319,12 +318,13 @@ class RelationsCache:
 
         :param BaseRelation relation: The underlying relation.
         """
+        flags = get_flags()
         cached = _CachedRelation(relation)
         fire_event_if(
             flags.LOG_CACHE_EVENTS,
             lambda: CacheDumpGraph(before_after="before", action="adding", dump=self.dump_graph()),
         )
-        fire_event(CacheAction(action="add_relation", ref_key=_make_ref_key_msg(cached)))
+        fire_event(CacheAction(action="add_relation", ref_key=_make_ref_key_dict(cached)))
 
         with self.lock:
             self._setdefault(cached)
@@ -358,7 +358,7 @@ class RelationsCache:
         :param str identifier: The identifier of the relation to drop.
         """
         dropped_key = _make_ref_key(relation)
-        dropped_key_msg = _make_ref_key_msg(relation)
+        dropped_key_msg = _make_ref_key_dict(relation)
         fire_event(CacheAction(action="drop_relation", ref_key=dropped_key_msg))
         with self.lock:
             if dropped_key not in self.relations:
@@ -366,7 +366,7 @@ class RelationsCache:
                 return
             consequences = self.relations[dropped_key].collect_consequences()
             # convert from a list of _ReferenceKeys to a list of ReferenceKeyMsgs
-            consequence_msgs = [_make_msg_from_ref_key(key) for key in consequences]
+            consequence_msgs = [key._asdict() for key in consequences]
             fire_event(
                 CacheAction(
                     action="drop_cascade", ref_key=dropped_key_msg, ref_list=consequence_msgs
@@ -396,9 +396,9 @@ class RelationsCache:
                 fire_event(
                     CacheAction(
                         action="update_reference",
-                        ref_key=_make_ref_key_msg(old_key),
-                        ref_key_2=_make_ref_key_msg(new_key),
-                        ref_key_3=_make_ref_key_msg(cached.key()),
+                        ref_key=_make_ref_key_dict(old_key),
+                        ref_key_2=_make_ref_key_dict(new_key),
+                        ref_key_3=_make_ref_key_dict(cached.key()),
                     )
                 )
 
@@ -429,9 +429,7 @@ class RelationsCache:
             raise TruncatedModelNameCausedCollisionError(new_key, self.relations)
 
         if old_key not in self.relations:
-            fire_event(
-                CacheAction(action="temporary_relation", ref_key=_make_msg_from_ref_key(old_key))
-            )
+            fire_event(CacheAction(action="temporary_relation", ref_key=old_key._asdict()))
             return False
         return True
 
@@ -452,11 +450,11 @@ class RelationsCache:
         fire_event(
             CacheAction(
                 action="rename_relation",
-                ref_key=_make_msg_from_ref_key(old_key),
-                ref_key_2=_make_msg_from_ref_key(new),
+                ref_key=old_key._asdict(),
+                ref_key_2=new_key._asdict(),
             )
         )
-
+        flags = get_flags()
         fire_event_if(
             flags.LOG_CACHE_EVENTS,
             lambda: CacheDumpGraph(before_after="before", action="rename", dump=self.dump_graph()),

core/dbt/adapters/factory.py

@@ -158,6 +158,9 @@ class AdapterContainer:
     def get_adapter_type_names(self, name: Optional[str]) -> List[str]:
         return [p.adapter.type() for p in self.get_adapter_plugins(name)]
 
+    def get_adapter_constraint_support(self, name: Optional[str]) -> List[str]:
+        return self.lookup_adapter(name).CONSTRAINT_SUPPORT  # type: ignore
+
 
 FACTORY: AdapterContainer = AdapterContainer()
 
@@ -214,6 +217,10 @@ def get_adapter_type_names(name: Optional[str]) -> List[str]:
     return FACTORY.get_adapter_type_names(name)
 
 
+def get_adapter_constraint_support(name: Optional[str]) -> List[str]:
+    return FACTORY.get_adapter_constraint_support(name)
+
+
 @contextmanager
 def adapter_management():
     reset_adapters()

core/dbt/adapters/reference_keys.py

@@ -2,7 +2,6 @@
 
 from collections import namedtuple
 from typing import Any, Optional
-from dbt.events.proto_types import ReferenceKeyMsg
 
 
 _ReferenceKey = namedtuple("_ReferenceKey", "database schema identifier")
@@ -30,11 +29,9 @@ def _make_ref_key(relation: Any) -> _ReferenceKey:
     )
 
 
-def _make_ref_key_msg(relation: Any):
-    return _make_msg_from_ref_key(_make_ref_key(relation))
-
-
-def _make_msg_from_ref_key(ref_key: _ReferenceKey) -> ReferenceKeyMsg:
-    return ReferenceKeyMsg(
-        database=ref_key.database, schema=ref_key.schema, identifier=ref_key.identifier
-    )
+def _make_ref_key_dict(relation: Any):
+    return {
+        "database": relation.database,
+        "schema": relation.schema,
+        "identifier": relation.identifier,
+    }

core/dbt/adapters/sql/connections.py

@@ -1,6 +1,6 @@
 import abc
 import time
-from typing import List, Optional, Tuple, Any, Iterable, Dict
+from typing import List, Optional, Tuple, Any, Iterable, Dict, Union
 
 import agate
 
@@ -52,6 +52,7 @@ class SQLConnectionManager(BaseConnectionManager):
         bindings: Optional[Any] = None,
         abridge_sql_log: bool = False,
     ) -> Tuple[Connection, Any]:
+
         connection = self.get_thread_connection()
         if auto_begin and connection.transaction_open is False:
             self.begin()
@@ -117,25 +118,36 @@ class SQLConnectionManager(BaseConnectionManager):
         return [dict(zip(column_names, row)) for row in rows]
 
     @classmethod
-    def get_result_from_cursor(cls, cursor: Any) -> agate.Table:
+    def get_result_from_cursor(cls, cursor: Any, limit: Optional[int]) -> agate.Table:
         data: List[Any] = []
         column_names: List[str] = []
 
         if cursor.description is not None:
             column_names = [col[0] for col in cursor.description]
-            rows = cursor.fetchall()
+            if limit:
+                rows = cursor.fetchmany(limit)
+            else:
+                rows = cursor.fetchall()
             data = cls.process_results(column_names, rows)
 
         return dbt.clients.agate_helper.table_from_data_flat(data, column_names)
 
+    @classmethod
+    def data_type_code_to_name(cls, type_code: Union[int, str]) -> str:
+        """Get the string representation of the data type from the type_code."""
+        # https://peps.python.org/pep-0249/#type-objects
+        raise dbt.exceptions.NotImplementedError(
+            "`data_type_code_to_name` is not implemented for this adapter!"
+        )
+
     def execute(
-        self, sql: str, auto_begin: bool = False, fetch: bool = False
+        self, sql: str, auto_begin: bool = False, fetch: bool = False, limit: Optional[int] = None
     ) -> Tuple[AdapterResponse, agate.Table]:
         sql = self._add_query_comment(sql)
         _, cursor = self.add_query(sql, auto_begin)
         response = self.get_response(cursor)
         if fetch:
-            table = self.get_result_from_cursor(cursor)
+            table = self.get_result_from_cursor(cursor, limit)
         else:
             table = dbt.clients.agate_helper.empty_table()
         return response, table
@@ -146,6 +158,10 @@ class SQLConnectionManager(BaseConnectionManager):
     def add_commit_query(self):
         return self.add_query("COMMIT", auto_begin=False)
 
+    def add_select_query(self, sql: str) -> Tuple[Connection, Any]:
+        sql = self._add_query_comment(sql)
+        return self.add_query(sql, auto_begin=False)
+
     def begin(self):
         connection = self.get_thread_connection()
         if connection.transaction_open is True:

core/dbt/adapters/sql/impl.py

@@ -4,7 +4,7 @@ from typing import Any, Optional, Tuple, Type, List
 from dbt.contracts.connection import Connection
 from dbt.exceptions import RelationTypeNullError
 from dbt.adapters.base import BaseAdapter, available
-from dbt.adapters.cache import _make_ref_key_msg
+from dbt.adapters.cache import _make_ref_key_dict
 from dbt.adapters.sql import SQLConnectionManager
 from dbt.events.functions import fire_event
 from dbt.events.types import ColTypeChange, SchemaCreation, SchemaDrop
@@ -109,7 +109,7 @@ class SQLAdapter(BaseAdapter):
                     ColTypeChange(
                         orig_type=target_column.data_type,
                         new_type=new_type,
-                        table=_make_ref_key_msg(current),
+                        table=_make_ref_key_dict(current),
                     )
                 )
 
@@ -152,7 +152,7 @@ class SQLAdapter(BaseAdapter):
 
     def create_schema(self, relation: BaseRelation) -> None:
         relation = relation.without_identifier()
-        fire_event(SchemaCreation(relation=_make_ref_key_msg(relation)))
+        fire_event(SchemaCreation(relation=_make_ref_key_dict(relation)))
         kwargs = {
             "relation": relation,
         }
@@ -163,7 +163,7 @@ class SQLAdapter(BaseAdapter):
 
     def drop_schema(self, relation: BaseRelation) -> None:
         relation = relation.without_identifier()
-        fire_event(SchemaDrop(relation=_make_ref_key_msg(relation)))
+        fire_event(SchemaDrop(relation=_make_ref_key_dict(relation)))
         kwargs = {
             "relation": relation,
         }

core/dbt/cli/README.md

@@ -1 +1,49 @@
-TODO
+# Exception Handling
+
+## `requires.py`
+
+### `postflight`
+In the postflight decorator, the click command is invoked (i.e. `func(*args, **kwargs)`) and wrapped in a `try/except` block to handle any exceptions thrown.
+Any exceptions thrown from `postflight` are wrapped by custom exceptions from the `dbt.cli.exceptions` module (i.e. `ResultExit`, `ExceptionExit`) to instruct click to complete execution with a particular exit code.
+
+Some `dbt-core` handled exceptions have an attribute named `results` which contains results from running nodes (e.g. `FailFastError`). These are wrapped in the `ResultExit` exception to represent runs that have failed in a way that `dbt-core` expects.
+If the invocation of the command does not throw any exceptions but does not succeed, `postflight` will still raise the `ResultExit` exception to make use of the exit code.
+These exceptions produce an exit code of `1`.
+
+Exceptions wrapped with `ExceptionExit` may be thrown by `dbt-core` intentionally (i.e. an exception that inherits from `dbt.exceptions.Exception`) or unintentionally (i.e. exceptions thrown by the python runtime). In either case these are considered errors that `dbt-core` did not expect and are treated as genuine exceptions.
+These exceptions produce an exit code of `2`.
+
+If no exceptions are thrown from invoking the command and the command succeeds, `postflight` will not raise any exceptions.
+When no exceptions are raised an exit code of `0` is produced.
+
+## `main.py`
+
+### `dbtRunner`
+`dbtRunner` provides a programmatic interface for our click CLI and wraps the invocation of the click commands to handle any exceptions thrown.
+
+`dbtRunner.invoke` should ideally only ever return an instantiated `dbtRunnerResult` which contains the following fields:
+- `success`: A boolean representing whether the command invocation was successful
+- `result`: The optional result of the command invoked. This attribute can have many types, please see the definition of `dbtRunnerResult` for more information
+- `exception`: If an exception was thrown during command invocation it will be saved here, otherwise it will be `None`. Please note that the exceptions held in this attribute are not the exceptions thrown by `preflight` but instead the exceptions that `ResultExit` and `ExceptionExit` wrap
+
+Programmatic exception handling might look like the following:
+```python
+res = dbtRunner().invoke(["run"])
+if not res.success:
+    ...
+if type(res.exception) == SomeExceptionType:
+    ...
+```
+
+## `dbt/tests/util.py`
+
+### `run_dbt`
+In many of our functional and integration tests, we want to be sure that an invocation of `dbt` raises a certain exception.
+A common pattern for these assertions:
+```python
+class TestSomething:
+    def test_something(self, project):
+        with pytest.raises(SomeException):
+            run_dbt(["run"])
+```
+To allow these tests to assert that exceptions have been thrown, the `run_dbt` function will raise any exceptions it recieves from the invocation of a `dbt` command.

core/dbt/cli/__init__.py

@@ -0,0 +1 @@
+from .main import cli as dbt_cli  # noqa

core/dbt/cli/context.py

@@ -0,0 +1,16 @@
+import click
+from typing import Optional
+
+from dbt.cli.main import cli as dbt
+
+
+def make_context(args, command=dbt) -> Optional[click.Context]:
+    try:
+        ctx = command.make_context(command.name, args)
+    except click.exceptions.Exit:
+        return None
+
+    ctx.invoked_subcommand = ctx.protected_args[0] if ctx.protected_args else None
+    ctx.obj = {}
+
+    return ctx

core/dbt/cli/exceptions.py

@@ -0,0 +1,43 @@
+from typing import Optional, IO
+
+from click.exceptions import ClickException
+from dbt.utils import ExitCodes
+
+
+class DbtUsageException(Exception):
+    pass
+
+
+class DbtInternalException(Exception):
+    pass
+
+
+class CliException(ClickException):
+    """The base exception class for our implementation of the click CLI.
+    The exit_code attribute is used by click to determine which exit code to produce
+    after an invocation."""
+
+    def __init__(self, exit_code: ExitCodes) -> None:
+        self.exit_code = exit_code.value
+
+    # the typing of _file is to satisfy the signature of ClickException.show
+    # overriding this method prevents click from printing any exceptions to stdout
+    def show(self, _file: Optional[IO] = None) -> None:
+        pass
+
+
+class ResultExit(CliException):
+    """This class wraps any exception that contains results while invoking dbt, or the
+    results of an invocation that did not succeed but did not throw any exceptions."""
+
+    def __init__(self, result) -> None:
+        super().__init__(ExitCodes.ModelError)
+        self.result = result
+
+
+class ExceptionExit(CliException):
+    """This class wraps any exception that does not contain results thrown while invoking dbt."""
+
+    def __init__(self, exception: Exception) -> None:
+        super().__init__(ExitCodes.UnhandledError)
+        self.exception = exception

core/dbt/cli/flags.py

@@ -1,44 +1,362 @@
-# TODO  Move this to /core/dbt/flags.py when we're ready to break things
 import os
+import sys
 from dataclasses import dataclass
+from importlib import import_module
 from multiprocessing import get_context
 from pprint import pformat as pf
+from typing import Any, Callable, Dict, List, Set, Union
 
-from click import get_current_context
+from click import Context, get_current_context
+from click.core import Command, Group, ParameterSource
+from dbt.cli.exceptions import DbtUsageException
+from dbt.cli.resolvers import default_log_path, default_project_dir
+from dbt.config.profile import read_user_config
+from dbt.contracts.project import UserConfig
+from dbt.exceptions import DbtInternalError
+from dbt.deprecations import renamed_env_var
+from dbt.helper_types import WarnErrorOptions
 
 if os.name != "nt":
     # https://bugs.python.org/issue41567
     import multiprocessing.popen_spawn_posix  # type: ignore  # noqa: F401
 
+FLAGS_DEFAULTS = {
+    "INDIRECT_SELECTION": "eager",
+    "TARGET_PATH": None,
+    # Cli args without user_config or env var option.
+    "FULL_REFRESH": False,
+    "STRICT_MODE": False,
+    "STORE_FAILURES": False,
+    "INTROSPECT": True,
+}
+
+DEPRECATED_PARAMS = {
+    "deprecated_defer": "defer",
+    "deprecated_favor_state": "favor_state",
+    "deprecated_print": "print",
+    "deprecated_state": "state",
+}
+
+
+def convert_config(config_name, config_value):
+    """Convert the values from config and original set_from_args to the correct type."""
+    ret = config_value
+    if config_name.lower() == "warn_error_options" and type(config_value) == dict:
+        ret = WarnErrorOptions(
+            include=config_value.get("include", []), exclude=config_value.get("exclude", [])
+        )
+    return ret
+
+
+def args_to_context(args: List[str]) -> Context:
+    """Convert a list of args to a click context with proper hierarchy for dbt commands"""
+    from dbt.cli.main import cli
+
+    cli_ctx = cli.make_context(cli.name, args)
+    # Split args if they're a comma seperated string.
+    if len(args) == 1 and "," in args[0]:
+        args = args[0].split(",")
+    sub_command_name, sub_command, args = cli.resolve_command(cli_ctx, args)
+
+    # Handle source and docs group.
+    if type(sub_command) == Group:
+        sub_command_name, sub_command, args = sub_command.resolve_command(cli_ctx, args)
+
+    assert type(sub_command) == Command
+    sub_command_ctx = sub_command.make_context(sub_command_name, args)
+    sub_command_ctx.parent = cli_ctx
+    return sub_command_ctx
+
 
 @dataclass(frozen=True)
 class Flags:
-    def __init__(self, ctx=None) -> None:
+    """Primary configuration artifact for running dbt"""
+
+    def __init__(self, ctx: Context = None, user_config: UserConfig = None) -> None:
+
+        # Set the default flags.
+        for key, value in FLAGS_DEFAULTS.items():
+            object.__setattr__(self, key, value)
 
         if ctx is None:
             ctx = get_current_context()
 
-        def assign_params(ctx):
+        def _get_params_by_source(ctx: Context, source_type: ParameterSource):
+            """Generates all params of a given source type."""
+            yield from [
+                name for name, source in ctx._parameter_source.items() if source is source_type
+            ]
+            if ctx.parent:
+                yield from _get_params_by_source(ctx.parent, source_type)
+
+        # Ensure that any params sourced from the commandline are not present more than once.
+        # Click handles this exclusivity, but only at a per-subcommand level.
+        seen_params = []
+        for param in _get_params_by_source(ctx, ParameterSource.COMMANDLINE):
+            if param in seen_params:
+                raise DbtUsageException(
+                    f"{param.lower()} was provided both before and after the subcommand, it can only be set either before or after.",
+                )
+            seen_params.append(param)
+
+        def _assign_params(
+            ctx: Context,
+            params_assigned_from_default: set,
+            deprecated_env_vars: Dict[str, Callable],
+        ):
             """Recursively adds all click params to flag object"""
             for param_name, param_value in ctx.params.items():
                 # N.B. You have to use the base MRO method (object.__setattr__) to set attributes
                 # when using frozen dataclasses.
                 # https://docs.python.org/3/library/dataclasses.html#frozen-instances
-                if hasattr(self, param_name):
-                    raise Exception(f"Duplicate flag names found in click command: {param_name}")
-                object.__setattr__(self, param_name.upper(), param_value)
+
+                # Handle deprecated env vars while still respecting old values
+                # e.g. DBT_NO_PRINT -> DBT_PRINT if DBT_NO_PRINT is set, it is
+                # respected over DBT_PRINT or --print.
+                new_name: Union[str, None] = None
+                if param_name in DEPRECATED_PARAMS:
+
+                    # Deprecated env vars can only be set via env var.
+                    # We use the deprecated option in click to serialize the value
+                    # from the env var string.
+                    param_source = ctx.get_parameter_source(param_name)
+                    if param_source == ParameterSource.DEFAULT:
+                        continue
+                    elif param_source != ParameterSource.ENVIRONMENT:
+                        raise DbtUsageException(
+                            "Deprecated parameters can only be set via environment variables",
+                        )
+
+                    # Rename for clarity.
+                    dep_name = param_name
+                    new_name = DEPRECATED_PARAMS.get(dep_name)
+                    try:
+                        assert isinstance(new_name, str)
+                    except AssertionError:
+                        raise Exception(
+                            f"No deprecated param name match in DEPRECATED_PARAMS from {dep_name} to {new_name}"
+                        )
+
+                    # Find param objects for their envvar name.
+                    try:
+                        dep_param = [x for x in ctx.command.params if x.name == dep_name][0]
+                        new_param = [x for x in ctx.command.params if x.name == new_name][0]
+                    except IndexError:
+                        raise Exception(
+                            f"No deprecated param name match in context from {dep_name} to {new_name}"
+                        )
+
+                    # Remove param from defaulted set since the deprecated
+                    # value is not set from default, but from an env var.
+                    if new_name in params_assigned_from_default:
+                        params_assigned_from_default.remove(new_name)
+
+                    # Add the deprecation warning function to the set.
+                    assert isinstance(dep_param.envvar, str)
+                    assert isinstance(new_param.envvar, str)
+                    deprecated_env_vars[new_name] = renamed_env_var(
+                        old_name=dep_param.envvar,
+                        new_name=new_param.envvar,
+                    )
+
+                # Set the flag value.
+                is_duplicate = hasattr(self, param_name.upper())
+                is_default = ctx.get_parameter_source(param_name) == ParameterSource.DEFAULT
+                flag_name = (new_name or param_name).upper()
+
+                if (is_duplicate and not is_default) or not is_duplicate:
+                    object.__setattr__(self, flag_name, param_value)
+
+                # Track default assigned params.
+                if is_default:
+                    params_assigned_from_default.add(param_name)
+
             if ctx.parent:
-                assign_params(ctx.parent)
+                _assign_params(ctx.parent, params_assigned_from_default, deprecated_env_vars)
 
-        assign_params(ctx)
+        params_assigned_from_default = set()  # type: Set[str]
+        deprecated_env_vars: Dict[str, Callable] = {}
+        _assign_params(ctx, params_assigned_from_default, deprecated_env_vars)
 
-        # Hard coded flags
-        object.__setattr__(self, "WHICH", ctx.info_name)
+        # Set deprecated_env_var_warnings to be fired later after events have been init.
+        object.__setattr__(
+            self, "deprecated_env_var_warnings", [x for x in deprecated_env_vars.values()]
+        )
+
+        # Get the invoked command flags.
+        invoked_subcommand_name = (
+            ctx.invoked_subcommand if hasattr(ctx, "invoked_subcommand") else None
+        )
+        if invoked_subcommand_name is not None:
+            invoked_subcommand = getattr(import_module("dbt.cli.main"), invoked_subcommand_name)
+            invoked_subcommand.allow_extra_args = True
+            invoked_subcommand.ignore_unknown_options = True
+            invoked_subcommand_ctx = invoked_subcommand.make_context(None, sys.argv)
+            _assign_params(
+                invoked_subcommand_ctx, params_assigned_from_default, deprecated_env_vars
+            )
+
+        if not user_config:
+            profiles_dir = getattr(self, "PROFILES_DIR", None)
+            user_config = read_user_config(profiles_dir) if profiles_dir else None
+
+        # Overwrite default assignments with user config if available.
+        if user_config:
+            param_assigned_from_default_copy = params_assigned_from_default.copy()
+            for param_assigned_from_default in params_assigned_from_default:
+                user_config_param_value = getattr(user_config, param_assigned_from_default, None)
+                if user_config_param_value is not None:
+                    object.__setattr__(
+                        self,
+                        param_assigned_from_default.upper(),
+                        convert_config(param_assigned_from_default, user_config_param_value),
+                    )
+                    param_assigned_from_default_copy.remove(param_assigned_from_default)
+            params_assigned_from_default = param_assigned_from_default_copy
+
+        # Set hard coded flags.
+        object.__setattr__(self, "WHICH", invoked_subcommand_name or ctx.info_name)
         object.__setattr__(self, "MP_CONTEXT", get_context("spawn"))
 
-        # Support console DO NOT TRACK initiave
-        if os.getenv("DO_NOT_TRACK", "").lower() in (1, "t", "true", "y", "yes"):
-            object.__setattr__(self, "ANONYMOUS_USAGE_STATS", False)
+        # Apply the lead/follow relationship between some parameters.
+        self._override_if_set("USE_COLORS", "USE_COLORS_FILE", params_assigned_from_default)
+        self._override_if_set("LOG_LEVEL", "LOG_LEVEL_FILE", params_assigned_from_default)
+        self._override_if_set("LOG_FORMAT", "LOG_FORMAT_FILE", params_assigned_from_default)
+
+        # Set default LOG_PATH from PROJECT_DIR, if available.
+        # Starting in v1.5, if `log-path` is set in `dbt_project.yml`, it will raise a deprecation warning,
+        # with the possibility of removing it in a future release.
+        if getattr(self, "LOG_PATH", None) is None:
+            project_dir = getattr(self, "PROJECT_DIR", default_project_dir())
+            version_check = getattr(self, "VERSION_CHECK", True)
+            object.__setattr__(self, "LOG_PATH", default_log_path(project_dir, version_check))
+
+        # Support console DO NOT TRACK initiative.
+        if os.getenv("DO_NOT_TRACK", "").lower() in ("1", "t", "true", "y", "yes"):
+            object.__setattr__(self, "SEND_ANONYMOUS_USAGE_STATS", False)
+
+        # Check mutual exclusivity once all flags are set.
+        self._assert_mutually_exclusive(
+            params_assigned_from_default, ["WARN_ERROR", "WARN_ERROR_OPTIONS"]
+        )
+
+        # Support lower cased access for legacy code.
+        params = set(
+            x for x in dir(self) if not callable(getattr(self, x)) and not x.startswith("__")
+        )
+        for param in params:
+            object.__setattr__(self, param.lower(), getattr(self, param))
 
     def __str__(self) -> str:
         return str(pf(self.__dict__))
+
+    def _override_if_set(self, lead: str, follow: str, defaulted: Set[str]) -> None:
+        """If the value of the lead parameter was set explicitly, apply the value to follow, unless follow was also set explicitly."""
+        if lead.lower() not in defaulted and follow.lower() in defaulted:
+            object.__setattr__(self, follow.upper(), getattr(self, lead.upper(), None))
+
+    def _assert_mutually_exclusive(
+        self, params_assigned_from_default: Set[str], group: List[str]
+    ) -> None:
+        """
+        Ensure no elements from group are simultaneously provided by a user, as inferred from params_assigned_from_default.
+        Raises click.UsageError if any two elements from group are simultaneously provided by a user.
+        """
+        set_flag = None
+        for flag in group:
+            flag_set_by_user = flag.lower() not in params_assigned_from_default
+            if flag_set_by_user and set_flag:
+                raise DbtUsageException(
+                    f"{flag.lower()}: not allowed with argument {set_flag.lower()}"
+                )
+            elif flag_set_by_user:
+                set_flag = flag
+
+    def fire_deprecations(self):
+        """Fires events for deprecated env_var usage."""
+        [dep_fn() for dep_fn in self.deprecated_env_var_warnings]
+        # It is necessary to remove this attr from the class so it does
+        # not get pickled when written to disk as json.
+        object.__delattr__(self, "deprecated_env_var_warnings")
+
+    @classmethod
+    def from_dict_for_cmd(cls, cmd: str, d: Dict[str, Any]) -> "Flags":
+        arg_list = get_args_for_cmd_from_dict(cmd, d)
+        ctx = args_to_context(arg_list)
+        flags = cls(ctx=ctx)
+        flags.fire_deprecations()
+        return flags
+
+
+def get_args_for_cmd_from_dict(cmd: str, d: Dict[str, Any]) -> List[str]:
+    """Given a command name and a dict, returns a list of strings representing
+    the CLI arguments that for a command. The order of this list is consistent with
+    which flags are expected at the parent level vs the command level.
+
+    e.g. fn("run", {"defer": True, "print": False}) -> ["--no-print", "run", "--defer"]
+
+    The result of this function can be passed in to the args_to_context function
+    to produce a click context to instantiate Flags with.
+    """
+
+    cmd_args = get_args_for_cmd(cmd)
+    parent_args = get_args_for_cmd("cli")
+    default_args = [x.lower() for x in FLAGS_DEFAULTS.keys()]
+
+    res = [cmd]
+
+    for k, v in d.items():
+        k = k.lower()
+
+        # if a "which" value exists in the args dict, it should match the cmd arg
+        if k == "which":
+            if v != cmd.lower():
+                raise DbtInternalError(f"cmd '{cmd}' does not match value of which '{v}'")
+            continue
+
+        # param was assigned from defaults and should not be included
+        if k not in cmd_args + parent_args and k in default_args:
+            continue
+
+        # if the param is in parent args, it should come before the arg name
+        # e.g. ["--print", "run"] vs ["run", "--print"]
+        add_fn = res.append
+        if k in parent_args:
+            add_fn = lambda x: res.insert(0, x)
+
+        spinal_cased = k.replace("_", "-")
+
+        if v in (None, False):
+            add_fn(f"--no-{spinal_cased}")
+        elif v is True:
+            add_fn(f"--{spinal_cased}")
+        else:
+            add_fn(f"--{spinal_cased}={v}")
+
+    return res
+
+
+def get_args_for_cmd(name: str) -> List[str]:
+    """Given the string name of a command, return a list of strings representing
+    the params that command takes. This function will not return params assigned
+    to a parent click click when passed the name of a child click command.
+
+    e.g. fn("run") -> ["defer", "favor_state", "exclude", ...]
+    """
+    import dbt.cli.main as cli
+    CMD_DICT = {
+        "build": cli.build,
+        "cli": cli.cli,
+        "compile": cli.compile,
+        "freshness": cli.freshness,
+        "generate": cli.docs_generate,
+        "run": cli.run,
+        "seed": cli.seed,
+        "show": cli.show,
+        "snapshot": cli.snapshot,
+        "test": cli.test,
+    }
+    cmd = CMD_DICT.get(name, None)
+    if cmd is None:
+        raise DbtInternalError(f"No command found for name '{name}'")
+    return [x.name for x in cmd.params if not x.name.lower().startswith("deprecated_")]

core/dbt/cli/main.py

@@ -1,22 +1,122 @@
-import inspect  # This is temporary for RAT-ing
 from copy import copy
-from pprint import pformat as pf  # This is temporary for RAT-ing
+from dataclasses import dataclass
+from typing import Callable, List, Optional, Union
 
 import click
-from dbt.adapters.factory import adapter_management
-from dbt.cli import params as p
-from dbt.cli.flags import Flags
-from dbt.profiler import profiler
+from click.exceptions import (
+    Exit as ClickExit,
+    BadOptionUsage,
+    NoSuchOption,
+    UsageError,
+)
+
+from dbt.cli import requires, params as p
+from dbt.cli.exceptions import (
+    DbtInternalException,
+    DbtUsageException,
+)
+from dbt.contracts.graph.manifest import Manifest
+from dbt.contracts.results import (
+    CatalogArtifact,
+    RunExecutionResult,
+    RunOperationResultsArtifact,
+)
+from dbt.events.base_types import EventMsg
+from dbt.task.build import BuildTask
+from dbt.task.clean import CleanTask
+from dbt.task.compile import CompileTask
+from dbt.task.debug import DebugTask
+from dbt.task.deps import DepsTask
+from dbt.task.freshness import FreshnessTask
+from dbt.task.generate import GenerateTask
+from dbt.task.init import InitTask
+from dbt.task.list import ListTask
+from dbt.task.retry import RetryTask
+from dbt.task.run import RunTask
+from dbt.task.run_operation import RunOperationTask
+from dbt.task.seed import SeedTask
+from dbt.task.serve import ServeTask
+from dbt.task.show import ShowTask
+from dbt.task.snapshot import SnapshotTask
+from dbt.task.test import TestTask
 
 
-def cli_runner():
-    # Alias "list" to "ls"
-    ls = copy(cli.commands["list"])
-    ls.hidden = True
-    cli.add_command(ls, "ls")
+@dataclass
+class dbtRunnerResult:
+    """Contains the result of an invocation of the dbtRunner"""
 
-    # Run the cli
-    cli()
+    success: bool
+
+    exception: Optional[BaseException] = None
+    result: Union[
+        bool,  # debug
+        CatalogArtifact,  # docs generate
+        List[str],  # list/ls
+        Manifest,  # parse
+        None,  # clean, deps, init, source
+        RunExecutionResult,  # build, compile, run, seed, snapshot, test
+        RunOperationResultsArtifact,  # run-operation
+    ] = None
+
+
+# Programmatic invocation
+class dbtRunner:
+    def __init__(
+        self,
+        manifest: Manifest = None,
+        callbacks: List[Callable[[EventMsg], None]] = None,
+    ):
+        self.manifest = manifest
+
+        if callbacks is None:
+            callbacks = []
+        self.callbacks = callbacks
+
+    def invoke(self, args: List[str], **kwargs) -> dbtRunnerResult:
+        try:
+            dbt_ctx = cli.make_context(cli.name, args)
+            dbt_ctx.obj = {
+                "manifest": self.manifest,
+                "callbacks": self.callbacks,
+            }
+
+            for key, value in kwargs.items():
+                dbt_ctx.params[key] = value
+                # Hack to set parameter source to custom string
+                dbt_ctx.set_parameter_source(key, "kwargs")  # type: ignore
+
+            result, success = cli.invoke(dbt_ctx)
+            return dbtRunnerResult(
+                result=result,
+                success=success,
+            )
+        except requires.ResultExit as e:
+            return dbtRunnerResult(
+                result=e.result,
+                success=False,
+            )
+        except requires.ExceptionExit as e:
+            return dbtRunnerResult(
+                exception=e.exception,
+                success=False,
+            )
+        except (BadOptionUsage, NoSuchOption, UsageError) as e:
+            return dbtRunnerResult(
+                exception=DbtUsageException(e.message),
+                success=False,
+            )
+        except ClickExit as e:
+            if e.exit_code == 0:
+                return dbtRunnerResult(success=True)
+            return dbtRunnerResult(
+                exception=DbtInternalException(f"unhandled exit code {e.exit_code}"),
+                success=False,
+            )
+        except BaseException as e:
+            return dbtRunnerResult(
+                exception=e,
+                success=False,
+            )
 
 
 # dbt
@@ -27,21 +127,29 @@ def cli_runner():
     epilog="Specify one of these sub-commands and you can find more help from there.",
 )
 @click.pass_context
-@p.anonymous_usage_stats
 @p.cache_selected_only
 @p.debug
+@p.deprecated_print
 @p.enable_legacy_logger
 @p.fail_fast
 @p.log_cache_events
 @p.log_format
+@p.log_format_file
+@p.log_level
+@p.log_level_file
+@p.log_path
 @p.macro_debugging
 @p.partial_parse
+@p.populate_cache
 @p.print
 @p.printer_width
 @p.quiet
 @p.record_timing_info
+@p.send_anonymous_usage_stats
+@p.single_threaded
 @p.static_parser
 @p.use_colors
+@p.use_colors_file
 @p.use_experimental_parser
 @p.version
 @p.version_check
@@ -52,49 +160,51 @@ def cli(ctx, **kwargs):
     """An ELT tool for managing your SQL transformations and data models.
     For more documentation on these commands, visit: docs.getdbt.com
     """
-    incomplete_flags = Flags()
-
-    # Profiling
-    if incomplete_flags.RECORD_TIMING_INFO:
-        ctx.with_resource(profiler(enable=True, outfile=incomplete_flags.RECORD_TIMING_INFO))
-
-    # Adapter management
-    ctx.with_resource(adapter_management())
-
-    # Version info
-    if incomplete_flags.VERSION:
-        click.echo(f"`version` called\n ctx.params: {pf(ctx.params)}")
-        return
-    else:
-        del ctx.params["version"]
 
 
 # dbt build
 @cli.command("build")
 @click.pass_context
 @p.defer
+@p.deprecated_defer
 @p.exclude
 @p.fail_fast
+@p.favor_state
+@p.deprecated_favor_state
 @p.full_refresh
 @p.indirect_selection
-@p.log_path
-@p.models
 @p.profile
 @p.profiles_dir
 @p.project_dir
+@p.resource_type
+@p.select
 @p.selector
 @p.show
 @p.state
+@p.deprecated_state
 @p.store_failures
 @p.target
 @p.target_path
 @p.threads
 @p.vars
 @p.version_check
+@requires.postflight
+@requires.preflight
+@requires.profile
+@requires.project
+@requires.runtime_config
+@requires.manifest
 def build(ctx, **kwargs):
-    """Run all Seeds, Models, Snapshots, and tests in DAG order"""
-    flags = Flags()
-    click.echo(f"`{inspect.stack()[0][3]}` called\n flags: {flags}")
+    """Run all seeds, models, snapshots, and tests in DAG order"""
+    task = BuildTask(
+        ctx.obj["flags"],
+        ctx.obj["runtime_config"],
+        ctx.obj["manifest"],
+    )
+
+    results = task.run()
+    success = task.interpret_results(results)
+    return results, success
 
 
 # dbt clean
@@ -105,10 +215,17 @@ def build(ctx, **kwargs):
 @p.project_dir
 @p.target
 @p.vars
+@requires.postflight
+@requires.preflight
+@requires.unset_profile
+@requires.project
 def clean(ctx, **kwargs):
     """Delete all folders in the clean-targets list (usually the dbt_packages and target directories.)"""
-    flags = Flags()
-    click.echo(f"`{inspect.stack()[0][3]}` called\n flags: {flags}")
+    task = CleanTask(ctx.obj["flags"], ctx.obj["project"])
+
+    results = task.run()
+    success = task.interpret_results(results)
+    return results, success
 
 
 # dbt docs
@@ -123,23 +240,40 @@ def docs(ctx, **kwargs):
 @click.pass_context
 @p.compile_docs
 @p.defer
+@p.deprecated_defer
 @p.exclude
-@p.log_path
-@p.models
+@p.favor_state
+@p.deprecated_favor_state
 @p.profile
 @p.profiles_dir
 @p.project_dir
+@p.select
 @p.selector
+@p.empty_catalog
 @p.state
+@p.deprecated_state
 @p.target
 @p.target_path
 @p.threads
 @p.vars
 @p.version_check
+@requires.postflight
+@requires.preflight
+@requires.profile
+@requires.project
+@requires.runtime_config
+@requires.manifest(write=False)
 def docs_generate(ctx, **kwargs):
     """Generate the documentation website for your project"""
-    flags = Flags()
-    click.echo(f"`{inspect.stack()[0][3]}` called\n flags: {flags}")
+    task = GenerateTask(
+        ctx.obj["flags"],
+        ctx.obj["runtime_config"],
+        ctx.obj["manifest"],
+    )
+
+    results = task.run()
+    success = task.interpret_results(results)
+    return results, success
 
 
 # dbt docs serve
@@ -152,35 +286,112 @@ def docs_generate(ctx, **kwargs):
 @p.project_dir
 @p.target
 @p.vars
+@requires.postflight
+@requires.preflight
+@requires.profile
+@requires.project
+@requires.runtime_config
 def docs_serve(ctx, **kwargs):
     """Serve the documentation website for your project"""
-    flags = Flags()
-    click.echo(f"`{inspect.stack()[0][3]}` called\n flags: {flags}")
+    task = ServeTask(
+        ctx.obj["flags"],
+        ctx.obj["runtime_config"],
+    )
+
+    results = task.run()
+    success = task.interpret_results(results)
+    return results, success
 
 
 # dbt compile
 @cli.command("compile")
 @click.pass_context
 @p.defer
+@p.deprecated_defer
 @p.exclude
+@p.favor_state
+@p.deprecated_favor_state
 @p.full_refresh
-@p.log_path
-@p.models
-@p.parse_only
+@p.show_output_format
+@p.indirect_selection
+@p.introspect
 @p.profile
 @p.profiles_dir
 @p.project_dir
+@p.select
 @p.selector
+@p.inline
 @p.state
+@p.deprecated_state
 @p.target
 @p.target_path
 @p.threads
 @p.vars
 @p.version_check
+@requires.postflight
+@requires.preflight
+@requires.profile
+@requires.project
+@requires.runtime_config
+@requires.manifest
 def compile(ctx, **kwargs):
-    """Generates executable SQL from source, model, test, and analysis files. Compiled SQL files are written to the target/ directory."""
-    flags = Flags()
-    click.echo(f"`{inspect.stack()[0][3]}` called\n flags: {flags}")
+    """Generates executable SQL from source, model, test, and analysis files. Compiled SQL files are written to the
+    target/ directory."""
+    task = CompileTask(
+        ctx.obj["flags"],
+        ctx.obj["runtime_config"],
+        ctx.obj["manifest"],
+    )
+
+    results = task.run()
+    success = task.interpret_results(results)
+    return results, success
+
+
+# dbt show
+@cli.command("show")
+@click.pass_context
+@p.defer
+@p.deprecated_defer
+@p.exclude
+@p.favor_state
+@p.deprecated_favor_state
+@p.full_refresh
+@p.show_output_format
+@p.show_limit
+@p.indirect_selection
+@p.introspect
+@p.profile
+@p.profiles_dir
+@p.project_dir
+@p.select
+@p.selector
+@p.inline
+@p.state
+@p.deprecated_state
+@p.target
+@p.target_path
+@p.threads
+@p.vars
+@p.version_check
+@requires.postflight
+@requires.preflight
+@requires.profile
+@requires.project
+@requires.runtime_config
+@requires.manifest
+def show(ctx, **kwargs):
+    """Generates executable SQL for a named resource or inline query, runs that SQL, and returns a preview of the
+    results. Does not materialize anything to the warehouse."""
+    task = ShowTask(
+        ctx.obj["flags"],
+        ctx.obj["runtime_config"],
+        ctx.obj["manifest"],
+    )
+
+    results = task.run()
+    success = task.interpret_results(results)
+    return results, success
 
 
 # dbt debug
@@ -188,15 +399,24 @@ def compile(ctx, **kwargs):
 @click.pass_context
 @p.config_dir
 @p.profile
-@p.profiles_dir
+@p.profiles_dir_exists_false
 @p.project_dir
 @p.target
 @p.vars
 @p.version_check
+@requires.postflight
+@requires.preflight
 def debug(ctx, **kwargs):
-    """Show some helpful information about dbt for debugging. Not to be confused with the --debug option which increases verbosity."""
-    flags = Flags()
-    click.echo(f"`{inspect.stack()[0][3]}` called\n flags: {flags}")
+    """Test the database connection and show information for debugging purposes. Not to be confused with the --debug option which increases verbosity."""
+
+    task = DebugTask(
+        ctx.obj["flags"],
+        None,
+    )
+
+    results = task.run()
+    success = task.interpret_results(results)
+    return results, success
 
 
 # dbt deps
@@ -207,25 +427,38 @@ def debug(ctx, **kwargs):
 @p.project_dir
 @p.target
 @p.vars
+@requires.postflight
+@requires.preflight
+@requires.unset_profile
+@requires.project
 def deps(ctx, **kwargs):
     """Pull the most recent version of the dependencies listed in packages.yml"""
-    flags = Flags()
-    click.echo(f"`{inspect.stack()[0][3]}` called\n flags: {flags}")
+    task = DepsTask(ctx.obj["flags"], ctx.obj["project"])
+    results = task.run()
+    success = task.interpret_results(results)
+    return results, success
 
 
 # dbt init
 @cli.command("init")
 @click.pass_context
+# for backwards compatibility, accept 'project_name' as an optional positional argument
+@click.argument("project_name", required=False)
 @p.profile
 @p.profiles_dir
 @p.project_dir
 @p.skip_profile_setup
 @p.target
 @p.vars
+@requires.postflight
+@requires.preflight
 def init(ctx, **kwargs):
-    """Initialize a new DBT project."""
-    flags = Flags()
-    click.echo(f"`{inspect.stack()[0][3]}` called\n flags: {flags}")
+    """Initialize a new dbt project."""
+    task = InitTask(ctx.obj["flags"], None)
+
+    results = task.run()
+    success = task.interpret_results(results)
+    return results, success
 
 
 # dbt list
@@ -240,21 +473,40 @@ def init(ctx, **kwargs):
 @p.profiles_dir
 @p.project_dir
 @p.resource_type
+@p.raw_select
 @p.selector
 @p.state
+@p.deprecated_state
 @p.target
 @p.vars
+@requires.postflight
+@requires.preflight
+@requires.profile
+@requires.project
+@requires.runtime_config
+@requires.manifest
 def list(ctx, **kwargs):
     """List the resources in your project"""
-    flags = Flags()
-    click.echo(f"`{inspect.stack()[0][3]}` called\n flags: {flags}")
+    task = ListTask(
+        ctx.obj["flags"],
+        ctx.obj["runtime_config"],
+        ctx.obj["manifest"],
+    )
+
+    results = task.run()
+    success = task.interpret_results(results)
+    return results, success
+
+
+# Alias "list" to "ls"
+ls = copy(cli.commands["list"])
+ls.hidden = True
+cli.add_command(ls, "ls")
 
 
 # dbt parse
 @cli.command("parse")
 @click.pass_context
-@p.compile_parse
-@p.log_path
 @p.profile
 @p.profiles_dir
 @p.project_dir
@@ -263,51 +515,126 @@ def list(ctx, **kwargs):
 @p.threads
 @p.vars
 @p.version_check
-@p.write_manifest
+@requires.postflight
+@requires.preflight
+@requires.profile
+@requires.project
+@requires.runtime_config
+@requires.manifest(write_perf_info=True)
 def parse(ctx, **kwargs):
     """Parses the project and provides information on performance"""
-    flags = Flags()
-    click.echo(f"`{inspect.stack()[0][3]}` called\n flags: {flags}")
+    # manifest generation and writing happens in @requires.manifest
+
+    return ctx.obj["manifest"], True
+
+
+# chenyu: it is actually kind of confusing what arguments I need to include here, some of
+# them are for project profile, feels like they should be defined alongside those decorators
+# dbt retry
+@cli.command("retry")
+@click.pass_context
+@p.defer
+@p.deprecated_defer
+@p.fail_fast
+@p.favor_state
+@p.deprecated_favor_state
+@p.indirect_selection #?
+@p.profile
+@p.profiles_dir
+@p.project_dir
+@p.state_required
+@p.deprecated_state
+@p.store_failures #?
+@p.target
+@p.target_path
+@p.threads
+@p.vars #?
+@requires.preflight
+@requires.profile
+@requires.project
+@requires.runtime_config
+@requires.manifest
+def retry(ctx, **kwargs):
+    """Rerun a previous failed command given a previous run result artifact"""
+    task = RetryTask(
+        ctx.obj["flags"],
+        ctx.obj["runtime_config"],
+        ctx.obj["manifest"],
+    )
+
+    results = task.run()
+    success = task.interpret_results(results)
+    return results, success
 
 
 # dbt run
 @cli.command("run")
 @click.pass_context
 @p.defer
+@p.deprecated_defer
+@p.favor_state
+@p.deprecated_favor_state
 @p.exclude
 @p.fail_fast
 @p.full_refresh
-@p.log_path
-@p.models
 @p.profile
 @p.profiles_dir
 @p.project_dir
+@p.select
 @p.selector
 @p.state
+@p.deprecated_state
 @p.target
 @p.target_path
 @p.threads
 @p.vars
 @p.version_check
+@requires.postflight
+@requires.preflight
+@requires.profile
+@requires.project
+@requires.runtime_config
+@requires.manifest
 def run(ctx, **kwargs):
     """Compile SQL and execute against the current target database."""
-    flags = Flags()
-    click.echo(f"`{inspect.stack()[0][3]}` called\n flags: {flags}")
+    task = RunTask(
+        ctx.obj["flags"],
+        ctx.obj["runtime_config"],
+        ctx.obj["manifest"],
+    )
+
+    results = task.run()
+    success = task.interpret_results(results)
+    return results, success
 
 
 # dbt run operation
 @cli.command("run-operation")
 @click.pass_context
+@click.argument("macro")
 @p.args
 @p.profile
 @p.profiles_dir
 @p.project_dir
 @p.target
 @p.vars
+@requires.postflight
+@requires.preflight
+@requires.profile
+@requires.project
+@requires.runtime_config
+@requires.manifest
 def run_operation(ctx, **kwargs):
     """Run the named macro with any supplied arguments."""
-    flags = Flags()
-    click.echo(f"`{inspect.stack()[0][3]}` called\n flags: {flags}")
+    task = RunOperationTask(
+        ctx.obj["flags"],
+        ctx.obj["runtime_config"],
+        ctx.obj["manifest"],
+    )
+
+    results = task.run()
+    success = task.interpret_results(results)
+    return results, success
 
 
 # dbt seed
@@ -315,43 +642,73 @@ def run_operation(ctx, **kwargs):
 @click.pass_context
 @p.exclude
 @p.full_refresh
-@p.log_path
-@p.models
 @p.profile
 @p.profiles_dir
 @p.project_dir
+@p.select
 @p.selector
 @p.show
 @p.state
+@p.deprecated_state
 @p.target
 @p.target_path
 @p.threads
 @p.vars
 @p.version_check
+@requires.postflight
+@requires.preflight
+@requires.profile
+@requires.project
+@requires.runtime_config
+@requires.manifest
 def seed(ctx, **kwargs):
     """Load data from csv files into your data warehouse."""
-    flags = Flags()
-    click.echo(f"`{inspect.stack()[0][3]}` called\n flags: {flags}")
+    task = SeedTask(
+        ctx.obj["flags"],
+        ctx.obj["runtime_config"],
+        ctx.obj["manifest"],
+    )
+    results = task.run()
+    success = task.interpret_results(results)
+    return results, success
 
 
 # dbt snapshot
 @cli.command("snapshot")
 @click.pass_context
 @p.defer
+@p.deprecated_defer
 @p.exclude
-@p.models
+@p.favor_state
+@p.deprecated_favor_state
 @p.profile
 @p.profiles_dir
 @p.project_dir
+@p.select
 @p.selector
 @p.state
+@p.deprecated_state
 @p.target
+@p.target_path
 @p.threads
 @p.vars
+@requires.postflight
+@requires.preflight
+@requires.profile
+@requires.project
+@requires.runtime_config
+@requires.manifest
 def snapshot(ctx, **kwargs):
     """Execute snapshots defined in your project"""
-    flags = Flags()
-    click.echo(f"`{inspect.stack()[0][3]}` called\n flags: {flags}")
+    task = SnapshotTask(
+        ctx.obj["flags"],
+        ctx.obj["runtime_config"],
+        ctx.obj["manifest"],
+    )
+
+    results = task.run()
+    success = task.interpret_results(results)
+    return results, success
 
 
 # dbt source
@@ -365,48 +722,84 @@ def source(ctx, **kwargs):
 @source.command("freshness")
 @click.pass_context
 @p.exclude
-@p.models
 @p.output_path  # TODO: Is this ok to re-use?  We have three different output params, how much can we consolidate?
 @p.profile
 @p.profiles_dir
 @p.project_dir
+@p.select
 @p.selector
 @p.state
+@p.deprecated_state
 @p.target
 @p.threads
 @p.vars
+@requires.postflight
+@requires.preflight
+@requires.profile
+@requires.project
+@requires.runtime_config
+@requires.manifest
 def freshness(ctx, **kwargs):
-    """Snapshots the current freshness of the project's sources"""
-    flags = Flags()
-    click.echo(f"`{inspect.stack()[0][3]}` called\n flags: {flags}")
+    """check the current freshness of the project's sources"""
+    task = FreshnessTask(
+        ctx.obj["flags"],
+        ctx.obj["runtime_config"],
+        ctx.obj["manifest"],
+    )
+
+    results = task.run()
+    success = task.interpret_results(results)
+    return results, success
+
+
+# Alias "source freshness" to "snapshot-freshness"
+snapshot_freshness = copy(cli.commands["source"].commands["freshness"])  # type: ignore
+snapshot_freshness.hidden = True
+cli.commands["source"].add_command(snapshot_freshness, "snapshot-freshness")  # type: ignore
 
 
 # dbt test
 @cli.command("test")
 @click.pass_context
 @p.defer
+@p.deprecated_defer
 @p.exclude
 @p.fail_fast
+@p.favor_state
+@p.deprecated_favor_state
 @p.indirect_selection
-@p.log_path
-@p.models
 @p.profile
 @p.profiles_dir
 @p.project_dir
+@p.select
 @p.selector
 @p.state
+@p.deprecated_state
 @p.store_failures
 @p.target
 @p.target_path
 @p.threads
 @p.vars
 @p.version_check
+@requires.postflight
+@requires.preflight
+@requires.profile
+@requires.project
+@requires.runtime_config
+@requires.manifest
 def test(ctx, **kwargs):
     """Runs tests on data in deployed models. Run this after `dbt run`"""
-    flags = Flags()
-    click.echo(f"`{inspect.stack()[0][3]}` called\n flags: {flags}")
+    task = TestTask(
+        ctx.obj["flags"],
+        ctx.obj["runtime_config"],
+        ctx.obj["manifest"],
+    )
+
+    results = task.run()
+    success = task.interpret_results(results)
+    return results, success
 
 
 # Support running as a module
 if __name__ == "__main__":
-    cli_runner()
+    cli()

core/dbt/cli/option_types.py

@@ -1,5 +1,7 @@
-from click import ParamType
-import yaml
+from click import ParamType, Choice
+
+from dbt.config.utils import parse_cli_vars
+from dbt.exceptions import ValidationError
 
 from dbt.helper_types import WarnErrorOptions
 
@@ -14,8 +16,8 @@ class YAML(ParamType):
         if not isinstance(value, str):
             self.fail(f"Cannot load YAML from type {type(value)}", param, ctx)
         try:
-            return yaml.load(value, Loader=yaml.Loader)
-        except yaml.parser.ParserError:
+            return parse_cli_vars(value)
+        except ValidationError:
             self.fail(f"String '{value}' is not valid YAML", param, ctx)
 
 
@@ -25,6 +27,7 @@ class WarnErrorOptionsType(YAML):
     name = "WarnErrorOptionsType"
 
     def convert(self, value, param, ctx):
+        # this function is being used by param in click
         include_exclude = super().convert(value, param, ctx)
 
         return WarnErrorOptions(
@@ -46,3 +49,13 @@ class Truthy(ParamType):
             return None
         else:
             return value
+
+
+class ChoiceTuple(Choice):
+    name = "CHOICE_TUPLE"
+
+    def convert(self, value, param, ctx):
+        for value_item in value:
+            super().convert(value_item, param, ctx)
+
+        return value

core/dbt/cli/options.py

@@ -0,0 +1,75 @@
+import click
+import inspect
+import typing as t
+from click import Context
+from dbt.cli.option_types import ChoiceTuple
+
+
+# Implementation from: https://stackoverflow.com/a/48394004
+# Note MultiOption options must be specified with type=tuple or type=ChoiceTuple (https://github.com/pallets/click/issues/2012)
+class MultiOption(click.Option):
+    def __init__(self, *args, **kwargs):
+        self.save_other_options = kwargs.pop("save_other_options", True)
+        nargs = kwargs.pop("nargs", -1)
+        assert nargs == -1, "nargs, if set, must be -1 not {}".format(nargs)
+        super(MultiOption, self).__init__(*args, **kwargs)
+        self._previous_parser_process = None
+        self._eat_all_parser = None
+
+        # validate that multiple=True
+        multiple = kwargs.pop("multiple", None)
+        msg = f"MultiOption named `{self.name}` must have multiple=True (rather than {multiple})"
+        assert multiple, msg
+
+        # validate that type=tuple or type=ChoiceTuple
+        option_type = kwargs.pop("type", None)
+        msg = f"MultiOption named `{self.name}` must be tuple or ChoiceTuple (rather than {option_type})"
+        if inspect.isclass(option_type):
+            assert issubclass(option_type, tuple), msg
+        else:
+            assert isinstance(option_type, ChoiceTuple), msg
+
+    def add_to_parser(self, parser, ctx):
+        def parser_process(value, state):
+            # method to hook to the parser.process
+            done = False
+            value = [value]
+            if self.save_other_options:
+                # grab everything up to the next option
+                while state.rargs and not done:
+                    for prefix in self._eat_all_parser.prefixes:
+                        if state.rargs[0].startswith(prefix):
+                            done = True
+                    if not done:
+                        value.append(state.rargs.pop(0))
+            else:
+                # grab everything remaining
+                value += state.rargs
+                state.rargs[:] = []
+            value = tuple(value)
+            # call the actual process
+            self._previous_parser_process(value, state)
+
+        retval = super(MultiOption, self).add_to_parser(parser, ctx)
+        for name in self.opts:
+            our_parser = parser._long_opt.get(name) or parser._short_opt.get(name)
+            if our_parser:
+                self._eat_all_parser = our_parser
+                self._previous_parser_process = our_parser.process
+                our_parser.process = parser_process
+                break
+        return retval
+
+    def type_cast_value(self, ctx: Context, value: t.Any) -> t.Any:
+        def flatten(data):
+            if isinstance(data, tuple):
+                for x in data:
+                    yield from flatten(x)
+            else:
+                yield data
+
+        # there will be nested tuples to flatten when multiple=True
+        value = super(MultiOption, self).type_cast_value(ctx, value)
+        if value:
+            value = tuple(flatten(value))
+        return value

core/dbt/cli/params.py

@@ -1,20 +1,10 @@
-from pathlib import Path, PurePath
+from pathlib import Path
 
 import click
-from dbt.cli.option_types import YAML, WarnErrorOptionsType
+from dbt.cli.options import MultiOption
+from dbt.cli.option_types import YAML, ChoiceTuple, WarnErrorOptionsType
 from dbt.cli.resolvers import default_project_dir, default_profiles_dir
-
-
-# TODO:  The name (reflected in flags) is a correction!
-# The original name was `SEND_ANONYMOUS_USAGE_STATS` and used an env var called "DBT_SEND_ANONYMOUS_USAGE_STATS"
-# Both of which break existing naming conventions (doesn't match param flag).
-# This will need to be fixed before use in the main codebase and communicated as a change to the community!
-anonymous_usage_stats = click.option(
-    "--anonymous-usage-stats/--no-anonymous-usage-stats",
-    envvar="DBT_ANONYMOUS_USAGE_STATS",
-    help="Send anonymous usage stats to dbt Labs.",
-    default=True,
-)
+from dbt.version import get_version_information
 
 args = click.option(
     "--args",
@@ -33,28 +23,28 @@ browser = click.option(
 cache_selected_only = click.option(
     "--cache-selected-only/--no-cache-selected-only",
     envvar="DBT_CACHE_SELECTED_ONLY",
-    help="Pre cache database objects relevant to selected resource only.",
+    help="At start of run, populate relational cache only for schemas containing selected nodes, or for all schemas of interest.",
+)
+
+introspect = click.option(
+    "--introspect/--no-introspect",
+    envvar="DBT_INTROSPECT",
+    help="Whether to scaffold introspective queries as part of compilation",
+    default=True,
 )
 
 compile_docs = click.option(
     "--compile/--no-compile",
     envvar=None,
-    help="Wether or not to run 'dbt compile' as part of docs generation",
-    default=True,
-)
-
-compile_parse = click.option(
-    "--compile/--no-compile",
-    envvar=None,
-    help="TODO: No help text currently available",
+    help="Whether or not to run 'dbt compile' as part of docs generation",
     default=True,
 )
 
 config_dir = click.option(
     "--config-dir",
     envvar=None,
-    help="If specified, DBT will show path information for this project",
-    type=click.STRING,
+    help="Show the configured location for the profiles.yml file and exit",
+    is_flag=True,
 )
 
 debug = click.option(
@@ -64,14 +54,19 @@ debug = click.option(
     help="Display debug logging during dbt execution. Useful for debugging and making bug reports.",
 )
 
-# TODO:  The env var and name (reflected in flags) are corrections!
-# The original name was `DEFER_MODE` and used an env var called "DBT_DEFER_TO_STATE"
-# Both of which break existing naming conventions.
-# This will need to be fixed before use in the main codebase and communicated as a change to the community!
+# flag was previously named DEFER_MODE
 defer = click.option(
     "--defer/--no-defer",
     envvar="DBT_DEFER",
-    help="If set, defer to the state variable for resolving unselected nodes.",
+    help="If set, resolve unselected nodes by deferring to the manifest within the --state directory.",
+)
+
+deprecated_defer = click.option(
+    "--deprecated-defer",
+    envvar="DBT_DEFER_TO_STATE",
+    help="Internal flag for deprecating old env var.",
+    default=False,
+    hidden=True,
 )
 
 enable_legacy_logger = click.option(
@@ -80,7 +75,14 @@ enable_legacy_logger = click.option(
     hidden=True,
 )
 
-exclude = click.option("--exclude", envvar=None, help="Specify the nodes to exclude.")
+exclude = click.option(
+    "--exclude",
+    envvar=None,
+    type=tuple,
+    cls=MultiOption,
+    multiple=True,
+    help="Specify the nodes to exclude.",
+)
 
 fail_fast = click.option(
     "--fail-fast/--no-fail-fast",
@@ -89,6 +91,18 @@ fail_fast = click.option(
     help="Stop execution on first failure.",
 )
 
+favor_state = click.option(
+    "--favor-state/--no-favor-state",
+    envvar="DBT_FAVOR_STATE",
+    help="If set, defer to the argument provided to the state flag for resolving unselected nodes, even if the node(s) exist as a database object in the current environment.",
+)
+
+deprecated_favor_state = click.option(
+    "--deprecated-favor-state",
+    envvar="DBT_FAVOR_STATE_MODE",
+    help="Internal flag for deprecating old env var.",
+)
+
 full_refresh = click.option(
     "--full-refresh",
     "-f",
@@ -100,30 +114,69 @@ full_refresh = click.option(
 indirect_selection = click.option(
     "--indirect-selection",
     envvar="DBT_INDIRECT_SELECTION",
-    help="Select all tests that are adjacent to selected resources, even if they those resources have been explicitly selected.",
-    type=click.Choice(["eager", "cautious"], case_sensitive=False),
+    help="Choose which tests to select that are adjacent to selected resources. Eager is most inclusive, cautious is most exclusive, and buildable is in between. Empty includes no tests at all.",
+    type=click.Choice(["eager", "cautious", "buildable", "empty"], case_sensitive=False),
     default="eager",
 )
 
 log_cache_events = click.option(
     "--log-cache-events/--no-log-cache-events",
-    help="Enable verbose adapter cache logging.",
+    help="Enable verbose logging for relational cache events to help when debugging.",
     envvar="DBT_LOG_CACHE_EVENTS",
 )
 
 log_format = click.option(
     "--log-format",
     envvar="DBT_LOG_FORMAT",
-    help="Specify the log format, overriding the command's default.",
-    type=click.Choice(["text", "json", "default"], case_sensitive=False),
+    help="Specify the format of logging to the console and the log file. Use --log-format-file to configure the format for the log file differently than the console.",
+    type=click.Choice(["text", "debug", "json", "default"], case_sensitive=False),
     default="default",
 )
 
+log_format_file = click.option(
+    "--log-format-file",
+    envvar="DBT_LOG_FORMAT_FILE",
+    help="Specify the format of logging to the log file by overriding the default value and the general --log-format setting.",
+    type=click.Choice(["text", "debug", "json", "default"], case_sensitive=False),
+    default="debug",
+)
+
+log_level = click.option(
+    "--log-level",
+    envvar="DBT_LOG_LEVEL",
+    help="Specify the minimum severity of events that are logged to the console and the log file. Use --log-level-file to configure the severity for the log file differently than the console.",
+    type=click.Choice(["debug", "info", "warn", "error", "none"], case_sensitive=False),
+    default="info",
+)
+
+log_level_file = click.option(
+    "--log-level-file",
+    envvar="DBT_LOG_LEVEL_FILE",
+    help="Specify the minimum severity of events that are logged to the log file by overriding the default value and the general --log-level setting.",
+    type=click.Choice(["debug", "info", "warn", "error", "none"], case_sensitive=False),
+    default="debug",
+)
+
+use_colors = click.option(
+    "--use-colors/--no-use-colors",
+    envvar="DBT_USE_COLORS",
+    help="Specify whether log output is colorized in the console and the log file. Use --use-colors-file/--no-use-colors-file to colorize the log file differently than the console.",
+    default=True,
+)
+
+use_colors_file = click.option(
+    "--use-colors-file/--no-use-colors-file",
+    envvar="DBT_USE_COLORS_FILE",
+    help="Specify whether log file output is colorized by overriding the default value and the general --use-colors/--no-use-colors setting.",
+    default=True,
+)
+
 log_path = click.option(
     "--log-path",
     envvar="DBT_LOG_PATH",
     help="Configure the 'log-path'. Only applies this setting for the current run. Overrides the 'DBT_LOG_PATH' if it is set.",
-    type=click.Path(),
+    default=None,
+    type=click.Path(resolve_path=True, path_type=Path),
 )
 
 macro_debugging = click.option(
@@ -132,41 +185,51 @@ macro_debugging = click.option(
     hidden=True,
 )
 
-models = click.option(
-    "-m",
-    "-s",
-    "models",
-    envvar=None,
-    help="Specify the nodes to include.",
-    multiple=True,
-)
-
+# This less standard usage of --output where output_path below is more standard
 output = click.option(
     "--output",
     envvar=None,
-    help="TODO: No current help text",
+    help="Specify the output format: either JSON or a newline-delimited list of selectors, paths, or names",
     type=click.Choice(["json", "name", "path", "selector"], case_sensitive=False),
-    default="name",
+    default="selector",
+)
+
+show_output_format = click.option(
+    "--output",
+    envvar=None,
+    help="Output format for dbt compile and dbt show",
+    type=click.Choice(["json", "text"], case_sensitive=False),
+    default="text",
+)
+
+show_limit = click.option(
+    "--limit",
+    envvar=None,
+    help="Limit the number of results returned by dbt show",
+    type=click.INT,
+    default=5,
 )
 
 output_keys = click.option(
-    "--output-keys", envvar=None, help="TODO: No current help text", type=click.STRING
+    "--output-keys",
+    envvar=None,
+    help=(
+        "Space-delimited listing of node properties to include as custom keys for JSON output "
+        "(e.g. `--output json --output-keys name resource_type description`)"
+    ),
+    type=tuple,
+    cls=MultiOption,
+    multiple=True,
+    default=[],
 )
 
 output_path = click.option(
     "--output",
     "-o",
     envvar=None,
-    help="Specify the output path for the json report. By default, outputs to 'target/sources.json'",
+    help="Specify the output path for the JSON report. By default, outputs to 'target/sources.json'",
     type=click.Path(file_okay=True, dir_okay=False, writable=True),
-    default=PurePath.joinpath(Path.cwd(), "target/sources.json"),
-)
-
-parse_only = click.option(
-    "--parse-only",
-    envvar=None,
-    help="TODO:  No help text currently available",
-    is_flag=True,
+    default=None,
 )
 
 partial_parse = click.option(
@@ -176,6 +239,13 @@ partial_parse = click.option(
     default=True,
 )
 
+populate_cache = click.option(
+    "--populate-cache/--no-populate-cache",
+    envvar="DBT_POPULATE_CACHE",
+    help="At start of run, use `show` or `information_schema` queries to populate a relational cache, which can speed up subsequent materializations.",
+    default=True,
+)
+
 port = click.option(
     "--port",
     envvar=None,
@@ -184,10 +254,6 @@ port = click.option(
     type=click.INT,
 )
 
-# TODO:  The env var and name (reflected in flags) are corrections!
-# The original name was `NO_PRINT` and used the env var `DBT_NO_PRINT`.
-# Both of which break existing naming conventions.
-# This will need to be fixed before use in the main codebase and communicated as a change to the community!
 print = click.option(
     "--print/--no-print",
     envvar="DBT_PRINT",
@@ -195,6 +261,15 @@ print = click.option(
     default=True,
 )
 
+deprecated_print = click.option(
+    "--deprecated-print/--deprecated-no-print",
+    envvar="DBT_NO_PRINT",
+    help="Internal flag for deprecating old env var.",
+    default=True,
+    hidden=True,
+    callback=lambda ctx, param, value: not value,
+)
+
 printer_width = click.option(
     "--printer-width",
     envvar="DBT_PRINTER_WIDTH",
@@ -213,20 +288,30 @@ profiles_dir = click.option(
     "--profiles-dir",
     envvar="DBT_PROFILES_DIR",
     help="Which directory to look in for the profiles.yml file. If not set, dbt will look in the current working directory first, then HOME/.dbt/",
-    default=default_profiles_dir(),
+    default=default_profiles_dir,
     type=click.Path(exists=True),
 )
 
+# `dbt debug` uses this because it implements custom behaviour for non-existent profiles.yml directories
+profiles_dir_exists_false = click.option(
+    "--profiles-dir",
+    envvar="DBT_PROFILES_DIR",
+    help="Which directory to look in for the profiles.yml file. If not set, dbt will look in the current working directory first, then HOME/.dbt/",
+    default=default_profiles_dir,
+    type=click.Path(exists=False),
+)
+
 project_dir = click.option(
     "--project-dir",
-    envvar=None,
+    envvar="DBT_PROJECT_DIR",
     help="Which directory to look in for the dbt_project.yml file. Default is the current working directory and its parents.",
-    default=default_project_dir(),
+    default=default_project_dir,
     type=click.Path(exists=True),
 )
 
 quiet = click.option(
     "--quiet/--no-quiet",
+    "-q",
     envvar="DBT_QUIET",
     help="Suppress all non-error logging to stdout. Does not affect {{ print() }} macro calls.",
 )
@@ -240,10 +325,11 @@ record_timing_info = click.option(
 )
 
 resource_type = click.option(
+    "--resource-types",
     "--resource-type",
     envvar=None,
-    help="TODO: No current help text",
-    type=click.Choice(
+    help="Restricts the types of resources that dbt will include",
+    type=ChoiceTuple(
         [
             "metric",
             "source",
@@ -258,35 +344,121 @@ resource_type = click.option(
         ],
         case_sensitive=False,
     ),
-    default="default",
+    cls=MultiOption,
+    multiple=True,
+    default=(),
 )
 
+model_decls = ("-m", "--models", "--model")
+select_decls = ("-s", "--select")
+select_attrs = {
+    "envvar": None,
+    "help": "Specify the nodes to include.",
+    "cls": MultiOption,
+    "multiple": True,
+    "type": tuple,
+}
+
+inline = click.option(
+    "--inline",
+    envvar=None,
+    help="Pass SQL inline to dbt compile and show",
+)
+
+# `--select` and `--models` are analogous for most commands except `dbt list` for legacy reasons.
+# Most CLI arguments should use the combined `select` option that aliases `--models` to `--select`.
+# However, if you need to split out these separators (like `dbt ls`), use the `models` and `raw_select` options instead.
+# See https://github.com/dbt-labs/dbt-core/pull/6774#issuecomment-1408476095 for more info.
+models = click.option(*model_decls, **select_attrs)
+raw_select = click.option(*select_decls, **select_attrs)
+select = click.option(*select_decls, *model_decls, **select_attrs)
+
 selector = click.option(
-    "--selector", envvar=None, help="The selector name to use, as defined in selectors.yml"
+    "--selector",
+    envvar=None,
+    help="The selector name to use, as defined in selectors.yml",
+)
+
+send_anonymous_usage_stats = click.option(
+    "--send-anonymous-usage-stats/--no-send-anonymous-usage-stats",
+    envvar="DBT_SEND_ANONYMOUS_USAGE_STATS",
+    help="Send anonymous usage stats to dbt Labs.",
+    default=True,
 )
 
 show = click.option(
-    "--show", envvar=None, help="Show a sample of the loaded data in the terminal", is_flag=True
+    "--show",
+    envvar=None,
+    help="Show a sample of the loaded data in the terminal",
+    is_flag=True,
+)
+
+# TODO:  The env var is a correction!
+# The original env var was `DBT_TEST_SINGLE_THREADED`.
+# This broke the existing naming convention.
+# This will need to be communicated as a change to the community!
+#
+# N.B. This flag is only used for testing, hence it's hidden from help text.
+single_threaded = click.option(
+    "--single-threaded/--no-single-threaded",
+    envvar="DBT_SINGLE_THREADED",
+    default=False,
+    hidden=True,
 )
 
 skip_profile_setup = click.option(
-    "--skip-profile-setup", "-s", envvar=None, help="Skip interactive profile setup.", is_flag=True
+    "--skip-profile-setup",
+    "-s",
+    envvar=None,
+    help="Skip interactive profile setup.",
+    is_flag=True,
+)
+
+empty_catalog = click.option(
+    "--empty-catalog",
+    help="If specified, generate empty catalog.json file during the `dbt docs generate` command.",
+    default=False,
+    is_flag=True,
 )
 
-# TODO:  The env var and name (reflected in flags) are corrections!
-# The original name was `ARTIFACT_STATE_PATH` and used the env var `DBT_ARTIFACT_STATE_PATH`.
-# Both of which break existing naming conventions.
-# This will need to be fixed before use in the main codebase and communicated as a change to the community!
 state = click.option(
     "--state",
     envvar="DBT_STATE",
-    help="If set, use the given directory as the source for json files to compare with this project.",
+    help="If set, use the given directory as the source for JSON files to compare with this project.",
     type=click.Path(
         dir_okay=True,
-        exists=True,
         file_okay=False,
         readable=True,
         resolve_path=True,
+        path_type=Path,
+    ),
+)
+
+state_required = click.option(
+    "--state",
+    envvar="DBT_STATE",
+    help="If set, use the given directory as the source for JSON files to compare with this project.",
+    required=True,
+    type=click.Path(
+        dir_okay=True,
+        file_okay=False,
+        readable=True,
+        resolve_path=True,
+        path_type=Path,
+    ),
+)
+
+deprecated_state = click.option(
+    "--deprecated-state",
+    envvar="DBT_ARTIFACT_STATE_PATH",
+    help="Internal flag for deprecating old env var.",
+    hidden=True,
+    type=click.Path(
+        dir_okay=True,
+        file_okay=False,
+        readable=True,
+        resolve_path=True,
+        path_type=Path,
     ),
 )
 
@@ -305,7 +477,10 @@ store_failures = click.option(
 )
 
 target = click.option(
-    "--target", "-t", envvar=None, help="Which target to load for the given profile"
+    "--target",
+    "-t",
+    envvar=None,
+    help="Which target to load for the given profile",
 )
 
 target_path = click.option(
@@ -319,17 +494,10 @@ threads = click.option(
     "--threads",
     envvar=None,
     help="Specify number of threads to use while executing models. Overrides settings in profiles.yml.",
-    default=1,
+    default=None,
     type=click.INT,
 )
 
-use_colors = click.option(
-    "--use-colors/--no-use-colors",
-    envvar="DBT_USE_COLORS",
-    help="Output is colorized by default and may also be set in a profile or at the command line.",
-    default=True,
-)
-
 use_experimental_parser = click.option(
     "--use-experimental-parser/--no-use-experimental-parser",
     envvar="DBT_USE_EXPERIMENTAL_PARSER",
@@ -341,19 +509,35 @@ vars = click.option(
     envvar=None,
     help="Supply variables to the project. This argument overrides variables defined in your dbt_project.yml file. This argument should be a YAML string, eg. '{my_variable: my_value}'",
     type=YAML(),
+    default="{}",
 )
 
+
+# TODO: when legacy flags are deprecated use
+# click.version_option instead of a callback
+def _version_callback(ctx, _param, value):
+    if not value or ctx.resilient_parsing:
+        return
+    click.echo(get_version_information())
+    ctx.exit()
+
+
 version = click.option(
     "--version",
+    "-V",
+    "-v",
+    callback=_version_callback,
     envvar=None,
-    help="Show version information",
+    expose_value=False,
+    help="Show version information and exit",
+    is_eager=True,
     is_flag=True,
 )
 
 version_check = click.option(
     "--version-check/--no-version-check",
     envvar="DBT_VERSION_CHECK",
-    help="Ensure dbt's version matches the one specified in the dbt_project.yml file ('require-dbt-version')",
+    help="If set, ensure the installed dbt version matches the require-dbt-version specified in the dbt_project.yml file (if any). Otherwise, allow them to differ.",
     default=True,
 )
 
@@ -362,13 +546,13 @@ warn_error = click.option(
     envvar="DBT_WARN_ERROR",
     help="If dbt would normally warn, instead raise an exception. Examples include --select that selects nothing, deprecations, configurations with no associated models, invalid test configurations, and missing sources/refs in tests.",
     default=None,
-    flag_value=True,
+    is_flag=True,
 )
 
 warn_error_options = click.option(
     "--warn-error-options",
     envvar="DBT_WARN_ERROR_OPTIONS",
-    default=None,
+    default="{}",
     help="""If dbt would normally warn, instead raise an exception based on include/exclude configuration. Examples include --select that selects nothing, deprecations, configurations with no associated models, invalid test configurations,
     and missing sources/refs in tests. This argument should be a YAML string, with keys 'include' or 'exclude'. eg. '{"include": "all", "exclude": ["NoNodesForSelectionCriteria"]}'""",
     type=WarnErrorOptionsType(),
@@ -377,13 +561,6 @@ warn_error_options = click.option(
 write_json = click.option(
     "--write-json/--no-write-json",
     envvar="DBT_WRITE_JSON",
-    help="Writing the manifest and run_results.json files to disk",
-    default=True,
-)
-
-write_manifest = click.option(
-    "--write-manifest/--no-write-manifest",
-    envvar=None,
-    help="TODO: No help text currently available",
+    help="Whether or not to write the manifest.json and run_results.json files to the target directory",
     default=True,
 )

core/dbt/cli/requires.py

@@ -0,0 +1,252 @@
+import dbt.tracking
+from dbt.version import installed as installed_version
+from dbt.adapters.factory import adapter_management, register_adapter
+from dbt.flags import set_flags, get_flag_dict
+from dbt.cli.exceptions import (
+    ExceptionExit,
+    ResultExit,
+)
+from dbt.cli.flags import Flags
+from dbt.cli.utils import get_profile, get_project
+from dbt.config import RuntimeConfig
+from dbt.config.runtime import load_project, load_profile, UnsetProfile
+from dbt.events.functions import fire_event, LOG_VERSION, set_invocation_id, setup_event_logger
+from dbt.events.types import (
+    CommandCompleted,
+    MainReportVersion,
+    MainReportArgs,
+    MainTrackingUserState,
+)
+from dbt.events.helpers import get_json_string_utcnow
+from dbt.events.types import MainEncounteredError, MainStackTrace
+from dbt.exceptions import Exception as DbtException, DbtProjectError, FailFastError
+from dbt.parser.manifest import ManifestLoader, write_manifest
+from dbt.profiler import profiler
+from dbt.tracking import active_user, initialize_from_flags, track_run
+from dbt.utils import cast_dict_to_dict_of_strings
+
+from click import Context
+from functools import update_wrapper
+import time
+import traceback
+
+
+def preflight(func):
+    def wrapper(*args, **kwargs):
+        ctx = args[0]
+        assert isinstance(ctx, Context)
+        ctx.obj = ctx.obj or {}
+
+        # Flags
+        flags = Flags(ctx)
+        ctx.obj["flags"] = flags
+        set_flags(flags)
+
+        # Logging
+        callbacks = ctx.obj.get("callbacks", [])
+        set_invocation_id()
+        setup_event_logger(flags=flags, callbacks=callbacks)
+
+        # Tracking
+        initialize_from_flags(flags.SEND_ANONYMOUS_USAGE_STATS, flags.PROFILES_DIR)
+        ctx.with_resource(track_run(run_command=flags.WHICH))
+
+        # Now that we have our logger, fire away!
+        fire_event(MainReportVersion(version=str(installed_version), log_version=LOG_VERSION))
+        flags_dict_str = cast_dict_to_dict_of_strings(get_flag_dict())
+        fire_event(MainReportArgs(args=flags_dict_str))
+
+        # Deprecation warnings
+        flags.fire_deprecations()
+
+        if active_user is not None:  # mypy appeasement, always true
+            fire_event(MainTrackingUserState(user_state=active_user.state()))
+
+        # Profiling
+        if flags.RECORD_TIMING_INFO:
+            ctx.with_resource(profiler(enable=True, outfile=flags.RECORD_TIMING_INFO))
+
+        # Adapter management
+        ctx.with_resource(adapter_management())
+
+        return func(*args, **kwargs)
+
+    return update_wrapper(wrapper, func)
+
+
+def postflight(func):
+    """The decorator that handles all exception handling for the click commands.
+    This decorator must be used before any other decorators that may throw an exception."""
+
+    def wrapper(*args, **kwargs):
+        ctx = args[0]
+        start_func = time.perf_counter()
+        success = False
+
+        try:
+            result, success = func(*args, **kwargs)
+        except FailFastError as e:
+            fire_event(MainEncounteredError(exc=str(e)))
+            raise ResultExit(e.result)
+        except DbtException as e:
+            fire_event(MainEncounteredError(exc=str(e)))
+            raise ExceptionExit(e)
+        except BaseException as e:
+            fire_event(MainEncounteredError(exc=str(e)))
+            fire_event(MainStackTrace(stack_trace=traceback.format_exc()))
+            raise ExceptionExit(e)
+        finally:
+            fire_event(
+                CommandCompleted(
+                    command=ctx.command_path,
+                    success=success,
+                    completed_at=get_json_string_utcnow(),
+                    elapsed=time.perf_counter() - start_func,
+                )
+            )
+
+        if not success:
+            raise ResultExit(result)
+
+        return (result, success)
+
+    return update_wrapper(wrapper, func)
+
+
+# TODO: UnsetProfile is necessary for deps and clean to load a project.
+# This decorator and its usage can be removed once https://github.com/dbt-labs/dbt-core/issues/6257 is closed.
+def unset_profile(func):
+    def wrapper(*args, **kwargs):
+        ctx = args[0]
+        assert isinstance(ctx, Context)
+
+        profile = UnsetProfile()
+        ctx.obj["profile"] = profile
+
+        return func(*args, **kwargs)
+
+    return update_wrapper(wrapper, func)
+
+
+def profile(func):
+    def wrapper(*args, **kwargs):
+        ctx = args[0]
+        assert isinstance(ctx, Context)
+
+        ctx.obj["profile"] = get_profile(ctx.obj["flags"])
+
+        return func(*args, **kwargs)
+
+    return update_wrapper(wrapper, func)
+
+
+def project(func):
+    def wrapper(*args, **kwargs):
+        ctx = args[0]
+        assert isinstance(ctx, Context)
+
+        # TODO: Decouple target from profile, and remove the need for profile here:
+        # https://github.com/dbt-labs/dbt-core/issues/6257
+        if not ctx.obj.get("profile"):
+            raise DbtProjectError("profile required for project")
+
+        flags = ctx.obj["flags"]
+        project = get_project(flags, ctx.obj["profile"])
+        ctx.obj["project"] = project
+
+        if dbt.tracking.active_user is not None:
+            project_id = None if project is None else project.hashed_name()
+
+            dbt.tracking.track_project_id({"project_id": project_id})
+
+        return func(*args, **kwargs)
+
+    return update_wrapper(wrapper, func)
+
+
+def runtime_config(func):
+    """A decorator used by click command functions for generating a runtime
+    config given a profile and project.
+    """
+
+    def wrapper(*args, **kwargs):
+        ctx = args[0]
+        assert isinstance(ctx, Context)
+
+        req_strs = ["profile", "project"]
+        reqs = [ctx.obj.get(req_str) for req_str in req_strs]
+
+        if None in reqs:
+            raise DbtProjectError("profile and project required for runtime_config")
+
+        config = RuntimeConfig.from_parts(
+            ctx.obj["project"],
+            ctx.obj["profile"],
+            ctx.obj["flags"],
+        )
+
+        ctx.obj["runtime_config"] = config
+
+        if dbt.tracking.active_user is not None:
+            adapter_type = (
+                getattr(config.credentials, "type", None)
+                if hasattr(config, "credentials")
+                else None
+            )
+            adapter_unique_id = (
+                config.credentials.hashed_unique_field()
+                if hasattr(config, "credentials")
+                else None
+            )
+
+            dbt.tracking.track_adapter_info(
+                {
+                    "adapter_type": adapter_type,
+                    "adapter_unique_id": adapter_unique_id,
+                }
+            )
+
+        return func(*args, **kwargs)
+
+    return update_wrapper(wrapper, func)
+
+
+def manifest(*args0, write=True, write_perf_info=False):
+    """A decorator used by click command functions for generating a manifest
+    given a profile, project, and runtime config. This also registers the adapter
+    from the runtime config and conditionally writes the manifest to disk.
+    """
+
+    def outer_wrapper(func):
+        def wrapper(*args, **kwargs):
+            ctx = args[0]
+            assert isinstance(ctx, Context)
+
+            req_strs = ["profile", "project", "runtime_config"]
+            reqs = [ctx.obj.get(dep) for dep in req_strs]
+
+            if None in reqs:
+                raise DbtProjectError("profile, project, and runtime_config required for manifest")
+
+            runtime_config = ctx.obj["runtime_config"]
+            register_adapter(runtime_config)
+
+            # a manifest has already been set on the context, so don't overwrite it
+            if ctx.obj.get("manifest") is None:
+                manifest = ManifestLoader.get_full_manifest(
+                    runtime_config, write_perf_info=write_perf_info
+                )
+
+                ctx.obj["manifest"] = manifest
+                if write and ctx.obj["flags"].write_json:
+                    write_manifest(manifest, ctx.obj["runtime_config"].target_path)
+
+            return func(*args, **kwargs)
+
+        return update_wrapper(wrapper, func)
+
+    # if there are no args, the decorator was used without params @decorator
+    # otherwise, the decorator was called with params @decorator(arg)
+    if len(args0) == 0:
+        return outer_wrapper
+    return outer_wrapper(args0[0])

core/dbt/cli/resolvers.py

@@ -1,11 +1,31 @@
 from pathlib import Path
+from dbt.config.project import PartialProject
+from dbt.exceptions import DbtProjectError
 
 
-def default_project_dir():
+def default_project_dir() -> Path:
     paths = list(Path.cwd().parents)
     paths.insert(0, Path.cwd())
     return next((x for x in paths if (x / "dbt_project.yml").exists()), Path.cwd())
 
 
-def default_profiles_dir():
+def default_profiles_dir() -> Path:
     return Path.cwd() if (Path.cwd() / "profiles.yml").exists() else Path.home() / ".dbt"
+
+
+def default_log_path(project_dir: Path, verify_version: bool = False) -> Path:
+    """If available, derive a default log path from dbt_project.yml. Otherwise, default to "logs".
+    Known limitations:
+    1. Using PartialProject here, so no jinja rendering of log-path.
+    2. Programmatic invocations of the cli via dbtRunner may pass a Project object directly,
+       which is not being taken into consideration here to extract a log-path.
+    """
+    default_log_path = Path("logs")
+    try:
+        partial = PartialProject.from_project_root(str(project_dir), verify_version=verify_version)
+        partial_log_path = partial.project_dict.get("log-path") or default_log_path
+        default_log_path = Path(project_dir) / partial_log_path
+    except DbtProjectError:
+        pass
+
+    return default_log_path

core/dbt/cli/utils.py

@@ -0,0 +1,29 @@
+from dbt.cli.flags import Flags
+from dbt.config import RuntimeConfig
+from dbt.config.runtime import Profile, Project, load_project, load_profile
+
+
+def get_profile(flags: Flags) -> Profile:
+    # TODO: Generalize safe access to flags.THREADS:
+    # https://github.com/dbt-labs/dbt-core/issues/6259
+    threads = getattr(flags, "THREADS", None)
+    return load_profile(flags.PROJECT_DIR, flags.VARS, flags.PROFILE, flags.TARGET, threads)
+
+
+def get_project(flags: Flags, profile: Profile) -> Project:
+    return load_project(
+        flags.PROJECT_DIR,
+        flags.VERSION_CHECK,
+        profile,
+        flags.VARS,
+    )
+
+
+def get_runtime_config(flags: Flags) -> RuntimeConfig:
+    profile = get_profile(flags)
+    project = get_project(flags, profile)
+    return RuntimeConfig.from_parts(
+        args=flags,
+        profile=profile,
+        project=project,
+    )

core/dbt/clients/jinja.py

@@ -40,7 +40,7 @@ from dbt.exceptions import (
     UndefinedCompilationError,
     UndefinedMacroError,
 )
-from dbt import flags
+from dbt.flags import get_flags
 from dbt.node_types import ModelLanguage
 
 
@@ -99,8 +99,9 @@ class MacroFuzzEnvironment(jinja2.sandbox.SandboxedEnvironment):
         If the value is 'write', also write the files to disk.
         WARNING: This can write a ton of data if you aren't careful.
         """
-        if filename == "<template>" and flags.MACRO_DEBUGGING:
-            write = flags.MACRO_DEBUGGING == "write"
+        macro_debugging = get_flags().MACRO_DEBUGGING
+        if filename == "<template>" and macro_debugging:
+            write = macro_debugging == "write"
             filename = _linecache_inject(source, write)
 
         return super()._compile(source, filename)  # type: ignore
@@ -482,7 +483,7 @@ def get_environment(
     native: bool = False,
 ) -> jinja2.Environment:
     args: Dict[str, List[Union[str, Type[jinja2.ext.Extension]]]] = {
-        "extensions": ["jinja2.ext.do"]
+        "extensions": ["jinja2.ext.do", "jinja2.ext.loopcontrols"]
     }
 
     if capture_macros:

core/dbt/clients/system.py

@@ -1,30 +1,31 @@
 import errno
-import functools
 import fnmatch
+import functools
 import json
 import os
 import os.path
 import re
 import shutil
+import stat
 import subprocess
 import sys
 import tarfile
-import requests
-import stat
-from typing import Type, NoReturn, List, Optional, Dict, Any, Tuple, Callable, Union
-from pathspec import PathSpec  # type: ignore
+from pathlib import Path
+from typing import Any, Callable, Dict, List, NoReturn, Optional, Tuple, Type, Union
 
+import dbt.exceptions
+import requests
 from dbt.events.functions import fire_event
 from dbt.events.types import (
-    SystemErrorRetrievingModTime,
     SystemCouldNotWrite,
     SystemExecutingCmd,
     SystemStdOut,
     SystemStdErr,
     SystemReportReturnCode,
 )
-import dbt.exceptions
+from dbt.exceptions import DbtInternalError
 from dbt.utils import _connection_exception_retry as connection_exception_retry
+from pathspec import PathSpec  # type: ignore
 
 if sys.platform == "win32":
     from ctypes import WinDLL, c_bool
@@ -75,11 +76,7 @@ def find_matching(
                 relative_path = os.path.relpath(absolute_path, absolute_path_to_search)
                 relative_path_to_root = os.path.join(relative_path_to_search, relative_path)
 
-                modification_time = 0.0
-                try:
-                    modification_time = os.path.getmtime(absolute_path)
-                except OSError:
-                    fire_event(SystemErrorRetrievingModTime(path=absolute_path))
+                modification_time = os.path.getmtime(absolute_path)
                 if reobj.match(local_file) and (
                     not ignore_spec or not ignore_spec.match_file(relative_path_to_root)
                 ):
@@ -106,12 +103,18 @@ def load_file_contents(path: str, strip: bool = True) -> str:
     return to_return
 
 
-def make_directory(path: str) -> None:
+@functools.singledispatch
+def make_directory(path=None) -> None:
     """
     Make a directory and any intermediate directories that don't already
     exist. This function handles the case where two threads try to create
     a directory at once.
     """
+    raise DbtInternalError(f"Can not create directory from {type(path)} ")
+
+
+@make_directory.register
+def _(path: str) -> None:
     path = convert_path(path)
     if not os.path.exists(path):
         # concurrent writes that try to create the same dir can fail
@@ -125,6 +128,11 @@ def make_directory(path: str) -> None:
                 raise e
 
 
+@make_directory.register
+def _(path: Path) -> None:
+    path.mkdir(parents=True, exist_ok=True)
+
+
 def make_file(path: str, contents: str = "", overwrite: bool = False) -> bool:
     """
     Make a file at `path` assuming that the directory it resides in already
@@ -441,8 +449,8 @@ def run_cmd(cwd: str, cmd: List[str], env: Optional[Dict[str, Any]] = None) -> T
     except OSError as exc:
         _interpret_oserror(exc, cwd, cmd)
 
-    fire_event(SystemStdOut(bmsg=out))
-    fire_event(SystemStdErr(bmsg=err))
+    fire_event(SystemStdOut(bmsg=str(out)))
+    fire_event(SystemStdErr(bmsg=str(err)))
 
     if proc.returncode != 0:
         fire_event(SystemReportReturnCode(returncode=proc.returncode))

core/dbt/compilation.py

@@ -1,12 +1,15 @@
-import os
-from collections import defaultdict
-from typing import List, Dict, Any, Tuple, Optional
+import argparse
+import json
 
 import networkx as nx  # type: ignore
+import os
 import pickle
 import sqlparse
 
-from dbt import flags
+from collections import defaultdict
+from typing import List, Dict, Any, Tuple, Optional
+
+from dbt.flags import get_flags
 from dbt.adapters.factory import get_adapter
 from dbt.clients import jinja
 from dbt.clients.system import make_directory
@@ -26,12 +29,13 @@ from dbt.exceptions import (
     DbtRuntimeError,
 )
 from dbt.graph import Graph
-from dbt.events.functions import fire_event
-from dbt.events.types import FoundStats, WritingInjectedSQLForNode
+from dbt.events.functions import fire_event, get_invocation_id
+from dbt.events.types import FoundStats, Note, WritingInjectedSQLForNode
 from dbt.events.contextvars import get_node_info
 from dbt.node_types import NodeType, ModelLanguage
 from dbt.events.format import pluralize
 import dbt.tracking
+import dbt.task.list as list_task
 
 graph_file_name = "graph.gpickle"
 
@@ -48,6 +52,7 @@ def print_compile_stats(stats):
         NodeType.Source: "source",
         NodeType.Exposure: "exposure",
         NodeType.Metric: "metric",
+        NodeType.Group: "group",
     }
 
     results = {k: 0 for k in names.keys()}
@@ -85,15 +90,18 @@ def _generate_stats(manifest: Manifest):
         stats[metric.resource_type] += 1
     for macro in manifest.macros.values():
         stats[macro.resource_type] += 1
+    for group in manifest.groups.values():
+        stats[group.resource_type] += 1
     return stats
 
 
 def _add_prepended_cte(prepended_ctes, new_cte):
     for cte in prepended_ctes:
-        if cte.id == new_cte.id:
+        if cte.id == new_cte.id and new_cte.sql:
             cte.sql = new_cte.sql
             return
-    prepended_ctes.append(new_cte)
+    if new_cte.sql:
+        prepended_ctes.append(new_cte)
 
 
 def _extend_prepended_ctes(prepended_ctes, new_prepended_ctes):
@@ -155,6 +163,109 @@ class Linker:
         with open(outfile, "wb") as outfh:
             pickle.dump(out_graph, outfh, protocol=pickle.HIGHEST_PROTOCOL)
 
+    def link_node(self, node: GraphMemberNode, manifest: Manifest):
+        self.add_node(node.unique_id)
+
+        for dependency in node.depends_on_nodes:
+            if dependency in manifest.nodes:
+                self.dependency(node.unique_id, (manifest.nodes[dependency].unique_id))
+            elif dependency in manifest.sources:
+                self.dependency(node.unique_id, (manifest.sources[dependency].unique_id))
+            elif dependency in manifest.metrics:
+                self.dependency(node.unique_id, (manifest.metrics[dependency].unique_id))
+            else:
+                raise GraphDependencyNotFoundError(node, dependency)
+
+    def link_graph(self, manifest: Manifest):
+        for source in manifest.sources.values():
+            self.add_node(source.unique_id)
+        for node in manifest.nodes.values():
+            self.link_node(node, manifest)
+        for exposure in manifest.exposures.values():
+            self.link_node(exposure, manifest)
+        for metric in manifest.metrics.values():
+            self.link_node(metric, manifest)
+
+        cycle = self.find_cycles()
+
+        if cycle:
+            raise RuntimeError("Found a cycle: {}".format(cycle))
+
+    def add_test_edges(self, manifest: Manifest) -> None:
+        """This method adds additional edges to the DAG. For a given non-test
+        executable node, add an edge from an upstream test to the given node if
+        the set of nodes the test depends on is a subset of the upstream nodes
+        for the given node."""
+
+        # Given a graph:
+        # model1 --> model2 --> model3
+        #   |             |
+        #   |            \/
+        #  \/          test 2
+        # test1
+        #
+        # Produce the following graph:
+        # model1 --> model2 --> model3
+        #   |       /\    |      /\ /\
+        #   |       |    \/      |  |
+        #  \/       |  test2 ----|  |
+        # test1 ----|---------------|
+
+        for node_id in self.graph:
+            # If node is executable (in manifest.nodes) and does _not_
+            # represent a test, continue.
+            if (
+                node_id in manifest.nodes
+                and manifest.nodes[node_id].resource_type != NodeType.Test
+            ):
+                # Get *everything* upstream of the node
+                all_upstream_nodes = nx.traversal.bfs_tree(self.graph, node_id, reverse=True)
+                # Get the set of upstream nodes not including the current node.
+                upstream_nodes = set([n for n in all_upstream_nodes if n != node_id])
+
+                # Get all tests that depend on any upstream nodes.
+                upstream_tests = []
+                for upstream_node in upstream_nodes:
+                    upstream_tests += _get_tests_for_node(manifest, upstream_node)
+
+                for upstream_test in upstream_tests:
+                    # Get the set of all nodes that the test depends on
+                    # including the upstream_node itself. This is necessary
+                    # because tests can depend on multiple nodes (ex:
+                    # relationship tests). Test nodes do not distinguish
+                    # between what node the test is "testing" and what
+                    # node(s) it depends on.
+                    test_depends_on = set(manifest.nodes[upstream_test].depends_on_nodes)
+
+                    # If the set of nodes that an upstream test depends on
+                    # is a subset of all upstream nodes of the current node,
+                    # add an edge from the upstream test to the current node.
+                    if test_depends_on.issubset(upstream_nodes):
+                        self.graph.add_edge(upstream_test, node_id, edge_type="parent_test")
+
+    def get_graph(self, manifest: Manifest) -> Graph:
+        self.link_graph(manifest)
+        return Graph(self.graph)
+
+    def get_graph_summary(self, manifest: Manifest) -> Dict[int, Dict[str, Any]]:
+        """Create a smaller summary of the graph, suitable for basic diagnostics
+        and performance tuning. The summary includes only the edge structure,
+        node types, and node names. Each of the n nodes is assigned an integer
+        index 0, 1, 2,..., n-1 for compactness"""
+        graph_nodes = dict()
+        index_dict = dict()
+        for node_index, node_name in enumerate(self.graph):
+            index_dict[node_name] = node_index
+            data = manifest.expect(node_name).to_dict(omit_none=True)
+            graph_nodes[node_index] = {"name": node_name, "type": data["resource_type"]}
+
+        for node_index, node in graph_nodes.items():
+            successors = [index_dict[n] for n in self.graph.successors(node["name"])]
+            if successors:
+                node["succ"] = [index_dict[n] for n in self.graph.successors(node["name"])]
+
+        return graph_nodes
+
 
 class Compiler:
     def __init__(self, config):
@@ -257,16 +368,18 @@ class Compiler:
         inserting CTEs into the SQL.
         """
         if model.compiled_code is None:
-            raise DbtRuntimeError("Cannot inject ctes into an unparsed node", model)
+            raise DbtRuntimeError("Cannot inject ctes into an uncompiled node", model)
+
+        # extra_ctes_injected flag says that we've already recursively injected the ctes
         if model.extra_ctes_injected:
             return (model, model.extra_ctes)
 
         # Just to make it plain that nothing is actually injected for this case
-        if not model.extra_ctes:
+        if len(model.extra_ctes) == 0:
+            # SeedNodes don't have compilation attributes
             if not isinstance(model, SeedNode):
                 model.extra_ctes_injected = True
-            manifest.update_node(model)
-            return (model, model.extra_ctes)
+            return (model, [])
 
         # This stores the ctes which will all be recursively
         # gathered and then "injected" into the model.
@@ -275,7 +388,8 @@ class Compiler:
         # extra_ctes are added to the model by
         # RuntimeRefResolver.create_relation, which adds an
         # extra_cte for every model relation which is an
-        # ephemeral model.
+        # ephemeral model. InjectedCTEs have a unique_id and sql.
+        # extra_ctes start out with sql set to None, and the sql is set in this loop.
         for cte in model.extra_ctes:
             if cte.id not in manifest.nodes:
                 raise DbtInternalError(
@@ -288,23 +402,23 @@ class Compiler:
             if not cte_model.is_ephemeral_model:
                 raise DbtInternalError(f"{cte.id} is not ephemeral")
 
-            # This model has already been compiled, so it's been
-            # through here before
-            if getattr(cte_model, "compiled", False):
+            # This model has already been compiled and extra_ctes_injected, so it's been
+            # through here before. We already checked above for extra_ctes_injected, but
+            # checking again because updates maybe have happened in another thread.
+            if cte_model.compiled is True and cte_model.extra_ctes_injected is True:
                 new_prepended_ctes = cte_model.extra_ctes
 
             # if the cte_model isn't compiled, i.e. first time here
             else:
                 # This is an ephemeral parsed model that we can compile.
-                # Compile and update the node
-                cte_model = self._compile_node(cte_model, manifest, extra_context)
-                # recursively call this method
+                # Render the raw_code and set compiled to True
+                cte_model = self._compile_code(cte_model, manifest, extra_context)
+                # recursively call this method, sets extra_ctes_injected to True
                 cte_model, new_prepended_ctes = self._recursively_prepend_ctes(
                     cte_model, manifest, extra_context
                 )
-                # Save compiled SQL file and sync manifest
+                # Write compiled SQL file
                 self._write_node(cte_model)
-                manifest.sync_update_node(cte_model)
 
             _extend_prepended_ctes(prepended_ctes, new_prepended_ctes)
 
@@ -318,20 +432,21 @@ class Compiler:
             model.compiled_code,
             prepended_ctes,
         )
-        model._pre_injected_sql = model.compiled_code
-        model.compiled_code = injected_sql
-        model.extra_ctes_injected = True
-        model.extra_ctes = prepended_ctes
-        model.validate(model.to_dict(omit_none=True))
-        manifest.update_node(model)
+        # Check again before updating for multi-threading
+        if not model.extra_ctes_injected:
+            model._pre_injected_sql = model.compiled_code
+            model.compiled_code = injected_sql
+            model.extra_ctes = prepended_ctes
+            model.extra_ctes_injected = True
 
-        return model, prepended_ctes
+        # if model.extra_ctes is not set to prepended ctes, something went wrong
+        return model, model.extra_ctes
 
-    # Sets compiled fields in the ManifestSQLNode passed in,
+    # Sets compiled_code and compiled flag in the ManifestSQLNode passed in,
     # creates a "context" dictionary for jinja rendering,
     # and then renders the "compiled_code" using the node, the
     # raw_code and the context.
-    def _compile_node(
+    def _compile_code(
         self,
         node: ManifestSQLNode,
         manifest: Manifest,
@@ -340,24 +455,7 @@ class Compiler:
         if extra_context is None:
             extra_context = {}
 
-        data = node.to_dict(omit_none=True)
-        data.update(
-            {
-                "compiled": False,
-                "compiled_code": None,
-                "extra_ctes_injected": False,
-                "extra_ctes": [],
-            }
-        )
-
         if node.language == ModelLanguage.python:
-            # TODO could we also 'minify' this code at all? just aesthetic, not functional
-
-            # quoating seems like something very specific to sql so far
-            # for all python implementations we are seeing there's no quating.
-            # TODO try to find better way to do this, given that
-            original_quoting = self.config.quoting
-            self.config.quoting = {key: False for key in original_quoting.keys()}
             context = self._create_node_context(node, manifest, extra_context)
 
             postfix = jinja.get_rendered(
@@ -367,8 +465,6 @@ class Compiler:
             )
             # we should NOT jinja render the python model's 'raw code'
             node.compiled_code = f"{node.raw_code}\n\n{postfix}"
-            # restore quoting settings in the end since context is lazy evaluated
-            self.config.quoting = original_quoting
 
         else:
             context = self._create_node_context(node, manifest, extra_context)
@@ -380,112 +476,74 @@ class Compiler:
 
         node.compiled = True
 
+        # relation_name is set at parse time, except for tests without store_failures,
+        # but cli param can turn on store_failures, so we set here.
+        if (
+            node.resource_type == NodeType.Test
+            and node.relation_name is None
+            and node.is_relational
+        ):
+            adapter = get_adapter(self.config)
+            relation_cls = adapter.Relation
+            relation_name = str(relation_cls.create_from(self.config, node))
+            node.relation_name = relation_name
+
         return node
 
-    def write_graph_file(self, linker: Linker, manifest: Manifest):
-        filename = graph_file_name
-        graph_path = os.path.join(self.config.target_path, filename)
-        if flags.WRITE_JSON:
-            linker.write_graph(graph_path, manifest)
-
-    def link_node(self, linker: Linker, node: GraphMemberNode, manifest: Manifest):
-        linker.add_node(node.unique_id)
-
-        for dependency in node.depends_on_nodes:
-            if dependency in manifest.nodes:
-                linker.dependency(node.unique_id, (manifest.nodes[dependency].unique_id))
-            elif dependency in manifest.sources:
-                linker.dependency(node.unique_id, (manifest.sources[dependency].unique_id))
-            elif dependency in manifest.metrics:
-                linker.dependency(node.unique_id, (manifest.metrics[dependency].unique_id))
-            else:
-                raise GraphDependencyNotFoundError(node, dependency)
-
-    def link_graph(self, linker: Linker, manifest: Manifest, add_test_edges: bool = False):
-        for source in manifest.sources.values():
-            linker.add_node(source.unique_id)
-        for node in manifest.nodes.values():
-            self.link_node(linker, node, manifest)
-        for exposure in manifest.exposures.values():
-            self.link_node(linker, exposure, manifest)
-        for metric in manifest.metrics.values():
-            self.link_node(linker, metric, manifest)
-
-        cycle = linker.find_cycles()
-
-        if cycle:
-            raise RuntimeError("Found a cycle: {}".format(cycle))
-
-        if add_test_edges:
-            manifest.build_parent_and_child_maps()
-            self.add_test_edges(linker, manifest)
-
-    def add_test_edges(self, linker: Linker, manifest: Manifest) -> None:
-        """This method adds additional edges to the DAG. For a given non-test
-        executable node, add an edge from an upstream test to the given node if
-        the set of nodes the test depends on is a subset of the upstream nodes
-        for the given node."""
-
-        # Given a graph:
-        # model1 --> model2 --> model3
-        #   |             |
-        #   |            \/
-        #  \/          test 2
-        # test1
-        #
-        # Produce the following graph:
-        # model1 --> model2 --> model3
-        #   |       /\    |      /\ /\
-        #   |       |    \/      |  |
-        #  \/       |  test2 ----|  |
-        # test1 ----|---------------|
-
-        for node_id in linker.graph:
-            # If node is executable (in manifest.nodes) and does _not_
-            # represent a test, continue.
-            if (
-                node_id in manifest.nodes
-                and manifest.nodes[node_id].resource_type != NodeType.Test
-            ):
-                # Get *everything* upstream of the node
-                all_upstream_nodes = nx.traversal.bfs_tree(linker.graph, node_id, reverse=True)
-                # Get the set of upstream nodes not including the current node.
-                upstream_nodes = set([n for n in all_upstream_nodes if n != node_id])
-
-                # Get all tests that depend on any upstream nodes.
-                upstream_tests = []
-                for upstream_node in upstream_nodes:
-                    upstream_tests += _get_tests_for_node(manifest, upstream_node)
-
-                for upstream_test in upstream_tests:
-                    # Get the set of all nodes that the test depends on
-                    # including the upstream_node itself. This is necessary
-                    # because tests can depend on multiple nodes (ex:
-                    # relationship tests). Test nodes do not distinguish
-                    # between what node the test is "testing" and what
-                    # node(s) it depends on.
-                    test_depends_on = set(manifest.nodes[upstream_test].depends_on_nodes)
-
-                    # If the set of nodes that an upstream test depends on
-                    # is a subset of all upstream nodes of the current node,
-                    # add an edge from the upstream test to the current node.
-                    if test_depends_on.issubset(upstream_nodes):
-                        linker.graph.add_edge(upstream_test, node_id)
-
+    # This method doesn't actually "compile" any of the nodes. That is done by the
+    # "compile_node" method. This creates a Linker and builds the networkx graph,
+    # writes out the graph.gpickle file, and prints the stats, returning a Graph object.
     def compile(self, manifest: Manifest, write=True, add_test_edges=False) -> Graph:
         self.initialize()
         linker = Linker()
+        linker.link_graph(manifest)
 
-        self.link_graph(linker, manifest, add_test_edges)
+        # Create a file containing basic information about graph structure,
+        # supporting diagnostics and performance analysis.
+        summaries: Dict = dict()
+        summaries["_invocation_id"] = get_invocation_id()
+        summaries["linked"] = linker.get_graph_summary(manifest)
+
+        if add_test_edges:
+            manifest.build_parent_and_child_maps()
+            linker.add_test_edges(manifest)
+
+            # Create another diagnostic summary, just as above, but this time
+            # including the test edges.
+            summaries["with_test_edges"] = linker.get_graph_summary(manifest)
+
+        with open(os.path.join(self.config.target_path, "graph_summary.json"), "w") as out_stream:
+            try:
+                out_stream.write(json.dumps(summaries))
+            except Exception as e:  # This is non-essential information, so merely note failures.
+                fire_event(
+                    Note(
+                        msg=f"An error was encountered writing the graph summary information: {e}"
+                    )
+                )
 
         stats = _generate_stats(manifest)
 
         if write:
             self.write_graph_file(linker, manifest)
-        print_compile_stats(stats)
+
+        # Do not print these for ListTask's
+        if not (
+            self.config.args.__class__ == argparse.Namespace
+            and self.config.args.cls == list_task.ListTask
+        ):
+            stats = _generate_stats(manifest)
+            print_compile_stats(stats)
 
         return Graph(linker.graph)
 
+    def write_graph_file(self, linker: Linker, manifest: Manifest):
+        filename = graph_file_name
+        graph_path = os.path.join(self.config.target_path, filename)
+        flags = get_flags()
+        if flags.WRITE_JSON:
+            linker.write_graph(graph_path, manifest)
+
     # writes the "compiled_code" into the target/compiled directory
     def _write_node(self, node: ManifestSQLNode) -> ManifestSQLNode:
         if not node.extra_ctes_injected or node.resource_type in (
@@ -510,11 +568,11 @@ class Compiler:
     ) -> ManifestSQLNode:
         """This is the main entry point into this code. It's called by
         CompileRunner.compile, GenericRPCRunner.compile, and
-        RunTask.get_hook_sql. It calls '_compile_node' to convert
-        the node into a compiled node, and then calls the
+        RunTask.get_hook_sql. It calls '_compile_code' to render
+        the node's raw_code into compiled_code, and then calls the
         recursive method to "prepend" the ctes.
         """
-        node = self._compile_node(node, manifest, extra_context)
+        node = self._compile_code(node, manifest, extra_context)
 
         node, _ = self._recursively_prepend_ctes(node, manifest, extra_context)
         if write:

core/dbt/config/__init__.py

@@ -1,4 +1,4 @@
 # all these are just exports, they need "noqa" so flake8 will not complain.
 from .profile import Profile, read_user_config  # noqa
-from .project import Project, IsFQNResource  # noqa
-from .runtime import RuntimeConfig, UnsetProfileConfig  # noqa
+from .project import Project, IsFQNResource, PartialProject  # noqa
+from .runtime import RuntimeConfig  # noqa

core/dbt/config/profile.py

@@ -4,7 +4,7 @@ import os
 
 from dbt.dataclass_schema import ValidationError
 
-from dbt import flags
+from dbt.flags import get_flags
 from dbt.clients.system import load_file_contents
 from dbt.clients.yaml_helper import load_yaml_text
 from dbt.contracts.connection import Credentials, HasCredentials
@@ -32,22 +32,6 @@ dbt encountered an error while trying to read your profiles.yml file.
 """
 
 
-NO_SUPPLIED_PROFILE_ERROR = """\
-dbt cannot run because no profile was specified for this dbt project.
-To specify a profile for this project, add a line like the this to
-your dbt_project.yml file:
-
-profile: [profile name]
-
-Here, [profile name] should be replaced with a profile name
-defined in your profiles.yml file. You can find profiles.yml here:
-
-{profiles_file}/profiles.yml
-""".format(
-    profiles_file=flags.DEFAULT_PROFILES_DIR
-)
-
-
 def read_profile(profiles_dir: str) -> Dict[str, Any]:
     path = os.path.join(profiles_dir, "profiles.yml")
 
@@ -197,10 +181,33 @@ class Profile(HasCredentials):
         args_profile_name: Optional[str],
         project_profile_name: Optional[str] = None,
     ) -> str:
+        # TODO: Duplicating this method as direct copy of the implementation in dbt.cli.resolvers
+        # dbt.cli.resolvers implementation can't be used because it causes a circular dependency.
+        # This should be removed and use a safe default access on the Flags module when
+        # https://github.com/dbt-labs/dbt-core/issues/6259 is closed.
+        def default_profiles_dir():
+            from pathlib import Path
+
+            return Path.cwd() if (Path.cwd() / "profiles.yml").exists() else Path.home() / ".dbt"
+
         profile_name = project_profile_name
         if args_profile_name is not None:
             profile_name = args_profile_name
         if profile_name is None:
+            NO_SUPPLIED_PROFILE_ERROR = """\
+dbt cannot run because no profile was specified for this dbt project.
+To specify a profile for this project, add a line like the this to
+your dbt_project.yml file:
+
+profile: [profile name]
+
+Here, [profile name] should be replaced with a profile name
+defined in your profiles.yml file. You can find profiles.yml here:
+
+{profiles_file}/profiles.yml
+""".format(
+                profiles_file=default_profiles_dir()
+            )
             raise DbtProjectError(NO_SUPPLIED_PROFILE_ERROR)
         return profile_name
 
@@ -401,11 +408,13 @@ class Profile(HasCredentials):
         )
 
     @classmethod
-    def render_from_args(
+    def render(
         cls,
-        args: Any,
         renderer: ProfileRenderer,
         project_profile_name: Optional[str],
+        profile_name_override: Optional[str] = None,
+        target_override: Optional[str] = None,
+        threads_override: Optional[int] = None,
     ) -> "Profile":
         """Given the raw profiles as read from disk and the name of the desired
         profile if specified, return the profile component of the runtime
@@ -421,10 +430,9 @@ class Profile(HasCredentials):
             target could not be found.
         :returns Profile: The new Profile object.
         """
-        threads_override = getattr(args, "threads", None)
-        target_override = getattr(args, "target", None)
+        flags = get_flags()
         raw_profiles = read_profile(flags.PROFILES_DIR)
-        profile_name = cls.pick_profile_name(getattr(args, "profile", None), project_profile_name)
+        profile_name = cls.pick_profile_name(profile_name_override, project_profile_name)
         return cls.from_raw_profiles(
             raw_profiles=raw_profiles,
             profile_name=profile_name,

core/dbt/config/project.py

@@ -12,10 +12,10 @@ from typing import (
 )
 from typing_extensions import Protocol, runtime_checkable
 
-import hashlib
 import os
 
-from dbt import flags, deprecations
+from dbt.flags import get_flags
+from dbt import deprecations
 from dbt.clients.system import path_exists, resolve_path_from_base, load_file_contents
 from dbt.clients.yaml_helper import load_yaml_text
 from dbt.contracts.connection import QueryComment
@@ -30,16 +30,16 @@ from dbt.graph import SelectionSpec
 from dbt.helper_types import NoValue
 from dbt.semver import VersionSpecifier, versions_compatible
 from dbt.version import get_installed_version
-from dbt.utils import MultiDict
+from dbt.utils import MultiDict, md5
 from dbt.node_types import NodeType
 from dbt.config.selectors import SelectorDict
 from dbt.contracts.project import (
     Project as ProjectContract,
     SemverString,
 )
-from dbt.contracts.project import PackageConfig
+from dbt.contracts.project import PackageConfig, ProjectPackageMetadata
 from dbt.dataclass_schema import ValidationError
-from .renderer import DbtProjectYamlRenderer
+from .renderer import DbtProjectYamlRenderer, PackageRenderer
 from .selectors import (
     selector_config_from_data,
     selector_data_from_root,
@@ -75,6 +75,11 @@ Validator Error:
 {error}
 """
 
+MISSING_DBT_PROJECT_ERROR = """\
+No dbt_project.yml found at expected path {path}
+Verify that each entry within packages.yml (and their transitive dependencies) contains a file named dbt_project.yml
+"""
+
 
 @runtime_checkable
 class IsFQNResource(Protocol):
@@ -132,11 +137,10 @@ def _all_source_paths(
     analysis_paths: List[str],
     macro_paths: List[str],
 ) -> List[str]:
-    # We need to turn a list of lists into just a list, then convert to a set to
-    # get only unique elements, then back to a list
-    return list(
-        set(list(chain(model_paths, seed_paths, snapshot_paths, analysis_paths, macro_paths)))
-    )
+    paths = chain(model_paths, seed_paths, snapshot_paths, analysis_paths, macro_paths)
+    # Strip trailing slashes since the path is the same even though the name is not
+    stripped_paths = map(lambda s: s.rstrip("/"), paths)
+    return list(set(stripped_paths))
 
 
 T = TypeVar("T")
@@ -156,16 +160,14 @@ def value_or(value: Optional[T], default: T) -> T:
         return value
 
 
-def _raw_project_from(project_root: str) -> Dict[str, Any]:
+def load_raw_project(project_root: str) -> Dict[str, Any]:
 
     project_root = os.path.normpath(project_root)
     project_yaml_filepath = os.path.join(project_root, "dbt_project.yml")
 
     # get the project.yml contents
     if not path_exists(project_yaml_filepath):
-        raise DbtProjectError(
-            "no dbt_project.yml found at expected path {}".format(project_yaml_filepath)
-        )
+        raise DbtProjectError(MISSING_DBT_PROJECT_ERROR.format(path=project_yaml_filepath))
 
     project_dict = _load_yaml(project_yaml_filepath)
 
@@ -289,23 +291,34 @@ class PartialProject(RenderComponents):
                 exc.path = os.path.join(self.project_root, "dbt_project.yml")
             raise
 
-    def check_config_path(self, project_dict, deprecated_path, exp_path):
+    def render_package_metadata(self, renderer: PackageRenderer) -> ProjectPackageMetadata:
+        packages_data = renderer.render_data(self.packages_dict)
+        packages_config = package_config_from_data(packages_data)
+        if not self.project_name:
+            raise DbtProjectError("Package dbt_project.yml must have a name!")
+        return ProjectPackageMetadata(self.project_name, packages_config.packages)
+
+    def check_config_path(
+        self, project_dict, deprecated_path, expected_path=None, default_value=None
+    ):
         if deprecated_path in project_dict:
-            if exp_path in project_dict:
+            if expected_path in project_dict:
                 msg = (
-                    "{deprecated_path} and {exp_path} cannot both be defined. The "
-                    "`{deprecated_path}` config has been deprecated in favor of `{exp_path}`. "
+                    "{deprecated_path} and {expected_path} cannot both be defined. The "
+                    "`{deprecated_path}` config has been deprecated in favor of `{expected_path}`. "
                     "Please update your `dbt_project.yml` configuration to reflect this "
                     "change."
                 )
                 raise DbtProjectError(
-                    msg.format(deprecated_path=deprecated_path, exp_path=exp_path)
+                    msg.format(deprecated_path=deprecated_path, expected_path=expected_path)
+                )
+            # this field is no longer supported, but many projects may specify it with the default value
+            # if so, let's only raise this deprecation warning if they set a custom value
+            if not default_value or project_dict[deprecated_path] != default_value:
+                deprecations.warn(
+                    f"project-config-{deprecated_path}",
+                    deprecated_path=deprecated_path,
                 )
-            deprecations.warn(
-                f"project-config-{deprecated_path}",
-                deprecated_path=deprecated_path,
-                exp_path=exp_path,
-            )
 
     def create_project(self, rendered: RenderComponents) -> "Project":
         unrendered = RenderComponents(
@@ -320,6 +333,8 @@ class PartialProject(RenderComponents):
 
         self.check_config_path(rendered.project_dict, "source-paths", "model-paths")
         self.check_config_path(rendered.project_dict, "data-paths", "seed-paths")
+        self.check_config_path(rendered.project_dict, "log-path", default_value="logs")
+        self.check_config_path(rendered.project_dict, "target-path", default_value="target")
 
         try:
             ProjectContract.validate(rendered.project_dict)
@@ -363,9 +378,13 @@ class PartialProject(RenderComponents):
 
         docs_paths: List[str] = value_or(cfg.docs_paths, all_source_paths)
         asset_paths: List[str] = value_or(cfg.asset_paths, [])
-        target_path: str = flag_or(flags.TARGET_PATH, cfg.target_path, "target")
+        flags = get_flags()
+
+        flag_target_path = str(flags.TARGET_PATH) if flags.TARGET_PATH else None
+        target_path: str = flag_or(flag_target_path, cfg.target_path, "target")
+        log_path: str = str(flags.LOG_PATH)
+
         clean_targets: List[str] = value_or(cfg.clean_targets, [target_path])
-        log_path: str = flag_or(flags.LOG_PATH, cfg.log_path, "logs")
         packages_install_path: str = value_or(cfg.packages_install_path, "dbt_packages")
         # in the default case we'll populate this once we know the adapter type
         # It would be nice to just pass along a Quoting here, but that would
@@ -485,14 +504,7 @@ class PartialProject(RenderComponents):
         cls, project_root: str, *, verify_version: bool = False
     ) -> "PartialProject":
         project_root = os.path.normpath(project_root)
-        project_dict = _raw_project_from(project_root)
-        config_version = project_dict.get("config-version", 1)
-        if config_version != 2:
-            raise DbtProjectError(
-                f"Invalid config version: {config_version}, expected 2",
-                path=os.path.join(project_root, "dbt_project.yml"),
-            )
-
+        project_dict = load_raw_project(project_root)
         packages_dict = package_data_from_root(project_root)
         selectors_dict = selector_data_from_root(project_root)
         return cls.from_dicts(
@@ -525,7 +537,7 @@ class VarProvider:
 @dataclass
 class Project:
     project_name: str
-    version: Union[SemverString, float]
+    version: Optional[Union[SemverString, float]]
     project_root: str
     profile_name: Optional[str]
     model_paths: List[str]
@@ -659,11 +671,11 @@ class Project:
         *,
         verify_version: bool = False,
     ) -> "Project":
-        partial = cls.partial_load(project_root, verify_version=verify_version)
+        partial = PartialProject.from_project_root(project_root, verify_version=verify_version)
         return partial.render(renderer)
 
     def hashed_name(self):
-        return hashlib.md5(self.project_name.encode("utf-8")).hexdigest()
+        return md5(self.project_name)
 
     def get_selector(self, name: str) -> Union[SelectionSpec, bool]:
         if name not in self.selectors:

core/dbt/config/renderer.py

@@ -107,7 +107,7 @@ class DbtProjectYamlRenderer(BaseRenderer):
         if cli_vars is None:
             cli_vars = {}
         if profile:
-            self.ctx_obj = TargetContext(profile, cli_vars)
+            self.ctx_obj = TargetContext(profile.to_target_dict(), cli_vars)
         else:
             self.ctx_obj = BaseContext(cli_vars)  # type:ignore
         context = self.ctx_obj.to_dict()

core/dbt/config/runtime.py

@@ -1,7 +1,7 @@
 import itertools
 import os
 from copy import deepcopy
-from dataclasses import dataclass, field
+from dataclasses import dataclass
 from pathlib import Path
 from typing import (
     Any,
@@ -13,17 +13,18 @@ from typing import (
     Optional,
     Tuple,
     Type,
-    Union,
 )
 
-from dbt import flags
+from dbt.flags import get_flags
 from dbt.adapters.factory import get_include_paths, get_relation_class_by_name
-from dbt.config.profile import read_user_config
-from dbt.contracts.connection import AdapterRequiredConfig, Credentials
+from dbt.config.project import load_raw_project
+from dbt.contracts.connection import AdapterRequiredConfig, Credentials, HasCredentials
 from dbt.contracts.graph.manifest import ManifestMetadata
 from dbt.contracts.project import Configuration, UserConfig
 from dbt.contracts.relation import ComponentName
 from dbt.dataclass_schema import ValidationError
+from dbt.events.functions import warn_or_error
+from dbt.events.types import UnusedResourceConfigPath
 from dbt.exceptions import (
     ConfigContractBrokenError,
     DbtProjectError,
@@ -31,14 +32,46 @@ from dbt.exceptions import (
     DbtRuntimeError,
     UninstalledPackagesFoundError,
 )
-from dbt.events.functions import warn_or_error
-from dbt.events.types import UnusedResourceConfigPath
 from dbt.helper_types import DictDefaultEmptyStr, FQNPath, PathSet
-
 from .profile import Profile
-from .project import Project, PartialProject
+from .project import Project
 from .renderer import DbtProjectYamlRenderer, ProfileRenderer
-from .utils import parse_cli_vars
+
+
+def load_project(
+    project_root: str,
+    version_check: bool,
+    profile: HasCredentials,
+    cli_vars: Optional[Dict[str, Any]] = None,
+) -> Project:
+    # get the project with all of the provided information
+    project_renderer = DbtProjectYamlRenderer(profile, cli_vars)
+    project = Project.from_project_root(
+        project_root, project_renderer, verify_version=version_check
+    )
+
+    # Save env_vars encountered in rendering for partial parsing
+    project.project_env_vars = project_renderer.ctx_obj.env_vars
+    return project
+
+
+def load_profile(
+    project_root: str,
+    cli_vars: Dict[str, Any],
+    profile_name_override: Optional[str] = None,
+    target_override: Optional[str] = None,
+    threads_override: Optional[int] = None,
+) -> Profile:
+    raw_project = load_raw_project(project_root)
+    raw_profile_name = raw_project.get("profile")
+    profile_renderer = ProfileRenderer(cli_vars)
+    profile_name = profile_renderer.render_value(raw_profile_name)
+    profile = Profile.render(
+        profile_renderer, profile_name, profile_name_override, target_override, threads_override
+    )
+    # Save env_vars encountered in rendering for partial parsing
+    profile.profile_env_vars = profile_renderer.ctx_obj.env_vars
+    return profile
 
 
 def _project_quoting_dict(proj: Project, profile: Profile) -> Dict[ComponentName, bool]:
@@ -62,6 +95,21 @@ class RuntimeConfig(Project, Profile, AdapterRequiredConfig):
     def __post_init__(self):
         self.validate()
 
+    @classmethod
+    def get_profile(
+        cls,
+        project_root: str,
+        cli_vars: Dict[str, Any],
+        args: Any,
+    ) -> Profile:
+        return load_profile(
+            project_root,
+            cli_vars,
+            args.profile,
+            args.target,
+            args.threads,
+        )
+
     # Called by 'new_project' and 'from_args'
     @classmethod
     def from_parts(
@@ -84,7 +132,7 @@ class RuntimeConfig(Project, Profile, AdapterRequiredConfig):
             .replace_dict(_project_quoting_dict(project, profile))
         ).to_dict(omit_none=True)
 
-        cli_vars: Dict[str, Any] = parse_cli_vars(getattr(args, "vars", "{}"))
+        cli_vars: Dict[str, Any] = getattr(args, "vars", {})
 
         return cls(
             project_name=project.project_name,
@@ -149,11 +197,10 @@ class RuntimeConfig(Project, Profile, AdapterRequiredConfig):
 
         # load the new project and its packages. Don't pass cli variables.
         renderer = DbtProjectYamlRenderer(profile)
-
         project = Project.from_project_root(
             project_root,
             renderer,
-            verify_version=bool(flags.VERSION_CHECK),
+            verify_version=bool(getattr(self.args, "VERSION_CHECK", True)),
         )
 
         runtime_config = self.from_parts(
@@ -189,64 +236,19 @@ class RuntimeConfig(Project, Profile, AdapterRequiredConfig):
         except ValidationError as e:
             raise ConfigContractBrokenError(e) from e
 
-    @classmethod
-    def _get_rendered_profile(
-        cls,
-        args: Any,
-        profile_renderer: ProfileRenderer,
-        profile_name: Optional[str],
-    ) -> Profile:
-
-        return Profile.render_from_args(args, profile_renderer, profile_name)
-
     @classmethod
     def collect_parts(cls: Type["RuntimeConfig"], args: Any) -> Tuple[Project, Profile]:
-
-        cli_vars: Dict[str, Any] = parse_cli_vars(getattr(args, "vars", "{}"))
-
-        profile = cls.collect_profile(args=args)
-        project_renderer = DbtProjectYamlRenderer(profile, cli_vars)
-        project = cls.collect_project(args=args, project_renderer=project_renderer)
-        assert type(project) is Project
-        return (project, profile)
-
-    @classmethod
-    def collect_profile(
-        cls: Type["RuntimeConfig"], args: Any, profile_name: Optional[str] = None
-    ) -> Profile:
-
-        cli_vars: Dict[str, Any] = parse_cli_vars(getattr(args, "vars", "{}"))
-        profile_renderer = ProfileRenderer(cli_vars)
-
-        # build the profile using the base renderer and the one fact we know
-        if profile_name is None:
-            # Note: only the named profile section is rendered here. The rest of the
-            # profile is ignored.
-            partial = cls.collect_project(args)
-            assert type(partial) is PartialProject
-            profile_name = partial.render_profile_name(profile_renderer)
-
-        profile = cls._get_rendered_profile(args, profile_renderer, profile_name)
-        # Save env_vars encountered in rendering for partial parsing
-        profile.profile_env_vars = profile_renderer.ctx_obj.env_vars
-        return profile
-
-    @classmethod
-    def collect_project(
-        cls: Type["RuntimeConfig"],
-        args: Any,
-        project_renderer: Optional[DbtProjectYamlRenderer] = None,
-    ) -> Union[Project, PartialProject]:
-
+        # profile_name from the project
         project_root = args.project_dir if args.project_dir else os.getcwd()
-        version_check = bool(flags.VERSION_CHECK)
-        partial = Project.partial_load(project_root, verify_version=version_check)
-        if project_renderer is None:
-            return partial
-        else:
-            project = partial.render(project_renderer)
-            project.project_env_vars = project_renderer.ctx_obj.env_vars
-            return project
+        cli_vars: Dict[str, Any] = getattr(args, "vars", {})
+        profile = cls.get_profile(
+            project_root,
+            cli_vars,
+            args,
+        )
+        flags = get_flags()
+        project = load_project(project_root, bool(flags.VERSION_CHECK), profile, cli_vars)
+        return project, profile
 
     # Called in main.py, lib.py, task/base.py
     @classmethod
@@ -411,8 +413,8 @@ class UnsetCredentials(Credentials):
         return ()
 
 
-# This is used by UnsetProfileConfig, for commands which do
-# not require a profile, i.e. dbt deps and clean
+# This is used by commands which do not require
+# a profile, i.e. dbt deps and clean
 class UnsetProfile(Profile):
     def __init__(self):
         self.credentials = UnsetCredentials()
@@ -431,182 +433,12 @@ class UnsetProfile(Profile):
         return Profile.__getattribute__(self, name)
 
 
-# This class is used by the dbt deps and clean commands, because they don't
-# require a functioning profile.
-@dataclass
-class UnsetProfileConfig(RuntimeConfig):
-    """This class acts a lot _like_ a RuntimeConfig, except if your profile is
-    missing, any access to profile members results in an exception.
-    """
-
-    profile_name: str = field(repr=False)
-    target_name: str = field(repr=False)
-
-    def __post_init__(self):
-        # instead of futzing with InitVar overrides or rewriting __init__, just
-        # `del` the attrs we don't want  users touching.
-        del self.profile_name
-        del self.target_name
-        # don't call super().__post_init__(), as that calls validate(), and
-        # this object isn't very valid
-
-    def __getattribute__(self, name):
-        # Override __getattribute__ to check that the attribute isn't 'banned'.
-        if name in {"profile_name", "target_name"}:
-            raise DbtRuntimeError(f'Error: disallowed attribute "{name}" - no profile!')
-
-        # avoid every attribute access triggering infinite recursion
-        return RuntimeConfig.__getattribute__(self, name)
-
-    def to_target_dict(self):
-        # re-override the poisoned profile behavior
-        return DictDefaultEmptyStr({})
-
-    def to_project_config(self, with_packages=False):
-        """Return a dict representation of the config that could be written to
-        disk with `yaml.safe_dump` to get this configuration.
-
-        Overrides dbt.config.Project.to_project_config to omit undefined profile
-        attributes.
-
-        :param with_packages bool: If True, include the serialized packages
-            file in the root.
-        :returns dict: The serialized profile.
-        """
-        result = deepcopy(
-            {
-                "name": self.project_name,
-                "version": self.version,
-                "project-root": self.project_root,
-                "profile": "",
-                "model-paths": self.model_paths,
-                "macro-paths": self.macro_paths,
-                "seed-paths": self.seed_paths,
-                "test-paths": self.test_paths,
-                "analysis-paths": self.analysis_paths,
-                "docs-paths": self.docs_paths,
-                "asset-paths": self.asset_paths,
-                "target-path": self.target_path,
-                "snapshot-paths": self.snapshot_paths,
-                "clean-targets": self.clean_targets,
-                "log-path": self.log_path,
-                "quoting": self.quoting,
-                "models": self.models,
-                "on-run-start": self.on_run_start,
-                "on-run-end": self.on_run_end,
-                "dispatch": self.dispatch,
-                "seeds": self.seeds,
-                "snapshots": self.snapshots,
-                "sources": self.sources,
-                "tests": self.tests,
-                "metrics": self.metrics,
-                "exposures": self.exposures,
-                "vars": self.vars.to_dict(),
-                "require-dbt-version": [v.to_version_string() for v in self.dbt_version],
-                "config-version": self.config_version,
-            }
-        )
-        if self.query_comment:
-            result["query-comment"] = self.query_comment.to_dict(omit_none=True)
-
-        if with_packages:
-            result.update(self.packages.to_dict(omit_none=True))
-
-        return result
-
-    @classmethod
-    def from_parts(
-        cls,
-        project: Project,
-        profile: Profile,
-        args: Any,
-        dependencies: Optional[Mapping[str, "RuntimeConfig"]] = None,
-    ) -> "RuntimeConfig":
-        """Instantiate a RuntimeConfig from its components.
-
-        :param profile: Ignored.
-        :param project: A parsed dbt Project.
-        :param args: The parsed command-line arguments.
-        :returns RuntimeConfig: The new configuration.
-        """
-        cli_vars: Dict[str, Any] = parse_cli_vars(getattr(args, "vars", "{}"))
-
-        return cls(
-            project_name=project.project_name,
-            version=project.version,
-            project_root=project.project_root,
-            model_paths=project.model_paths,
-            macro_paths=project.macro_paths,
-            seed_paths=project.seed_paths,
-            test_paths=project.test_paths,
-            analysis_paths=project.analysis_paths,
-            docs_paths=project.docs_paths,
-            asset_paths=project.asset_paths,
-            target_path=project.target_path,
-            snapshot_paths=project.snapshot_paths,
-            clean_targets=project.clean_targets,
-            log_path=project.log_path,
-            packages_install_path=project.packages_install_path,
-            quoting=project.quoting,  # we never use this anyway.
-            models=project.models,
-            on_run_start=project.on_run_start,
-            on_run_end=project.on_run_end,
-            dispatch=project.dispatch,
-            seeds=project.seeds,
-            snapshots=project.snapshots,
-            dbt_version=project.dbt_version,
-            packages=project.packages,
-            manifest_selectors=project.manifest_selectors,
-            selectors=project.selectors,
-            query_comment=project.query_comment,
-            sources=project.sources,
-            tests=project.tests,
-            metrics=project.metrics,
-            exposures=project.exposures,
-            vars=project.vars,
-            config_version=project.config_version,
-            unrendered=project.unrendered,
-            project_env_vars=project.project_env_vars,
-            profile_env_vars=profile.profile_env_vars,
-            profile_name="",
-            target_name="",
-            user_config=UserConfig(),
-            threads=getattr(args, "threads", 1),
-            credentials=UnsetCredentials(),
-            args=args,
-            cli_vars=cli_vars,
-            dependencies=dependencies,
-        )
-
-    @classmethod
-    def _get_rendered_profile(
-        cls,
-        args: Any,
-        profile_renderer: ProfileRenderer,
-        profile_name: Optional[str],
-    ) -> Profile:
-
-        profile = UnsetProfile()
-        # The profile (for warehouse connection) is not needed, but we want
-        # to get the UserConfig, which is also in profiles.yml
-        user_config = read_user_config(flags.PROFILES_DIR)
-        profile.user_config = user_config
-        return profile
-
-    @classmethod
-    def from_args(cls: Type[RuntimeConfig], args: Any) -> "RuntimeConfig":
-        """Given arguments, read in dbt_project.yml from the current directory,
-        read in packages.yml if it exists, and use them to find the profile to
-        load.
-
-        :param args: The arguments as parsed from the cli.
-        :raises DbtProjectError: If the project is invalid or missing.
-        :raises DbtProfileError: If the profile is invalid or missing.
-        :raises DbtValidationError: If the cli variables are invalid.
-        """
-        project, profile = cls.collect_parts(args)
-
-        return cls.from_parts(project=project, profile=profile, args=args)
+UNUSED_RESOURCE_CONFIGURATION_PATH_MESSAGE = """\
+Configuration paths exist in your dbt_project.yml file which do not \
+apply to any resources.
+There are {} unused configuration paths:
+{}
+"""
 
 
 def _is_config_used(path, fqns):

core/dbt/config/utils.py

@@ -1,12 +1,7 @@
-from argparse import Namespace
-from typing import Any, Dict, Optional, Union
-from xmlrpc.client import Boolean
-from dbt.contracts.project import UserConfig
+from typing import Any, Dict
+
 
-import dbt.flags as flags
 from dbt.clients import yaml_helper
-from dbt.config import Profile, Project, read_user_config
-from dbt.config.renderer import DbtProjectYamlRenderer, ProfileRenderer
 from dbt.events.functions import fire_event
 from dbt.events.types import InvalidOptionYAML
 from dbt.exceptions import DbtValidationError, OptionNotYamlDictError
@@ -27,49 +22,3 @@ def parse_cli_yaml_string(var_string: str, cli_option_name: str) -> Dict[str, An
     except DbtValidationError:
         fire_event(InvalidOptionYAML(option_name=cli_option_name))
         raise
-
-
-def get_project_config(
-    project_path: str,
-    profile_name: str,
-    args: Namespace = Namespace(),
-    cli_vars: Optional[Dict[str, Any]] = None,
-    profile: Optional[Profile] = None,
-    user_config: Optional[UserConfig] = None,
-    return_dict: Boolean = True,
-) -> Union[Project, Dict]:
-    """Returns a project config (dict or object) from a given project path and profile name.
-
-    Args:
-        project_path: Path to project
-        profile_name: Name of profile
-        args: An argparse.Namespace that represents what would have been passed in on the
-            command line (optional)
-        cli_vars: A dict of any vars that would have been passed in on the command line (optional)
-            (see parse_cli_vars above for formatting details)
-        profile: A dbt.config.profile.Profile object (optional)
-        user_config: A dbt.contracts.project.UserConfig object (optional)
-        return_dict: Return a dict if true, return the full dbt.config.project.Project object if false
-
-    Returns:
-        A full project config
-
-    """
-    # Generate a profile if not provided
-    if profile is None:
-        # Generate user_config if not provided
-        if user_config is None:
-            user_config = read_user_config(flags.PROFILES_DIR)
-        # Update flags
-        flags.set_from_args(args, user_config)
-        if cli_vars is None:
-            cli_vars = {}
-        profile = Profile.render_from_args(args, ProfileRenderer(cli_vars), profile_name)
-    # Generate a project
-    project = Project.from_project_root(
-        project_path,
-        DbtProjectYamlRenderer(profile),
-        verify_version=bool(flags.VERSION_CHECK),
-    )
-    # Return
-    return project.to_project_config() if return_dict else project

core/dbt/constants.py

@@ -8,3 +8,7 @@ MAXIMUM_SEED_SIZE_NAME = "1MB"
 PIN_PACKAGE_URL = (
     "https://docs.getdbt.com/docs/package-management#section-specifying-package-versions"
 )
+
+DEPENDENCIES_FILE_NAME = "dependencies.yml"
+MANIFEST_FILE_NAME = "manifest.json"
+PARTIAL_PARSE_FILE_NAME = "partial_parse.msgpack"

core/dbt/context/base.py

@@ -2,7 +2,8 @@ import json
 import os
 from typing import Any, Dict, NoReturn, Optional, Mapping, Iterable, Set, List
 
-from dbt import flags
+from dbt.flags import get_flags
+import dbt.flags as flags_module
 from dbt import tracking
 from dbt import utils
 from dbt.clients.jinja import get_rendered
@@ -635,7 +636,7 @@ class BaseContext(metaclass=ContextMeta):
 
         This supports all flags defined in flags submodule (core/dbt/flags.py)
         """
-        return flags.get_flag_obj()
+        return flags_module.get_flag_obj()
 
     @contextmember
     @staticmethod
@@ -651,7 +652,7 @@ class BaseContext(metaclass=ContextMeta):
             {% endmacro %}"
         """
 
-        if not flags.NO_PRINT:
+        if get_flags().PRINT:
             print(msg)
         return ""
 

core/dbt/context/configured.py

@@ -16,7 +16,8 @@ class ConfiguredContext(TargetContext):
     config: AdapterRequiredConfig
 
     def __init__(self, config: AdapterRequiredConfig) -> None:
-        super().__init__(config, config.cli_vars)
+        super().__init__(config.to_target_dict(), config.cli_vars)
+        self.config = config
 
     @contextproperty
     def project_name(self) -> str:
@@ -51,10 +52,11 @@ class ConfiguredVar(Var):
         adapter_type = self._config.credentials.type
         lookup = FQNLookup(self._project_name)
         active_vars = self._config.vars.vars_for(lookup, adapter_type)
-        all_vars = MultiDict([active_vars])
 
+        all_vars = MultiDict()
         if self._config.project_name != my_config.project_name:
             all_vars.add(my_config.vars.vars_for(lookup, adapter_type))
+        all_vars.add(active_vars)
 
         if var_name in all_vars:
             return all_vars[var_name]

core/dbt/context/exceptions_jinja.py

@@ -23,6 +23,8 @@ from dbt.exceptions import (
     PropertyYMLError,
     NotImplementedError,
     RelationWrongTypeError,
+    ContractError,
+    ColumnTypeMissingError,
 )
 
 
@@ -65,6 +67,10 @@ def raise_compiler_error(msg, node=None) -> NoReturn:
     raise CompilationError(msg, node)
 
 
+def raise_contract_error(yaml_columns, sql_columns) -> NoReturn:
+    raise ContractError(yaml_columns, sql_columns)
+
+
 def raise_database_error(msg, node=None) -> NoReturn:
     raise DbtDatabaseError(msg, node)
 
@@ -97,6 +103,10 @@ def relation_wrong_type(relation, expected_type, model=None) -> NoReturn:
     raise RelationWrongTypeError(relation, expected_type, model)
 
 
+def column_type_missing(column_names) -> NoReturn:
+    raise ColumnTypeMissingError(column_names)
+
+
 # Update this when a new function should be added to the
 # dbt context's `exceptions` key!
 CONTEXT_EXPORTS = {
@@ -119,6 +129,8 @@ CONTEXT_EXPORTS = {
         raise_invalid_property_yml_version,
         raise_not_implemented,
         relation_wrong_type,
+        raise_contract_error,
+        column_type_missing,
     ]
 }
 

core/dbt/context/providers.py

@@ -37,8 +37,11 @@ from dbt.contracts.graph.nodes import (
     SourceDefinition,
     Resource,
     ManifestNode,
+    RefArgs,
+    AccessType,
 )
 from dbt.contracts.graph.metrics import MetricReference, ResolvedMetricReference
+from dbt.contracts.graph.unparsed import NodeVersion
 from dbt.events.functions import get_metadata_vars
 from dbt.exceptions import (
     CompilationError,
@@ -52,6 +55,7 @@ from dbt.exceptions import (
     LoadAgateTableNotSeedError,
     LoadAgateTableValueError,
     MacroDispatchArgError,
+    MacroResultAlreadyLoadedError,
     MacrosSourcesUnWriteableError,
     MetricArgsError,
     MissingConfigError,
@@ -63,11 +67,12 @@ from dbt.exceptions import (
     DbtRuntimeError,
     TargetNotFoundError,
     DbtValidationError,
+    DbtReferenceError,
 )
 from dbt.config import IsFQNResource
 from dbt.node_types import NodeType, ModelLanguage
 
-from dbt.utils import merge, AttrDict, MultiDict, args_to_dict
+from dbt.utils import merge, AttrDict, MultiDict, args_to_dict, cast_to_str
 
 from dbt import selected_resources
 
@@ -212,16 +217,17 @@ class BaseResolver(metaclass=abc.ABCMeta):
 
 class BaseRefResolver(BaseResolver):
     @abc.abstractmethod
-    def resolve(self, name: str, package: Optional[str] = None) -> RelationProxy:
+    def resolve(
+        self, name: str, package: Optional[str] = None, version: Optional[NodeVersion] = None
+    ) -> RelationProxy:
         ...
 
-    def _repack_args(self, name: str, package: Optional[str]) -> List[str]:
-        if package is None:
-            return [name]
-        else:
-            return [package, name]
+    def _repack_args(
+        self, name: str, package: Optional[str], version: Optional[NodeVersion]
+    ) -> RefArgs:
+        return RefArgs(package=package, name=name, version=version)
 
-    def validate_args(self, name: str, package: Optional[str]):
+    def validate_args(self, name: str, package: Optional[str], version: Optional[NodeVersion]):
         if not isinstance(name, str):
             raise CompilationError(
                 f"The name argument to ref() must be a string, got {type(name)}"
@@ -232,9 +238,15 @@ class BaseRefResolver(BaseResolver):
                 f"The package argument to ref() must be a string or None, got {type(package)}"
             )
 
-    def __call__(self, *args: str) -> RelationProxy:
+        if version is not None and not isinstance(version, (str, int, float)):
+            raise CompilationError(
+                f"The version argument to ref() must be a string, int, float, or None - got {type(version)}"
+            )
+
+    def __call__(self, *args: str, **kwargs) -> RelationProxy:
         name: str
         package: Optional[str] = None
+        version: Optional[NodeVersion] = None
 
         if len(args) == 1:
             name = args[0]
@@ -242,8 +254,10 @@ class BaseRefResolver(BaseResolver):
             package, name = args
         else:
             raise RefArgsError(node=self.model, args=args)
-        self.validate_args(name, package)
-        return self.resolve(name, package)
+
+        version = kwargs.get("version") or kwargs.get("v")
+        self.validate_args(name, package, version)
+        return self.resolve(name, package, version)
 
 
 class BaseSourceResolver(BaseResolver):
@@ -448,9 +462,12 @@ class RuntimeDatabaseWrapper(BaseDatabaseWrapper):
 
 # `ref` implementations
 class ParseRefResolver(BaseRefResolver):
-    def resolve(self, name: str, package: Optional[str] = None) -> RelationProxy:
-        self.model.refs.append(self._repack_args(name, package))
+    def resolve(
+        self, name: str, package: Optional[str] = None, version: Optional[NodeVersion] = None
+    ) -> RelationProxy:
+        self.model.refs.append(self._repack_args(name, package, version))
 
+        # This is not the ref for the "name" passed in, but for the current model.
         return self.Relation.create_from(self.config, self.model)
 
 
@@ -458,10 +475,17 @@ ResolveRef = Union[Disabled, ManifestNode]
 
 
 class RuntimeRefResolver(BaseRefResolver):
-    def resolve(self, target_name: str, target_package: Optional[str] = None) -> RelationProxy:
+    def resolve(
+        self,
+        target_name: str,
+        target_package: Optional[str] = None,
+        target_version: Optional[NodeVersion] = None,
+    ) -> RelationProxy:
         target_model = self.manifest.resolve_ref(
+            self.model,
             target_name,
             target_package,
+            target_version,
             self.current_project,
             self.model.package_name,
         )
@@ -472,23 +496,46 @@ class RuntimeRefResolver(BaseRefResolver):
                 target_name=target_name,
                 target_kind="node",
                 target_package=target_package,
+                target_version=target_version,
                 disabled=isinstance(target_model, Disabled),
             )
-        self.validate(target_model, target_name, target_package)
-        return self.create_relation(target_model, target_name)
+        elif (
+            target_model.resource_type == NodeType.Model
+            and target_model.access == AccessType.Private
+        ):
+            if not self.model.group or self.model.group != target_model.group:
+                raise DbtReferenceError(
+                    unique_id=self.model.unique_id,
+                    ref_unique_id=target_model.unique_id,
+                    group=cast_to_str(target_model.group),
+                )
 
-    def create_relation(self, target_model: ManifestNode, name: str) -> RelationProxy:
-        if target_model.is_ephemeral_model:
+        self.validate(target_model, target_name, target_package, target_version)
+        return self.create_relation(target_model)
+
+    def create_relation(self, target_model: ManifestNode) -> RelationProxy:
+        if target_model.is_public_node:
+            # Get quoting from publication artifact
+            pub_metadata = self.manifest.publications[target_model.package_name].metadata
+            return self.Relation.create_from_node(pub_metadata, target_model)
+        elif target_model.is_ephemeral_model:
             self.model.set_cte(target_model.unique_id, None)
             return self.Relation.create_ephemeral_from_node(self.config, target_model)
         else:
             return self.Relation.create_from(self.config, target_model)
 
     def validate(
-        self, resolved: ManifestNode, target_name: str, target_package: Optional[str]
+        self,
+        resolved: ManifestNode,
+        target_name: str,
+        target_package: Optional[str],
+        target_version: Optional[NodeVersion],
     ) -> None:
-        if resolved.unique_id not in self.model.depends_on.nodes:
-            args = self._repack_args(target_name, target_package)
+        if (
+            resolved.unique_id not in self.model.depends_on.nodes
+            and resolved.unique_id not in self.model.depends_on.public_nodes
+        ):
+            args = self._repack_args(target_name, target_package, target_version)
             raise RefBadContextError(node=self.model, args=args)
 
 
@@ -498,16 +545,17 @@ class OperationRefResolver(RuntimeRefResolver):
         resolved: ManifestNode,
         target_name: str,
         target_package: Optional[str],
+        target_version: Optional[NodeVersion],
     ) -> None:
         pass
 
-    def create_relation(self, target_model: ManifestNode, name: str) -> RelationProxy:
+    def create_relation(self, target_model: ManifestNode) -> RelationProxy:
         if target_model.is_ephemeral_model:
             # In operations, we can't ref() ephemeral nodes, because
             # Macros do not support set_cte
             raise OperationsCannotRefEphemeralNodesError(target_model.name, node=self.model)
         else:
-            return super().create_relation(target_model, name)
+            return super().create_relation(target_model)
 
 
 # `source` implementations
@@ -679,7 +727,7 @@ class ProviderContext(ManifestContext):
         self.config: RuntimeConfig
         self.model: Union[Macro, ManifestNode] = model
         super().__init__(config, manifest, model.package_name)
-        self.sql_results: Dict[str, AttrDict] = {}
+        self.sql_results: Dict[str, Optional[AttrDict]] = {}
         self.context_config: Optional[ContextConfig] = context_config
         self.provider: Provider = provider
         self.adapter = get_adapter(self.config)
@@ -707,12 +755,29 @@ class ProviderContext(ManifestContext):
         return args_to_dict(self.config.args)
 
     @contextproperty
-    def _sql_results(self) -> Dict[str, AttrDict]:
+    def _sql_results(self) -> Dict[str, Optional[AttrDict]]:
         return self.sql_results
 
     @contextmember
     def load_result(self, name: str) -> Optional[AttrDict]:
-        return self.sql_results.get(name)
+        if name in self.sql_results:
+            # handle the special case of "main" macro
+            # See: https://github.com/dbt-labs/dbt-core/blob/ada8860e48b32ac712d92e8b0977b2c3c9749981/core/dbt/task/run.py#L228
+            if name == "main":
+                return self.sql_results["main"]
+
+            # handle a None, which indicates this name was populated but has since been loaded
+            elif self.sql_results[name] is None:
+                raise MacroResultAlreadyLoadedError(name)
+
+            # Handle the regular use case
+            else:
+                ret_val = self.sql_results[name]
+                self.sql_results[name] = None
+                return ret_val
+        else:
+            # Handle trying to load a result that was never stored
+            return None
 
     @contextmember
     def store_result(
@@ -1408,10 +1473,18 @@ def generate_runtime_macro_context(
 
 
 class ExposureRefResolver(BaseResolver):
-    def __call__(self, *args) -> str:
-        if len(args) not in (1, 2):
+    def __call__(self, *args, **kwargs) -> str:
+        package = None
+        if len(args) == 1:
+            name = args[0]
+        elif len(args) == 2:
+            package, name = args
+        else:
             raise RefArgsError(node=self.model, args=args)
-        self.model.refs.append(list(args))
+
+        version = kwargs.get("version") or kwargs.get("v")
+
+        self.model.refs.append(RefArgs(package=package, name=name, version=version))
         return ""
 
 
@@ -1461,7 +1534,7 @@ def generate_parse_exposure(
 
 
 class MetricRefResolver(BaseResolver):
-    def __call__(self, *args) -> str:
+    def __call__(self, *args, **kwargs) -> str:
         package = None
         if len(args) == 1:
             name = args[0]
@@ -1469,11 +1542,14 @@ class MetricRefResolver(BaseResolver):
             package, name = args
         else:
             raise RefArgsError(node=self.model, args=args)
-        self.validate_args(name, package)
-        self.model.refs.append(list(args))
+
+        version = kwargs.get("version") or kwargs.get("v")
+        self.validate_args(name, package, version)
+
+        self.model.refs.append(RefArgs(package=package, name=name, version=version))
         return ""
 
-    def validate_args(self, name, package):
+    def validate_args(self, name, package, version):
         if not isinstance(name, str):
             raise ParsingError(
                 f"In a metrics section in {self.model.original_file_path} "

core/dbt/context/target.py

@@ -1,15 +1,13 @@
 from typing import Any, Dict
 
-from dbt.contracts.connection import HasCredentials
-
 from dbt.context.base import BaseContext, contextproperty
 
 
 class TargetContext(BaseContext):
     # subclass is ConfiguredContext
-    def __init__(self, config: HasCredentials, cli_vars: Dict[str, Any]):
+    def __init__(self, target_dict: Dict[str, Any], cli_vars: Dict[str, Any]):
         super().__init__(cli_vars=cli_vars)
-        self.config = config
+        self.target_dict = target_dict
 
     @contextproperty
     def target(self) -> Dict[str, Any]:
@@ -73,9 +71,4 @@ class TargetContext(BaseContext):
             |----------|-----------|------------------------------------------|
 
         """
-        return self.config.to_target_dict()
-
-
-def generate_target_context(config: HasCredentials, cli_vars: Dict[str, Any]) -> Dict[str, Any]:
-    ctx = TargetContext(config, cli_vars)
-    return ctx.to_dict()
+        return self.target_dict

core/dbt/contracts/connection.py

@@ -1,6 +1,5 @@
 import abc
 import itertools
-import hashlib
 from dataclasses import dataclass, field
 from typing import (
     Any,
@@ -13,7 +12,7 @@ from typing import (
     Callable,
 )
 from dbt.exceptions import DbtInternalError
-from dbt.utils import translate_aliases
+from dbt.utils import translate_aliases, md5
 from dbt.events.functions import fire_event
 from dbt.events.types import NewConnectionOpening
 from dbt.events.contextvars import get_node_info
@@ -142,7 +141,7 @@ class Credentials(ExtensibleDbtClassMixin, Replaceable, metaclass=abc.ABCMeta):
         raise NotImplementedError("unique_field not implemented for base credentials class")
 
     def hashed_unique_field(self) -> str:
-        return hashlib.md5(self.unique_field.encode("utf-8")).hexdigest()
+        return md5(self.unique_field)
 
     def connection_info(self, *, with_aliases: bool = False) -> Iterable[Tuple[str, Any]]:
         """Return an ordered iterator of key/value pairs for pretty-printing."""

core/dbt/contracts/files.py

@@ -61,8 +61,6 @@ class FilePath(dbtClassMixin):
 
     @property
     def original_file_path(self) -> str:
-        # this is mostly used for reporting errors. It doesn't show the project
-        # name, should it?
         return os.path.join(self.searched_path, self.relative_path)
 
     def seed_too_large(self) -> bool:
@@ -227,6 +225,7 @@ class SchemaSourceFile(BaseSourceFile):
     sources: List[str] = field(default_factory=list)
     exposures: List[str] = field(default_factory=list)
     metrics: List[str] = field(default_factory=list)
+    groups: List[str] = field(default_factory=list)
     # node patches contain models, seeds, snapshots, analyses
     ndp: List[str] = field(default_factory=list)
     # any macro patches in this file by macro unique_id.

core/dbt/contracts/graph/manifest.py

@@ -22,6 +22,8 @@ from typing import (
 from typing_extensions import Protocol
 from uuid import UUID
 
+from dbt.contracts.publication import ProjectDependencies, PublicationConfig, PublicModel
+
 from dbt.contracts.graph.nodes import (
     Macro,
     Documentation,
@@ -29,13 +31,16 @@ from dbt.contracts.graph.nodes import (
     GenericTestNode,
     Exposure,
     Metric,
+    Group,
     UnpatchedSourceDefinition,
     ManifestNode,
     GraphMemberNode,
     ResultNode,
     BaseNode,
+    ManifestOrPublicNode,
 )
-from dbt.contracts.graph.unparsed import SourcePatch
+from dbt.contracts.graph.unparsed import SourcePatch, NodeVersion, UnparsedVersion
+from dbt.contracts.graph.manifest_upgrade import upgrade_manifest_json
 from dbt.contracts.files import SourceFile, SchemaSourceFile, FileHash, AnySourceFile
 from dbt.contracts.util import BaseArtifactMetadata, SourceKey, ArtifactMixin, schema_version
 from dbt.dataclass_schema import dbtClassMixin
@@ -44,12 +49,14 @@ from dbt.exceptions import (
     DuplicateResourceNameError,
     DuplicateMacroInPackageError,
     DuplicateMaterializationNameError,
+    AmbiguousResourceNameRefError,
 )
 from dbt.helper_types import PathSet
 from dbt.events.functions import fire_event
-from dbt.events.types import MergedFromState
+from dbt.events.types import MergedFromState, UnpinnedRefNewVersionAvailable
+from dbt.events.contextvars import get_node_info
 from dbt.node_types import NodeType
-from dbt import flags
+from dbt.flags import get_flags, MP_CONTEXT
 from dbt import tracking
 import dbt.utils
 
@@ -144,39 +151,115 @@ class SourceLookup(dbtClassMixin):
 class RefableLookup(dbtClassMixin):
     # model, seed, snapshot
     _lookup_types: ClassVar[set] = set(NodeType.refable())
+    _versioned_types: ClassVar[set] = set(NodeType.versioned())
 
-    # refables are actually unique, so the Dict[PackageName, UniqueID] will
-    # only ever have exactly one value, but doing 3 dict lookups instead of 1
-    # is not a big deal at all and retains consistency
     def __init__(self, manifest: "Manifest"):
         self.storage: Dict[str, Dict[PackageName, UniqueID]] = {}
         self.populate(manifest)
+        self.populate_public_nodes(manifest)
 
-    def get_unique_id(self, key, package: Optional[PackageName]):
-        return find_unique_id_for_package(self.storage, key, package)
+    def get_unique_id(
+        self,
+        key: str,
+        package: Optional[PackageName],
+        version: Optional[NodeVersion],
+        node: Optional[GraphMemberNode] = None,
+    ):
+        if version:
+            key = f"{key}.v{version}"
 
-    def find(self, key, package: Optional[PackageName], manifest: "Manifest"):
-        unique_id = self.get_unique_id(key, package)
+        unique_ids = self._find_unique_ids_for_package(key, package)
+        if len(unique_ids) > 1:
+            raise AmbiguousResourceNameRefError(key, unique_ids, node)
+        else:
+            return unique_ids[0] if unique_ids else None
+
+    def find(
+        self,
+        key: str,
+        package: Optional[PackageName],
+        version: Optional[NodeVersion],
+        manifest: "Manifest",
+        source_node: Optional[GraphMemberNode] = None,
+    ):
+        unique_id = self.get_unique_id(key, package, version, source_node)
         if unique_id is not None:
-            return self.perform_lookup(unique_id, manifest)
+            node = self.perform_lookup(unique_id, manifest)
+            # If this is an unpinned ref (no 'version' arg was passed),
+            # AND this is a versioned node,
+            # AND this ref is being resolved at runtime -- get_node_info != {}
+            if version is None and node.is_versioned and get_node_info():
+                # Check to see if newer versions are available, and log an "FYI" if so
+                max_version: UnparsedVersion = max(
+                    [
+                        UnparsedVersion(v.version)
+                        for v in manifest.nodes.values()
+                        if v.name == node.name and v.version is not None
+                    ]
+                )
+                assert node.latest_version  # for mypy, whenever i may find it
+                if max_version > UnparsedVersion(node.latest_version):
+                    fire_event(
+                        UnpinnedRefNewVersionAvailable(
+                            node_info=get_node_info(),
+                            ref_node_name=node.name,
+                            ref_node_package=node.package_name,
+                            ref_node_version=str(node.version),
+                            ref_max_version=str(max_version.v),
+                        )
+                    )
+
+            return node
         return None
 
-    def add_node(self, node: ManifestNode):
+    def add_node(self, node: ManifestOrPublicNode):
         if node.resource_type in self._lookup_types:
             if node.name not in self.storage:
                 self.storage[node.name] = {}
-            self.storage[node.name][node.package_name] = node.unique_id
+
+            if node.is_versioned:
+                if node.search_name not in self.storage:
+                    self.storage[node.search_name] = {}
+                self.storage[node.search_name][node.package_name] = node.unique_id
+                if node.is_latest_version:  # type: ignore
+                    self.storage[node.name][node.package_name] = node.unique_id
+            else:
+                self.storage[node.name][node.package_name] = node.unique_id
 
     def populate(self, manifest):
         for node in manifest.nodes.values():
             self.add_node(node)
 
-    def perform_lookup(self, unique_id: UniqueID, manifest) -> ManifestNode:
-        if unique_id not in manifest.nodes:
+    def populate_public_nodes(self, manifest):
+        for node in manifest.public_nodes.values():
+            self.add_node(node)
+
+    def perform_lookup(self, unique_id: UniqueID, manifest) -> ManifestOrPublicNode:
+        if unique_id in manifest.nodes:
+            node = manifest.nodes[unique_id]
+        elif unique_id in manifest.public_nodes:
+            node = manifest.public_nodes[unique_id]
+        else:
             raise dbt.exceptions.DbtInternalError(
                 f"Node {unique_id} found in cache but not found in manifest"
             )
-        return manifest.nodes[unique_id]
+        return node
+
+    def _find_unique_ids_for_package(self, key, package: Optional[PackageName]) -> List[str]:
+        if key not in self.storage:
+            return []
+
+        pkg_dct: Mapping[PackageName, UniqueID] = self.storage[key]
+
+        if package is None:
+            if not pkg_dct:
+                return []
+            else:
+                return list(pkg_dct.values())
+        elif package in pkg_dct:
+            return [pkg_dct[package]]
+        else:
+            return []
 
 
 class MetricLookup(dbtClassMixin):
@@ -231,7 +314,12 @@ class DisabledLookup(dbtClassMixin):
 
     # This should return a list of disabled nodes. It's different from
     # the other Lookup functions in that it returns full nodes, not just unique_ids
-    def find(self, search_name, package: Optional[PackageName]):
+    def find(
+        self, search_name, package: Optional[PackageName], version: Optional[NodeVersion] = None
+    ):
+        if version:
+            search_name = f"{search_name}.v{version}"
+
         if search_name not in self.storage:
             return None
 
@@ -250,9 +338,10 @@ class DisabledLookup(dbtClassMixin):
 
 class AnalysisLookup(RefableLookup):
     _lookup_types: ClassVar[set] = set([NodeType.Analysis])
+    _versioned_types: ClassVar[set] = set()
 
 
-def _search_packages(
+def _packages_to_search(
     current_project: str,
     node_package: str,
     target_package: Optional[str] = None,
@@ -303,7 +392,7 @@ class ManifestMetadata(BaseArtifactMetadata):
             self.user_id = tracking.active_user.id
 
         if self.send_anonymous_usage_stats is None:
-            self.send_anonymous_usage_stats = flags.SEND_ANONYMOUS_USAGE_STATS
+            self.send_anonymous_usage_stats = get_flags().SEND_ANONYMOUS_USAGE_STATS
 
     @classmethod
     def default(cls):
@@ -329,7 +418,8 @@ def build_node_edges(nodes: List[ManifestNode]):
     forward_edges: Dict[str, List[str]] = {n.unique_id: [] for n in nodes}
     for node in nodes:
         backward_edges[node.unique_id] = node.depends_on_nodes[:]
-        for unique_id in node.depends_on_nodes:
+        backward_edges[node.unique_id].extend(node.depends_on_public_nodes[:])
+        for unique_id in backward_edges[node.unique_id]:
             if unique_id in forward_edges.keys():
                 forward_edges[unique_id].append(node.unique_id)
     return _sort_values(forward_edges), _sort_values(backward_edges)
@@ -473,25 +563,6 @@ MaybeNonSource = Optional[Union[ManifestNode, Disabled[ManifestNode]]]
 T = TypeVar("T", bound=GraphMemberNode)
 
 
-def _update_into(dest: MutableMapping[str, T], new_item: T):
-    """Update dest to overwrite whatever is at dest[new_item.unique_id] with
-    new_itme. There must be an existing value to overwrite, and the two nodes
-    must have the same original file path.
-    """
-    unique_id = new_item.unique_id
-    if unique_id not in dest:
-        raise dbt.exceptions.DbtRuntimeError(
-            f"got an update_{new_item.resource_type} call with an "
-            f"unrecognized {new_item.resource_type}: {new_item.unique_id}"
-        )
-    existing = dest[unique_id]
-    if new_item.original_file_path != existing.original_file_path:
-        raise dbt.exceptions.DbtRuntimeError(
-            f"cannot update a {new_item.resource_type} to have a new file path!"
-        )
-    dest[unique_id] = new_item
-
-
 # This contains macro methods that are in both the Manifest
 # and the MacroManifest
 class MacroMethods:
@@ -599,6 +670,7 @@ class Manifest(MacroMethods, DataClassMessagePackMixin, dbtClassMixin):
     docs: MutableMapping[str, Documentation] = field(default_factory=dict)
     exposures: MutableMapping[str, Exposure] = field(default_factory=dict)
     metrics: MutableMapping[str, Metric] = field(default_factory=dict)
+    groups: MutableMapping[str, Group] = field(default_factory=dict)
     selectors: MutableMapping[str, Any] = field(default_factory=dict)
     files: MutableMapping[str, AnySourceFile] = field(default_factory=dict)
     metadata: ManifestMetadata = field(default_factory=ManifestMetadata)
@@ -607,6 +679,9 @@ class Manifest(MacroMethods, DataClassMessagePackMixin, dbtClassMixin):
     source_patches: MutableMapping[SourceKey, SourcePatch] = field(default_factory=dict)
     disabled: MutableMapping[str, List[GraphMemberNode]] = field(default_factory=dict)
     env_vars: MutableMapping[str, str] = field(default_factory=dict)
+    public_nodes: MutableMapping[str, PublicModel] = field(default_factory=dict)
+    project_dependencies: Optional[ProjectDependencies] = None
+    publications: MutableMapping[str, PublicationConfig] = field(default_factory=dict)
 
     _doc_lookup: Optional[DocLookup] = field(
         default=None, metadata={"serialize": lambda x: None, "deserialize": lambda x: None}
@@ -631,7 +706,7 @@ class Manifest(MacroMethods, DataClassMessagePackMixin, dbtClassMixin):
         metadata={"serialize": lambda x: None, "deserialize": lambda x: None},
     )
     _lock: Lock = field(
-        default_factory=flags.MP_CONTEXT.Lock,
+        default_factory=MP_CONTEXT.Lock,
         metadata={"serialize": lambda x: None, "deserialize": lambda x: None},
     )
 
@@ -643,39 +718,9 @@ class Manifest(MacroMethods, DataClassMessagePackMixin, dbtClassMixin):
 
     @classmethod
     def __post_deserialize__(cls, obj):
-        obj._lock = flags.MP_CONTEXT.Lock()
+        obj._lock = MP_CONTEXT.Lock()
         return obj
 
-    def sync_update_node(self, new_node: ManifestNode) -> ManifestNode:
-        """update the node with a lock. The only time we should want to lock is
-        when compiling an ephemeral ancestor of a node at runtime, because
-        multiple threads could be just-in-time compiling the same ephemeral
-        dependency, and we want them to have a consistent view of the manifest.
-
-        If the existing node is not compiled, update it with the new node and
-        return that. If the existing node is compiled, do not update the
-        manifest and return the existing node.
-        """
-        with self._lock:
-            existing = self.nodes[new_node.unique_id]
-            if getattr(existing, "compiled", False):
-                # already compiled
-                return existing
-            _update_into(self.nodes, new_node)
-            return new_node
-
-    def update_exposure(self, new_exposure: Exposure):
-        _update_into(self.exposures, new_exposure)
-
-    def update_metric(self, new_metric: Metric):
-        _update_into(self.metrics, new_metric)
-
-    def update_node(self, new_node: ManifestNode):
-        _update_into(self.nodes, new_node)
-
-    def update_source(self, new_source: SourceDefinition):
-        _update_into(self.sources, new_source)
-
     def build_flat_graph(self):
         """This attribute is used in context.common by each node, so we want to
         only build it once and avoid any concurrency issues around it.
@@ -684,9 +729,11 @@ class Manifest(MacroMethods, DataClassMessagePackMixin, dbtClassMixin):
         """
         self.flat_graph = {
             "exposures": {k: v.to_dict(omit_none=False) for k, v in self.exposures.items()},
+            "groups": {k: v.to_dict(omit_none=False) for k, v in self.groups.items()},
             "metrics": {k: v.to_dict(omit_none=False) for k, v in self.metrics.items()},
             "nodes": {k: v.to_dict(omit_none=False) for k, v in self.nodes.items()},
             "sources": {k: v.to_dict(omit_none=False) for k, v in self.sources.items()},
+            "public_nodes": {k: v.to_dict(omit_none=False) for k, v in self.public_nodes.items()},
         }
 
     def build_disabled_by_file_id(self):
@@ -775,9 +822,11 @@ class Manifest(MacroMethods, DataClassMessagePackMixin, dbtClassMixin):
             docs={k: _deepcopy(v) for k, v in self.docs.items()},
             exposures={k: _deepcopy(v) for k, v in self.exposures.items()},
             metrics={k: _deepcopy(v) for k, v in self.metrics.items()},
+            groups={k: _deepcopy(v) for k, v in self.groups.items()},
             selectors={k: _deepcopy(v) for k, v in self.selectors.items()},
             metadata=self.metadata,
             disabled={k: _deepcopy(v) for k, v in self.disabled.items()},
+            public_nodes={k: _deepcopy(v) for k, v in self.public_nodes.items()},
             files={k: _deepcopy(v) for k, v in self.files.items()},
             state_check=_deepcopy(self.state_check),
         )
@@ -791,6 +840,7 @@ class Manifest(MacroMethods, DataClassMessagePackMixin, dbtClassMixin):
                 self.sources.values(),
                 self.exposures.values(),
                 self.metrics.values(),
+                self.public_nodes.values(),
             )
         )
         forward_edges, backward_edges = build_node_edges(edge_members)
@@ -807,8 +857,22 @@ class Manifest(MacroMethods, DataClassMessagePackMixin, dbtClassMixin):
         forward_edges = build_macro_edges(edge_members)
         return forward_edges
 
+    def build_group_map(self):
+        groupable_nodes = list(
+            chain(
+                self.nodes.values(),
+                self.metrics.values(),
+            )
+        )
+        group_map = {group.name: [] for group in self.groups.values()}
+        for node in groupable_nodes:
+            if node.group is not None:
+                group_map[node.group].append(node.unique_id)
+        self.group_map = group_map
+
     def writable_manifest(self):
         self.build_parent_and_child_maps()
+        self.build_group_map()
         return WritableManifest(
             nodes=self.nodes,
             sources=self.sources,
@@ -816,11 +880,14 @@ class Manifest(MacroMethods, DataClassMessagePackMixin, dbtClassMixin):
             docs=self.docs,
             exposures=self.exposures,
             metrics=self.metrics,
+            groups=self.groups,
             selectors=self.selectors,
             metadata=self.metadata,
             disabled=self.disabled,
+            public_nodes=self.public_nodes,
             child_map=self.child_map,
             parent_map=self.parent_map,
+            group_map=self.group_map,
         )
 
     def write(self, path):
@@ -891,12 +958,14 @@ class Manifest(MacroMethods, DataClassMessagePackMixin, dbtClassMixin):
             self._analysis_lookup = AnalysisLookup(self)
         return self._analysis_lookup
 
-    # Called by dbt.parser.manifest._resolve_refs_for_exposure
+    # Called by dbt.parser.manifest._process_refs_for_exposure, _process_refs_for_metric,
     # and dbt.parser.manifest._process_refs_for_node
     def resolve_ref(
         self,
+        source_node: GraphMemberNode,
         target_model_name: str,
         target_model_package: Optional[str],
+        target_model_version: Optional[NodeVersion],
         current_project: str,
         node_package: str,
     ) -> MaybeNonSource:
@@ -904,16 +973,20 @@ class Manifest(MacroMethods, DataClassMessagePackMixin, dbtClassMixin):
         node: Optional[ManifestNode] = None
         disabled: Optional[List[ManifestNode]] = None
 
-        candidates = _search_packages(current_project, node_package, target_model_package)
+        candidates = _packages_to_search(current_project, node_package, target_model_package)
         for pkg in candidates:
-            node = self.ref_lookup.find(target_model_name, pkg, self)
+            node = self.ref_lookup.find(
+                target_model_name, pkg, target_model_version, self, source_node
+            )
 
-            if node is not None and node.config.enabled:
+            if node is not None and (
+                (hasattr(node, "config") and node.config.enabled) or node.is_public_node
+            ):
                 return node
 
             # it's possible that the node is disabled
             if disabled is None:
-                disabled = self.disabled_lookup.find(target_model_name, pkg)
+                disabled = self.disabled_lookup.find(target_model_name, pkg, target_model_version)
 
         if disabled:
             return Disabled(disabled[0])
@@ -929,7 +1002,7 @@ class Manifest(MacroMethods, DataClassMessagePackMixin, dbtClassMixin):
         node_package: str,
     ) -> MaybeParsedSource:
         search_name = f"{target_source_name}.{target_table_name}"
-        candidates = _search_packages(current_project, node_package)
+        candidates = _packages_to_search(current_project, node_package)
 
         source: Optional[SourceDefinition] = None
         disabled: Optional[List[SourceDefinition]] = None
@@ -959,7 +1032,7 @@ class Manifest(MacroMethods, DataClassMessagePackMixin, dbtClassMixin):
         metric: Optional[Metric] = None
         disabled: Optional[List[Metric]] = None
 
-        candidates = _search_packages(current_project, node_package, target_metric_package)
+        candidates = _packages_to_search(current_project, node_package, target_metric_package)
         for pkg in candidates:
             metric = self.metric_lookup.find(target_metric_name, pkg, self)
 
@@ -985,7 +1058,7 @@ class Manifest(MacroMethods, DataClassMessagePackMixin, dbtClassMixin):
         resolve_ref except the is_enabled checks are unnecessary as docs are
         always enabled.
         """
-        candidates = _search_packages(current_project, node_package, package)
+        candidates = _packages_to_search(current_project, node_package, package)
 
         for pkg in candidates:
             result = self.doc_lookup.find(name, pkg, self)
@@ -1070,6 +1143,8 @@ class Manifest(MacroMethods, DataClassMessagePackMixin, dbtClassMixin):
                 source_file.metrics.append(node.unique_id)
             if isinstance(node, Exposure):
                 source_file.exposures.append(node.unique_id)
+            if isinstance(node, Group):
+                source_file.groups.append(node.unique_id)
         else:
             source_file.nodes.append(node.unique_id)
 
@@ -1083,6 +1158,11 @@ class Manifest(MacroMethods, DataClassMessagePackMixin, dbtClassMixin):
         self.metrics[metric.unique_id] = metric
         source_file.metrics.append(metric.unique_id)
 
+    def add_group(self, source_file: SchemaSourceFile, group: Group):
+        _check_duplicates(group, self.groups)
+        self.groups[group.unique_id] = group
+        source_file.groups.append(group.unique_id)
+
     def add_disabled_nofile(self, node: GraphMemberNode):
         # There can be multiple disabled nodes for the same unique_id
         if node.unique_id in self.disabled:
@@ -1125,6 +1205,7 @@ class Manifest(MacroMethods, DataClassMessagePackMixin, dbtClassMixin):
             self.docs,
             self.exposures,
             self.metrics,
+            self.groups,
             self.selectors,
             self.files,
             self.metadata,
@@ -1133,6 +1214,7 @@ class Manifest(MacroMethods, DataClassMessagePackMixin, dbtClassMixin):
             self.source_patches,
             self.disabled,
             self.env_vars,
+            self.public_nodes,
             self._doc_lookup,
             self._source_lookup,
             self._ref_lookup,
@@ -1156,7 +1238,7 @@ AnyManifest = Union[Manifest, MacroManifest]
 
 
 @dataclass
-@schema_version("manifest", 8)
+@schema_version("manifest", 10)
 class WritableManifest(ArtifactMixin):
     nodes: Mapping[UniqueID, ManifestNode] = field(
         metadata=dict(description=("The nodes defined in the dbt project and its dependencies"))
@@ -1178,10 +1260,13 @@ class WritableManifest(ArtifactMixin):
     metrics: Mapping[UniqueID, Metric] = field(
         metadata=dict(description=("The metrics defined in the dbt project and its dependencies"))
     )
+    groups: Mapping[UniqueID, Group] = field(
+        metadata=dict(description=("The groups defined in the dbt project"))
+    )
     selectors: Mapping[UniqueID, Any] = field(
         metadata=dict(description=("The selectors defined in selectors.yml"))
     )
-    disabled: Optional[Mapping[UniqueID, List[ResultNode]]] = field(
+    disabled: Optional[Mapping[UniqueID, List[GraphMemberNode]]] = field(
         metadata=dict(description="A mapping of the disabled nodes in the target")
     )
     parent_map: Optional[NodeEdgeMap] = field(
@@ -1194,6 +1279,14 @@ class WritableManifest(ArtifactMixin):
             description="A mapping from parent nodes to their dependents",
         )
     )
+    group_map: Optional[NodeEdgeMap] = field(
+        metadata=dict(
+            description="A mapping from group names to their nodes",
+        )
+    )
+    public_nodes: Mapping[UniqueID, PublicModel] = field(
+        metadata=dict(description=("The public models used in the dbt project"))
+    )
     metadata: ManifestMetadata = field(
         metadata=dict(
             description="Metadata about the manifest",
@@ -1202,7 +1295,22 @@ class WritableManifest(ArtifactMixin):
 
     @classmethod
     def compatible_previous_versions(self):
-        return [("manifest", 4), ("manifest", 5), ("manifest", 6), ("manifest", 7)]
+        return [
+            ("manifest", 4),
+            ("manifest", 5),
+            ("manifest", 6),
+            ("manifest", 7),
+            ("manifest", 8),
+            ("manifest", 9),
+        ]
+
+    @classmethod
+    def upgrade_schema_version(cls, data):
+        """This overrides the "upgrade_schema_version" call in VersionedSchema (via
+        ArtifactMixin) to modify the dictionary passed in from earlier versions of the manifest."""
+        if get_manifest_schema_version(data) <= 9:
+            data = upgrade_manifest_json(data)
+        return cls.from_dict(data)
 
     def __post_serialize__(self, dct):
         for unique_id, node in dct["nodes"].items():
@@ -1211,6 +1319,13 @@ class WritableManifest(ArtifactMixin):
         return dct
 
 
+def get_manifest_schema_version(dct: dict) -> int:
+    schema_version = dct.get("metadata", {}).get("dbt_schema_version", None)
+    if not schema_version:
+        raise ValueError("Manifest doesn't have schema version")
+    return int(schema_version.split(".")[-2][-1])
+
+
 def _check_duplicates(value: BaseNode, src: Mapping[str, BaseNode]):
     if value.unique_id in src:
         raise DuplicateResourceNameError(value, src[value.unique_id])

core/dbt/contracts/graph/manifest_upgrade.py

@@ -0,0 +1,130 @@
+from dbt import deprecations
+from dbt.dataclass_schema import ValidationError
+
+
+# we renamed these properties in v1.3
+# this method allows us to be nice to the early adopters
+def rename_metric_attr(data: dict, raise_deprecation_warning: bool = False) -> dict:
+    metric_name = data["name"]
+    if raise_deprecation_warning and (
+        "sql" in data.keys()
+        or "type" in data.keys()
+        or data.get("calculation_method") == "expression"
+    ):
+        deprecations.warn("metric-attr-renamed", metric_name=metric_name)
+    duplicated_attribute_msg = """\n
+The metric '{}' contains both the deprecated metric property '{}'
+and the up-to-date metric property '{}'. Please remove the deprecated property.
+"""
+    if "sql" in data.keys():
+        if "expression" in data.keys():
+            raise ValidationError(
+                duplicated_attribute_msg.format(metric_name, "sql", "expression")
+            )
+        else:
+            data["expression"] = data.pop("sql")
+    if "type" in data.keys():
+        if "calculation_method" in data.keys():
+            raise ValidationError(
+                duplicated_attribute_msg.format(metric_name, "type", "calculation_method")
+            )
+        else:
+            calculation_method = data.pop("type")
+            data["calculation_method"] = calculation_method
+    # we also changed "type: expression" -> "calculation_method: derived"
+    if data.get("calculation_method") == "expression":
+        data["calculation_method"] = "derived"
+    return data
+
+
+def rename_sql_attr(node_content: dict) -> dict:
+    if "raw_sql" in node_content:
+        node_content["raw_code"] = node_content.pop("raw_sql")
+    if "compiled_sql" in node_content:
+        node_content["compiled_code"] = node_content.pop("compiled_sql")
+    node_content["language"] = "sql"
+    return node_content
+
+
+def upgrade_ref_content(node_content: dict) -> dict:
+    # In v1.5 we switched Node.refs from List[List[str]] to List[Dict[str, Union[NodeVersion, str]]]
+    # Previous versions did not have a version keyword argument for ref
+    if "refs" in node_content:
+        upgraded_refs = []
+        for ref in node_content["refs"]:
+            if isinstance(ref, list):
+                if len(ref) == 1:
+                    upgraded_refs.append({"package": None, "name": ref[0], "version": None})
+                else:
+                    upgraded_refs.append({"package": ref[0], "name": ref[1], "version": None})
+        node_content["refs"] = upgraded_refs
+    return node_content
+
+
+def upgrade_node_content(node_content):
+    rename_sql_attr(node_content)
+    upgrade_ref_content(node_content)
+    if node_content["resource_type"] != "seed" and "root_path" in node_content:
+        del node_content["root_path"]
+
+
+def upgrade_seed_content(node_content):
+    # Remove compilation related attributes
+    for attr_name in (
+        "language",
+        "refs",
+        "sources",
+        "metrics",
+        "compiled_path",
+        "compiled",
+        "compiled_code",
+        "extra_ctes_injected",
+        "extra_ctes",
+        "relation_name",
+    ):
+        if attr_name in node_content:
+            del node_content[attr_name]
+        # In v1.4, we switched SeedNode.depends_on from DependsOn to MacroDependsOn
+        node_content.get("depends_on", {}).pop("nodes", None)
+
+
+def upgrade_manifest_json(manifest: dict) -> dict:
+    for node_content in manifest.get("nodes", {}).values():
+        upgrade_node_content(node_content)
+        if node_content["resource_type"] == "seed":
+            upgrade_seed_content(node_content)
+    for disabled in manifest.get("disabled", {}).values():
+        # There can be multiple disabled nodes for the same unique_id
+        # so make sure all the nodes get the attr renamed
+        for node_content in disabled:
+            upgrade_node_content(node_content)
+            if node_content["resource_type"] == "seed":
+                upgrade_seed_content(node_content)
+    # add group key
+    if "groups" not in manifest:
+        manifest["groups"] = {}
+    if "group_map" not in manifest:
+        manifest["group_map"] = {}
+    if "public_nodes" not in manifest:
+        manifest["public_nodes"] = {}
+    for metric_content in manifest.get("metrics", {}).values():
+        # handle attr renames + value translation ("expression" -> "derived")
+        metric_content = rename_metric_attr(metric_content)
+        metric_content = upgrade_ref_content(metric_content)
+        if "root_path" in metric_content:
+            del metric_content["root_path"]
+    for exposure_content in manifest.get("exposures", {}).values():
+        exposure_content = upgrade_ref_content(exposure_content)
+        if "root_path" in exposure_content:
+            del exposure_content["root_path"]
+    for source_content in manifest.get("sources", {}).values():
+        if "root_path" in source_content:
+            del source_content["root_path"]
+    for macro_content in manifest.get("macros", {}).values():
+        if "root_path" in macro_content:
+            del macro_content["root_path"]
+    for doc_content in manifest.get("docs", {}).values():
+        if "root_path" in doc_content:
+            del doc_content["root_path"]
+        doc_content["resource_type"] = "doc"
+    return manifest

core/dbt/contracts/graph/model_config.py

@@ -189,6 +189,11 @@ class Severity(str):
 register_pattern(Severity, insensitive_patterns("warn", "error"))
 
 
+@dataclass
+class ContractConfig(dbtClassMixin, Replaceable):
+    enforced: bool = False
+
+
 @dataclass
 class Hook(dbtClassMixin, Replaceable):
     sql: str
@@ -286,7 +291,7 @@ class BaseConfig(AdditionalPropertiesAllowed, Replaceable):
     # 'meta' moved here from node
     mergebehavior = {
         "append": ["pre-hook", "pre_hook", "post-hook", "post_hook", "tags"],
-        "update": ["quoting", "column_types", "meta", "docs"],
+        "update": ["quoting", "column_types", "meta", "docs", "contract"],
         "dict_key_append": ["grants"],
     }
 
@@ -366,6 +371,7 @@ class BaseConfig(AdditionalPropertiesAllowed, Replaceable):
 @dataclass
 class MetricConfig(BaseConfig):
     enabled: bool = True
+    group: Optional[str] = None
 
 
 @dataclass
@@ -403,6 +409,10 @@ class NodeAndTestConfig(BaseConfig):
         default_factory=dict,
         metadata=MergeBehavior.Update.meta(),
     )
+    group: Optional[str] = field(
+        default=None,
+        metadata=CompareBehavior.Exclude.meta(),
+    )
 
 
 @dataclass
@@ -446,9 +456,13 @@ class NodeConfig(NodeAndTestConfig):
         default_factory=Docs,
         metadata=MergeBehavior.Update.meta(),
     )
+    contract: ContractConfig = field(
+        default_factory=ContractConfig,
+        metadata=MergeBehavior.Update.meta(),
+    )
 
-    # we validate that node_color has a suitable value to prevent dbt-docs from crashing
     def __post_init__(self):
+        # we validate that node_color has a suitable value to prevent dbt-docs from crashing
         if self.docs.node_color:
             node_color = self.docs.node_color
             if not validate_color(node_color):
@@ -457,6 +471,17 @@ class NodeConfig(NodeAndTestConfig):
                     "It is neither a valid HTML color name nor a valid HEX code."
                 )
 
+        if (
+            self.contract.enforced
+            and self.materialized == "incremental"
+            and self.on_schema_change != "append_new_columns"
+        ):
+            raise ValidationError(
+                f"Invalid value for on_schema_change: {self.on_schema_change}. Models "
+                "materialized as incremental with contracts enabled must set "
+                "on_schema_change to 'append_new_columns'"
+            )
+
     @classmethod
     def __pre_deserialize__(cls, data):
         data = super().__pre_deserialize__(data)

core/dbt/contracts/graph/nodes.py

@@ -1,6 +1,9 @@
 import os
 import time
 from dataclasses import dataclass, field
+from enum import Enum
+import hashlib
+
 from mashumaro.types import SerializableType
 from typing import (
     Optional,
@@ -18,36 +21,36 @@ from dbt.dataclass_schema import dbtClassMixin, ExtensibleDbtClassMixin
 from dbt.clients.system import write_file
 from dbt.contracts.files import FileHash
 from dbt.contracts.graph.unparsed import (
-    Quoting,
     Docs,
-    FreshnessThreshold,
+    ExposureType,
     ExternalTable,
+    FreshnessThreshold,
     HasYamlMetadata,
     MacroArgument,
-    UnparsedSourceDefinition,
-    UnparsedSourceTableDefinition,
-    UnparsedColumn,
-    TestDef,
-    ExposureOwner,
-    ExposureType,
     MaturityType,
     MetricFilter,
     MetricTime,
+    Owner,
+    Quoting,
+    TestDef,
+    NodeVersion,
+    UnparsedSourceDefinition,
+    UnparsedSourceTableDefinition,
+    UnparsedColumn,
 )
 from dbt.contracts.util import Replaceable, AdditionalPropertiesMixin
-from dbt.events.proto_types import NodeInfo
 from dbt.events.functions import warn_or_error
+from dbt.exceptions import ParsingError, InvalidAccessTypeError, ContractBreakingChangeError
 from dbt.events.types import (
     SeedIncreased,
     SeedExceedsLimitSamePath,
     SeedExceedsLimitAndPathChanged,
     SeedExceedsLimitChecksumChanged,
+    ValidationWarning,
 )
 from dbt.events.contextvars import set_contextvars
-from dbt import flags
-from dbt.node_types import ModelLanguage, NodeType
-from dbt.utils import cast_dict_to_dict_of_strings
-
+from dbt.flags import get_flags
+from dbt.node_types import ModelLanguage, NodeType, AccessType
 
 from .model_config import (
     NodeConfig,
@@ -59,6 +62,13 @@ from .model_config import (
     EmptySnapshotConfig,
     SnapshotConfig,
 )
+import sys
+
+if sys.version_info >= (3, 8):
+    from typing import Protocol
+else:
+    from typing_extensions import Protocol
+
 
 # =====================================================================
 # This contains the classes for all of the nodes and node-like objects
@@ -115,6 +125,10 @@ class BaseNode(dbtClassMixin, Replaceable):
     def is_relational(self):
         return self.resource_type in NodeType.refable()
 
+    @property
+    def is_versioned(self):
+        return self.resource_type in NodeType.versioned() and self.version is not None
+
     @property
     def is_ephemeral(self):
         return self.config.materialized == "ephemeral"
@@ -137,6 +151,62 @@ class GraphNode(BaseNode):
         return self.fqn == other.fqn
 
 
+@dataclass
+class RefArgs(dbtClassMixin):
+    name: str
+    package: Optional[str] = None
+    version: Optional[NodeVersion] = None
+
+    @property
+    def positional_args(self) -> List[str]:
+        if self.package:
+            return [self.package, self.name]
+        else:
+            return [self.name]
+
+    @property
+    def keyword_args(self) -> Dict[str, Optional[NodeVersion]]:
+        if self.version:
+            return {"version": self.version}
+        else:
+            return {}
+
+
+class ConstraintType(str, Enum):
+    check = "check"
+    not_null = "not_null"
+    unique = "unique"
+    primary_key = "primary_key"
+    foreign_key = "foreign_key"
+    custom = "custom"
+
+    @classmethod
+    def is_valid(cls, item):
+        try:
+            cls(item)
+        except ValueError:
+            return False
+        return True
+
+
+@dataclass
+class ColumnLevelConstraint(dbtClassMixin):
+    type: ConstraintType
+    name: Optional[str] = None
+    expression: Optional[str] = None
+    warn_unenforced: bool = (
+        True  # Warn if constraint cannot be enforced by platform but will be in DDL
+    )
+    warn_unsupported: bool = (
+        True  # Warn if constraint is not supported by the platform and won't be in DDL
+    )
+
+
+@dataclass
+class ModelLevelConstraint(ColumnLevelConstraint):
+    columns: List[str] = field(default_factory=list)
+
+
 @dataclass
 class ColumnInfo(AdditionalPropertiesMixin, ExtensibleDbtClassMixin, Replaceable):
     """Used in all ManifestNodes and SourceDefinition"""
@@ -145,11 +215,18 @@ class ColumnInfo(AdditionalPropertiesMixin, ExtensibleDbtClassMixin, Replaceable
     description: str = ""
     meta: Dict[str, Any] = field(default_factory=dict)
     data_type: Optional[str] = None
+    constraints: List[ColumnLevelConstraint] = field(default_factory=list)
     quote: Optional[bool] = None
     tags: List[str] = field(default_factory=list)
     _extra: Dict[str, Any] = field(default_factory=dict)
 
 
+@dataclass
+class Contract(dbtClassMixin, Replaceable):
+    enforced: bool = False
+    checksum: Optional[str] = None
+
+
 # Metrics, exposures,
 @dataclass
 class HasRelationMetadata(dbtClassMixin, Replaceable):
@@ -182,11 +259,16 @@ class MacroDependsOn(dbtClassMixin, Replaceable):
 @dataclass
 class DependsOn(MacroDependsOn):
     nodes: List[str] = field(default_factory=list)
+    public_nodes: List[str] = field(default_factory=list)
 
     def add_node(self, value: str):
         if value not in self.nodes:
             self.nodes.append(value)
 
+    def add_public_node(self, value: str):
+        if value not in self.public_nodes:
+            self.public_nodes.append(value)
+
 
 @dataclass
 class ParsedNodeMandatory(GraphNode, HasRelationMetadata, Replaceable):
@@ -207,8 +289,6 @@ class NodeInfoMixin:
 
     @property
     def node_info(self):
-        meta = getattr(self, "meta", {})
-        meta_stringified = cast_dict_to_dict_of_strings(meta)
         node_info = {
             "node_path": getattr(self, "path", None),
             "node_name": getattr(self, "name", None),
@@ -218,10 +298,15 @@ class NodeInfoMixin:
             "node_status": str(self._event_status.get("node_status")),
             "node_started_at": self._event_status.get("started_at"),
             "node_finished_at": self._event_status.get("finished_at"),
-            "meta": meta_stringified,
+            "meta": getattr(self, "meta", {}),
+            "node_relation": {
+                "database": getattr(self, "database", None),
+                "schema": getattr(self, "schema", None),
+                "alias": getattr(self, "alias", None),
+                "relation_name": getattr(self, "relation_name", None),
+            },
         }
-        node_info_msg = NodeInfo(**node_info)
-        return node_info_msg
+        return node_info
 
     def update_event_status(self, **kwargs):
         for k, v in kwargs.items():
@@ -238,6 +323,7 @@ class ParsedNode(NodeInfoMixin, ParsedNodeMandatory, SerializableType):
     description: str = field(default="")
     columns: Dict[str, ColumnInfo] = field(default_factory=dict)
     meta: Dict[str, Any] = field(default_factory=dict)
+    group: Optional[str] = None
     docs: Docs = field(default_factory=Docs)
     patch_path: Optional[str] = None
     build_path: Optional[str] = None
@@ -350,8 +436,18 @@ class ParsedNode(NodeInfoMixin, ParsedNodeMandatory, SerializableType):
             old.unrendered_config,
         )
 
+    def build_contract_checksum(self):
+        pass
+
+    def same_contract(self, old, adapter_type=None) -> bool:
+        # This would only apply to seeds
+        return True
+
     def patch(self, patch: "ParsedNodePatch"):
         """Given a ParsedNodePatch, add the new information to the node."""
+        # NOTE: Constraint patching is awkwardly done in the parse_patch function
+        # which calls this one. We need to combine the logic.
+
         # explicitly pick out the parts to update so we don't inadvertently
         # step on the model name or anything
         # Note: config should already be updated
@@ -360,20 +456,71 @@ class ParsedNode(NodeInfoMixin, ParsedNodeMandatory, SerializableType):
         self.created_at = time.time()
         self.description = patch.description
         self.columns = patch.columns
+        self.name = patch.name
 
-    def same_contents(self, old) -> bool:
+        # TODO: version, latest_version, and access are specific to ModelNodes, consider splitting out to ModelNode
+        if self.resource_type != NodeType.Model:
+            if patch.version:
+                warn_or_error(
+                    ValidationWarning(
+                        field_name="version",
+                        resource_type=self.resource_type.value,
+                        node_name=patch.name,
+                    )
+                )
+            if patch.latest_version:
+                warn_or_error(
+                    ValidationWarning(
+                        field_name="latest_version",
+                        resource_type=self.resource_type.value,
+                        node_name=patch.name,
+                    )
+                )
+        self.version = patch.version
+        self.latest_version = patch.latest_version
+
+        # This might not be the ideal place to validate the "access" field,
+        # but at this point we have the information we need to properly
+        # validate and we don't before this.
+        if patch.access:
+            if self.resource_type == NodeType.Model:
+                if AccessType.is_valid(patch.access):
+                    self.access = AccessType(patch.access)
+                else:
+                    raise InvalidAccessTypeError(
+                        unique_id=self.unique_id,
+                        field_value=patch.access,
+                    )
+            else:
+                warn_or_error(
+                    ValidationWarning(
+                        field_name="access",
+                        resource_type=self.resource_type.value,
+                        node_name=patch.name,
+                    )
+                )
+
+    def same_contents(self, old, adapter_type) -> bool:
         if old is None:
             return False
 
+        # Need to ensure that same_contract is called because it
+        # could throw an error
+        same_contract = self.same_contract(old, adapter_type)
         return (
             self.same_body(old)
             and self.same_config(old)
             and self.same_persisted_description(old)
             and self.same_fqn(old)
             and self.same_database_representation(old)
+            and same_contract
             and True
         )
 
+    @property
+    def is_public_node(self):
+        return False
+
 
 @dataclass
 class InjectedCTE(dbtClassMixin, Replaceable):
@@ -389,7 +536,7 @@ class CompiledNode(ParsedNode):
     so all ManifestNodes except SeedNode."""
 
     language: str = "sql"
-    refs: List[List[str]] = field(default_factory=list)
+    refs: List[RefArgs] = field(default_factory=list)
     sources: List[List[str]] = field(default_factory=list)
     metrics: List[List[str]] = field(default_factory=list)
     depends_on: DependsOn = field(default_factory=DependsOn)
@@ -399,6 +546,7 @@ class CompiledNode(ParsedNode):
     extra_ctes_injected: bool = False
     extra_ctes: List[InjectedCTE] = field(default_factory=list)
     _pre_injected_sql: Optional[str] = None
+    contract: Contract = field(default_factory=Contract)
 
     @property
     def empty(self):
@@ -409,8 +557,10 @@ class CompiledNode(ParsedNode):
         do if extra_ctes were an OrderedDict
         """
         for cte in self.extra_ctes:
+            # Because it's possible that multiple threads are compiling the
+            # node at the same time, we don't want to overwrite already compiled
+            # sql in the extra_ctes with empty sql.
             if cte.id == cte_id:
-                cte.sql = sql
                 break
         else:
             self.extra_ctes.append(InjectedCTE(id=cte_id, sql=sql))
@@ -433,6 +583,10 @@ class CompiledNode(ParsedNode):
     def depends_on_nodes(self):
         return self.depends_on.nodes
 
+    @property
+    def depends_on_public_nodes(self):
+        return self.depends_on.public_nodes
+
     @property
     def depends_on_macros(self):
         return self.depends_on.macros
@@ -457,6 +611,166 @@ class HookNode(CompiledNode):
 @dataclass
 class ModelNode(CompiledNode):
     resource_type: NodeType = field(metadata={"restrict": [NodeType.Model]})
+    access: AccessType = AccessType.Protected
+    constraints: List[ModelLevelConstraint] = field(default_factory=list)
+    version: Optional[NodeVersion] = None
+    latest_version: Optional[NodeVersion] = None
+
+    @property
+    def is_latest_version(self) -> bool:
+        return self.version is not None and self.version == self.latest_version
+
+    @property
+    def search_name(self):
+        if self.version is None:
+            return self.name
+        else:
+            return f"{self.name}.v{self.version}"
+
+    @property
+    def materialization_enforces_constraints(self) -> bool:
+        return self.config.materialized in ["table", "incremental"]
+
+    def build_contract_checksum(self):
+        # We don't need to construct the checksum if the model does not
+        # have contract enforced, because it won't be used.
+        # This needs to be executed after contract config is set
+        if self.contract.enforced is True:
+            contract_state = ""
+            # We need to sort the columns so that order doesn't matter
+            # columns is a str: ColumnInfo dictionary
+            sorted_columns = sorted(self.columns.values(), key=lambda col: col.name)
+            for column in sorted_columns:
+                contract_state += f"|{column.name}"
+                contract_state += str(column.data_type)
+                contract_state += str(column.constraints)
+            if self.materialization_enforces_constraints:
+                contract_state += self.config.materialized
+                contract_state += str(self.constraints)
+            data = contract_state.encode("utf-8")
+            self.contract.checksum = hashlib.new("sha256", data).hexdigest()
+
+    def same_contract(self, old, adapter_type=None) -> bool:
+        # If the contract wasn't previously enforced:
+        if old.contract.enforced is False and self.contract.enforced is False:
+            # No change -- same_contract: True
+            return True
+        if old.contract.enforced is False and self.contract.enforced is True:
+            # Now it's enforced. This is a change, but not a breaking change -- same_contract: False
+            return False
+
+        # Otherwise: The contract was previously enforced, and we need to check for changes.
+        # Happy path: The contract is still being enforced, and the checksums are identical.
+        if self.contract.enforced is True and self.contract.checksum == old.contract.checksum:
+            # No change -- same_contract: True
+            return True
+
+        # Otherwise: There has been a change.
+        # We need to determine if it is a **breaking** change.
+        # These are the categories of breaking changes:
+        contract_enforced_disabled: bool = False
+        columns_removed: List[str] = []
+        column_type_changes: List[Tuple[str, str, str]] = []
+        enforced_column_constraint_removed: List[Tuple[str, str]] = []  # column, constraint_type
+        enforced_model_constraint_removed: List[
+            Tuple[str, List[str]]
+        ] = []  # constraint_type, columns
+        materialization_changed: List[str] = []
+
+        if old.contract.enforced is True and self.contract.enforced is False:
+            # Breaking change: the contract was previously enforced, and it no longer is
+            contract_enforced_disabled = True
+
+        # TODO: this avoid the circular imports but isn't ideal
+        from dbt.adapters.factory import get_adapter_constraint_support
+        from dbt.adapters.base import ConstraintSupport
+
+        constraint_support = get_adapter_constraint_support(adapter_type)
+        column_constraints_exist = False
+
+        # Next, compare each column from the previous contract (old.columns)
+        for old_key, old_value in sorted(old.columns.items()):
+            # Has this column been removed?
+            if old_key not in self.columns.keys():
+                columns_removed.append(old_value.name)
+            # Has this column's data type changed?
+            elif old_value.data_type != self.columns[old_key].data_type:
+                column_type_changes.append(
+                    (
+                        str(old_value.name),
+                        str(old_value.data_type),
+                        str(self.columns[old_key].data_type),
+                    )
+                )
+
+            # track if there are any column level constraints for the materialization check late
+            if old_value.constraints:
+                column_constraints_exist = True
+
+            # Have enforced columns level constraints changed?
+            # Constraints are only enforced for table and incremental materializations.
+            # We only really care if the old node was one of those materializations for breaking changes
+            if (
+                old_key in self.columns.keys()
+                and old_value.constraints != self.columns[old_key].constraints
+                and old.materialization_enforces_constraints
+            ):
+
+                for old_constraint in old_value.constraints:
+                    if (
+                        old_constraint not in self.columns[old_key].constraints
+                        and constraint_support[old_constraint.type] == ConstraintSupport.ENFORCED
+                    ):
+                        enforced_column_constraint_removed.append(
+                            (old_key, str(old_constraint.type))
+                        )
+
+        # Now compare the model level constraints
+        if old.constraints != self.constraints and old.materialization_enforces_constraints:
+            for old_constraint in old.constraints:
+                if (
+                    old_constraint not in self.constraints
+                    and constraint_support[old_constraint.type] == ConstraintSupport.ENFORCED
+                ):
+                    enforced_model_constraint_removed.append(
+                        (str(old_constraint.type), old_constraint.columns)
+                    )
+
+        # Check for relevant materialization changes.
+        if (
+            old.materialization_enforces_constraints
+            and not self.materialization_enforces_constraints
+            and (old.constraints or column_constraints_exist)
+        ):
+            materialization_changed = [old.config.materialized, self.config.materialized]
+
+        # If a column has been added, it will be missing in the old.columns, and present in self.columns
+        # That's a change (caught by the different checksums), but not a breaking change
+
+        # Did we find any changes that we consider breaking? If so, that's an error
+        if (
+            contract_enforced_disabled
+            or columns_removed
+            or column_type_changes
+            or enforced_model_constraint_removed
+            or enforced_column_constraint_removed
+            or materialization_changed
+        ):
+            raise (
+                ContractBreakingChangeError(
+                    contract_enforced_disabled=contract_enforced_disabled,
+                    columns_removed=columns_removed,
+                    column_type_changes=column_type_changes,
+                    enforced_column_constraint_removed=enforced_column_constraint_removed,
+                    enforced_model_constraint_removed=enforced_model_constraint_removed,
+                    materialization_changed=materialization_changed,
+                    node=self,
+                )
+            )
+
+        # Otherwise, though we didn't find any *breaking* changes, the contract has still changed -- same_contract: False
+        else:
+            return False
 
 
 # TODO: rm?
@@ -482,6 +796,7 @@ class SeedNode(ParsedNode):  # No SQLDefaults!
     # seeds need the root_path because the contents are not loaded initially
     # and we need the root_path to load the seed later
     root_path: Optional[str] = None
+    depends_on: MacroDependsOn = field(default_factory=MacroDependsOn)
 
     def same_seeds(self, other: "SeedNode") -> bool:
         # for seeds, we check the hashes. If the hashes are different types,
@@ -523,6 +838,39 @@ class SeedNode(ParsedNode):  # No SQLDefaults!
         """Seeds are never empty"""
         return False
 
+    def _disallow_implicit_dependencies(self):
+        """Disallow seeds to take implicit upstream dependencies via pre/post hooks"""
+        # Seeds are root nodes in the DAG. They cannot depend on other nodes.
+        # However, it's possible to define pre- and post-hooks on seeds, and for those
+        # hooks to include {{ ref(...) }}. This worked in previous versions, but it
+        # was never officially documented or supported behavior. Let's raise an explicit error,
+        # which will surface during parsing if the user has written code such that we attempt
+        # to capture & record a ref/source/metric call on the SeedNode.
+        # For more details: https://github.com/dbt-labs/dbt-core/issues/6806
+        hooks = [f'- pre_hook: "{hook.sql}"' for hook in self.config.pre_hook] + [
+            f'- post_hook: "{hook.sql}"' for hook in self.config.post_hook
+        ]
+        hook_list = "\n".join(hooks)
+        message = f"""
+Seeds cannot depend on other nodes. dbt detected a seed with a pre- or post-hook
+that calls 'ref', 'source', or 'metric', either directly or indirectly via other macros.
+
+Error raised for '{self.unique_id}', which has these hooks defined: \n{hook_list}
+        """
+        raise ParsingError(message)
+
+    @property
+    def refs(self):
+        self._disallow_implicit_dependencies()
+
+    @property
+    def sources(self):
+        self._disallow_implicit_dependencies()
+
+    @property
+    def metrics(self):
+        self._disallow_implicit_dependencies()
+
     def same_body(self, other) -> bool:
         return self.same_seeds(other)
 
@@ -531,9 +879,13 @@ class SeedNode(ParsedNode):  # No SQLDefaults!
         return []
 
     @property
-    def depends_on_macros(self):
+    def depends_on_public_nodes(self):
         return []
 
+    @property
+    def depends_on_macros(self) -> List[str]:
+        return self.depends_on.macros
+
     @property
     def extra_ctes(self):
         return []
@@ -557,7 +909,7 @@ class TestShouldStoreFailures:
     def should_store_failures(self):
         if self.config.store_failures:
             return self.config.store_failures
-        return flags.STORE_FAILURES
+        return get_flags().STORE_FAILURES
 
     @property
     def is_relational(self):
@@ -608,8 +960,9 @@ class GenericTestNode(TestShouldStoreFailures, CompiledNode, HasTestMetadata):
     # Was not able to make mypy happy and keep the code working. We need to
     # refactor the various configs.
     config: TestConfig = field(default_factory=TestConfig)  # type: ignore
+    attached_node: Optional[str] = None
 
-    def same_contents(self, other) -> bool:
+    def same_contents(self, other, adapter_type: Optional[str]) -> bool:
         if other is None:
             return False
 
@@ -823,7 +1176,7 @@ class SourceDefinition(NodeInfoMixin, ParsedSourceMandatory):
         if old is None:
             return True
 
-        # config changes are changes (because the only config is "enabled", and
+        # config changes are changes (because the only config is "enforced", and
         # enabling a source is a change!)
         # changing the database/schema/identifier is a change
         # messing around with external stuff is a change (uh, right?)
@@ -863,6 +1216,10 @@ class SourceDefinition(NodeInfoMixin, ParsedSourceMandatory):
     def depends_on_nodes(self):
         return []
 
+    @property
+    def depends_on_public_nodes(self):
+        return []
+
     @property
     def depends_on(self):
         return DependsOn(macros=[], nodes=[])
@@ -892,7 +1249,7 @@ class SourceDefinition(NodeInfoMixin, ParsedSourceMandatory):
 @dataclass
 class Exposure(GraphNode):
     type: ExposureType
-    owner: ExposureOwner
+    owner: Owner
     resource_type: NodeType = field(metadata={"restrict": [NodeType.Exposure]})
     description: str = ""
     label: Optional[str] = None
@@ -903,7 +1260,7 @@ class Exposure(GraphNode):
     unrendered_config: Dict[str, Any] = field(default_factory=dict)
     url: Optional[str] = None
     depends_on: DependsOn = field(default_factory=DependsOn)
-    refs: List[List[str]] = field(default_factory=list)
+    refs: List[RefArgs] = field(default_factory=list)
     sources: List[List[str]] = field(default_factory=list)
     metrics: List[List[str]] = field(default_factory=list)
     created_at: float = field(default_factory=lambda: time.time())
@@ -912,6 +1269,10 @@ class Exposure(GraphNode):
     def depends_on_nodes(self):
         return self.depends_on.nodes
 
+    @property
+    def depends_on_public_nodes(self):
+        return self.depends_on.public_nodes
+
     @property
     def search_name(self):
         return self.name
@@ -995,14 +1356,19 @@ class Metric(GraphNode):
     unrendered_config: Dict[str, Any] = field(default_factory=dict)
     sources: List[List[str]] = field(default_factory=list)
     depends_on: DependsOn = field(default_factory=DependsOn)
-    refs: List[List[str]] = field(default_factory=list)
+    refs: List[RefArgs] = field(default_factory=list)
     metrics: List[List[str]] = field(default_factory=list)
     created_at: float = field(default_factory=lambda: time.time())
+    group: Optional[str] = None
 
     @property
     def depends_on_nodes(self):
         return self.depends_on.nodes
 
+    @property
+    def depends_on_public_nodes(self):
+        return self.depends_on.public_nodes
+
     @property
     def search_name(self):
         return self.name
@@ -1065,6 +1431,18 @@ class Metric(GraphNode):
         )
 
 
+# ====================================
+# Group node
+# ====================================
+
+
+@dataclass
+class Group(BaseNode):
+    name: str
+    owner: Owner
+    resource_type: NodeType = field(metadata={"restrict": [NodeType.Group]})
+
+
 # ====================================
 # Patches
 # ====================================
@@ -1085,6 +1463,9 @@ class ParsedPatch(HasYamlMetadata, Replaceable):
 @dataclass
 class ParsedNodePatch(ParsedPatch):
     columns: Dict[str, ColumnInfo]
+    access: Optional[str]
+    version: Optional[NodeVersion]
+    latest_version: Optional[NodeVersion]
 
 
 @dataclass
@@ -1097,6 +1478,46 @@ class ParsedMacroPatch(ParsedPatch):
 # ====================================
 
 
+class ManifestOrPublicNode(Protocol):
+    name: str
+    package_name: str
+    unique_id: str
+    version: Optional[NodeVersion]
+    latest_version: Optional[NodeVersion]
+    relation_name: str
+    database: Optional[str]
+    schema: Optional[str]
+    identifier: Optional[str]
+
+    @property
+    def is_latest_version(self):
+        pass
+
+    @property
+    def resource_type(self):
+        pass
+
+    @property
+    def access(self):
+        pass
+
+    @property
+    def search_name(self):
+        pass
+
+    @property
+    def is_public_node(self):
+        pass
+
+    @property
+    def is_versioned(self):
+        pass
+
+    @property
+    def alias(self):
+        pass
+
+
 # ManifestNode without SeedNode, which doesn't have the
 # SQL related attributes
 ManifestSQLNode = Union[
@@ -1133,6 +1554,7 @@ Resource = Union[
     GraphMemberNode,
     Documentation,
     Macro,
+    Group,
 ]
 
 TestNode = Union[

core/dbt/contracts/graph/unparsed.py

@@ -6,12 +6,12 @@ from dbt.contracts.util import (
     AdditionalPropertiesMixin,
     Mergeable,
     Replaceable,
-    rename_metric_attr,
 )
+from dbt.contracts.graph.manifest_upgrade import rename_metric_attr
 
 # trigger the PathEncoder
 import dbt.helper_types  # noqa:F401
-from dbt.exceptions import CompilationError, ParsingError
+from dbt.exceptions import CompilationError, ParsingError, DbtInternalError
 
 from dbt.dataclass_schema import dbtClassMixin, StrEnum, ExtensibleDbtClassMixin, ValidationError
 
@@ -88,11 +88,12 @@ class Docs(dbtClassMixin, Replaceable):
 
 
 @dataclass
-class HasDocs(AdditionalPropertiesMixin, ExtensibleDbtClassMixin, Replaceable):
+class HasColumnProps(AdditionalPropertiesMixin, ExtensibleDbtClassMixin, Replaceable):
     name: str
     description: str = ""
     meta: Dict[str, Any] = field(default_factory=dict)
     data_type: Optional[str] = None
+    constraints: List[Dict[str, Any]] = field(default_factory=list)
     docs: Docs = field(default_factory=Docs)
     _extra: Dict[str, Any] = field(default_factory=dict)
 
@@ -101,27 +102,23 @@ TestDef = Union[Dict[str, Any], str]
 
 
 @dataclass
-class HasTests(HasDocs):
-    tests: Optional[List[TestDef]] = None
-
-    def __post_init__(self):
-        if self.tests is None:
-            self.tests = []
+class HasColumnAndTestProps(HasColumnProps):
+    tests: List[TestDef] = field(default_factory=list)
 
 
 @dataclass
-class UnparsedColumn(HasTests):
+class UnparsedColumn(HasColumnAndTestProps):
     quote: Optional[bool] = None
     tags: List[str] = field(default_factory=list)
 
 
 @dataclass
 class HasColumnDocs(dbtClassMixin, Replaceable):
-    columns: Sequence[HasDocs] = field(default_factory=list)
+    columns: Sequence[HasColumnProps] = field(default_factory=list)
 
 
 @dataclass
-class HasColumnTests(HasColumnDocs):
+class HasColumnTests(dbtClassMixin, Replaceable):
     columns: Sequence[UnparsedColumn] = field(default_factory=list)
 
 
@@ -141,14 +138,121 @@ class HasConfig:
     config: Dict[str, Any] = field(default_factory=dict)
 
 
-@dataclass
-class UnparsedAnalysisUpdate(HasConfig, HasColumnDocs, HasDocs, HasYamlMetadata):
-    pass
+NodeVersion = Union[str, float]
 
 
 @dataclass
-class UnparsedNodeUpdate(HasConfig, HasColumnTests, HasTests, HasYamlMetadata):
+class UnparsedVersion(dbtClassMixin):
+    v: NodeVersion
+    defined_in: Optional[str] = None
+    description: str = ""
+    access: Optional[str] = None
+    config: Dict[str, Any] = field(default_factory=dict)
+    constraints: List[Dict[str, Any]] = field(default_factory=list)
+    docs: Docs = field(default_factory=Docs)
+    tests: Optional[List[TestDef]] = None
+    columns: Sequence[Union[dbt.helper_types.IncludeExclude, UnparsedColumn]] = field(
+        default_factory=list
+    )
+
+    def __lt__(self, other):
+        try:
+            v = type(other.v)(self.v)
+            return v < other.v
+        except ValueError:
+            try:
+                other_v = type(self.v)(other.v)
+                return self.v < other_v
+            except ValueError:
+                return str(self.v) < str(other.v)
+
+    @property
+    def include_exclude(self) -> dbt.helper_types.IncludeExclude:
+        return self._include_exclude
+
+    @property
+    def unparsed_columns(self) -> List:
+        return self._unparsed_columns
+
+    @property
+    def formatted_v(self) -> str:
+        return f"v{self.v}"
+
+    def __post_init__(self):
+        has_include_exclude = False
+        self._include_exclude = dbt.helper_types.IncludeExclude(include="*")
+        self._unparsed_columns = []
+        for column in self.columns:
+            if isinstance(column, dbt.helper_types.IncludeExclude):
+                if not has_include_exclude:
+                    self._include_exclude = column
+                    has_include_exclude = True
+                else:
+                    raise ParsingError("version can have at most one include/exclude element")
+            else:
+                self._unparsed_columns.append(column)
+
+
+@dataclass
+class UnparsedAnalysisUpdate(HasConfig, HasColumnDocs, HasColumnProps, HasYamlMetadata):
+    access: Optional[str] = None
+
+
+@dataclass
+class UnparsedNodeUpdate(HasConfig, HasColumnTests, HasColumnAndTestProps, HasYamlMetadata):
     quote_columns: Optional[bool] = None
+    access: Optional[str] = None
+
+
+@dataclass
+class UnparsedModelUpdate(UnparsedNodeUpdate):
+    quote_columns: Optional[bool] = None
+    access: Optional[str] = None
+    latest_version: Optional[NodeVersion] = None
+    versions: Sequence[UnparsedVersion] = field(default_factory=list)
+
+    def __post_init__(self):
+        if self.latest_version:
+            version_values = [version.v for version in self.versions]
+            if self.latest_version not in version_values:
+                raise ParsingError(
+                    f"latest_version: {self.latest_version} is not one of model '{self.name}' versions: {version_values} "
+                )
+
+        seen_versions: set[str] = set()
+        for version in self.versions:
+            if str(version.v) in seen_versions:
+                raise ParsingError(
+                    f"Found duplicate version: '{version.v}' in versions list of model '{self.name}'"
+                )
+            seen_versions.add(str(version.v))
+
+        self._version_map = {version.v: version for version in self.versions}
+
+    def get_columns_for_version(self, version: NodeVersion) -> List[UnparsedColumn]:
+        if version not in self._version_map:
+            raise DbtInternalError(
+                f"get_columns_for_version called for version '{version}' not in version map"
+            )
+
+        version_columns = []
+        unparsed_version = self._version_map[version]
+        for base_column in self.columns:
+            if unparsed_version.include_exclude.includes(base_column.name):
+                version_columns.append(base_column)
+
+        for column in unparsed_version.unparsed_columns:
+            version_columns.append(column)
+
+        return version_columns
+
+    def get_tests_for_version(self, version: NodeVersion) -> List[TestDef]:
+        if version not in self._version_map:
+            raise DbtInternalError(
+                f"get_tests_for_version called for version '{version}' not in version map"
+            )
+        unparsed_version = self._version_map[version]
+        return unparsed_version.tests if unparsed_version.tests is not None else self.tests
 
 
 @dataclass
@@ -159,7 +263,7 @@ class MacroArgument(dbtClassMixin):
 
 
 @dataclass
-class UnparsedMacroUpdate(HasConfig, HasDocs, HasYamlMetadata):
+class UnparsedMacroUpdate(HasConfig, HasColumnProps, HasYamlMetadata):
     arguments: List[MacroArgument] = field(default_factory=list)
 
 
@@ -246,7 +350,7 @@ class Quoting(dbtClassMixin, Mergeable):
 
 
 @dataclass
-class UnparsedSourceTableDefinition(HasColumnTests, HasTests):
+class UnparsedSourceTableDefinition(HasColumnTests, HasColumnAndTestProps):
     config: Dict[str, Any] = field(default_factory=dict)
     loaded_at_field: Optional[str] = None
     identifier: Optional[str] = None
@@ -424,8 +528,8 @@ class MaturityType(StrEnum):
 
 
 @dataclass
-class ExposureOwner(dbtClassMixin, Replaceable):
-    email: str
+class Owner(AdditionalPropertiesAllowed, Replaceable):
+    email: Optional[str] = None
     name: Optional[str] = None
 
 
@@ -433,7 +537,7 @@ class ExposureOwner(dbtClassMixin, Replaceable):
 class UnparsedExposure(dbtClassMixin, Replaceable):
     name: str
     type: ExposureType
-    owner: ExposureOwner
+    owner: Owner
     description: str = ""
     label: Optional[str] = None
     maturity: Optional[MaturityType] = None
@@ -451,6 +555,9 @@ class UnparsedExposure(dbtClassMixin, Replaceable):
             if not (re.match(r"[\w-]+$", data["name"])):
                 deprecations.warn("exposure-name", exposure=data["name"])
 
+        if data["owner"].get("name") is None and data["owner"].get("email") is None:
+            raise ValidationError("Exposure owner must have at least one of 'name' or 'email'.")
+
 
 @dataclass
 class MetricFilter(dbtClassMixin, Replaceable):
@@ -533,3 +640,15 @@ class UnparsedMetric(dbtClassMixin, Replaceable):
 
         if data.get("model") is not None and data.get("calculation_method") == "derived":
             raise ValidationError("Derived metrics cannot have a 'model' property")
+
+
+@dataclass
+class UnparsedGroup(dbtClassMixin, Replaceable):
+    name: str
+    owner: Owner
+
+    @classmethod
+    def validate(cls, data):
+        super(UnparsedGroup, cls).validate(data)
+        if data["owner"].get("name") is None and data["owner"].get("email") is None:
+            raise ValidationError("Group owner must have at least one of 'name' or 'email'.")

core/dbt/contracts/project.py

@@ -184,8 +184,8 @@ BANNED_PROJECT_NAMES = {
 @dataclass
 class Project(HyphenatedDbtClassMixin, Replaceable):
     name: Identifier
-    version: Union[SemverString, float]
-    config_version: int
+    config_version: Optional[int] = 2
+    version: Optional[Union[SemverString, float]] = None
     project_root: Optional[str] = None
     source_paths: Optional[List[str]] = None
     model_paths: Optional[List[str]] = None
@@ -243,21 +243,26 @@ class Project(HyphenatedDbtClassMixin, Replaceable):
 
 @dataclass
 class UserConfig(ExtensibleDbtClassMixin, Replaceable, UserConfigContract):
-    send_anonymous_usage_stats: bool = DEFAULT_SEND_ANONYMOUS_USAGE_STATS
-    use_colors: Optional[bool] = None
+    cache_selected_only: Optional[bool] = None
+    debug: Optional[bool] = None
+    fail_fast: Optional[bool] = None
+    indirect_selection: Optional[str] = None
+    log_format: Optional[str] = None
+    log_format_file: Optional[str] = None
+    log_level: Optional[str] = None
+    log_level_file: Optional[str] = None
     partial_parse: Optional[bool] = None
+    populate_cache: Optional[bool] = None
     printer_width: Optional[int] = None
-    write_json: Optional[bool] = None
+    send_anonymous_usage_stats: bool = DEFAULT_SEND_ANONYMOUS_USAGE_STATS
+    static_parser: Optional[bool] = None
+    use_colors: Optional[bool] = None
+    use_colors_file: Optional[bool] = None
+    use_experimental_parser: Optional[bool] = None
+    version_check: Optional[bool] = None
     warn_error: Optional[bool] = None
     warn_error_options: Optional[Dict[str, Union[str, List[str]]]] = None
-    log_format: Optional[str] = None
-    debug: Optional[bool] = None
-    version_check: Optional[bool] = None
-    fail_fast: Optional[bool] = None
-    use_experimental_parser: Optional[bool] = None
-    static_parser: Optional[bool] = None
-    indirect_selection: Optional[str] = None
-    cache_selected_only: Optional[bool] = None
+    write_json: Optional[bool] = None
 
 
 @dataclass