wait for successful connection before setting up db

* Specify macro_namespace of global_project dispatch macros

* Dispatch get_custom_alias, too

* Add integration tests

* Add changelog entry

.bumpversion.cfg

@@ -1,23 +1,27 @@
 [bumpversion]
-current_version = 0.20.0rc1
+current_version = 0.21.0b1
 parse = (?P<major>\d+)
 	\.(?P<minor>\d+)
 	\.(?P<patch>\d+)
-	((?P<prerelease>[a-z]+)(?P<num>\d+))?
+	((?P<prekind>a|b|rc)
+	(?P<pre>\d+)  # pre-release version num
+	)?
 serialize = 
-	{major}.{minor}.{patch}{prerelease}{num}
+	{major}.{minor}.{patch}{prekind}{pre}
 	{major}.{minor}.{patch}
 commit = False
 tag = False
 
-[bumpversion:part:prerelease]
+[bumpversion:part:prekind]
 first_value = a
+optional_value = final
 values = 
 	a
 	b
 	rc
+	final
 
-[bumpversion:part:num]
+[bumpversion:part:pre]
 first_value = 1
 
 [bumpversion:file:setup.py]

.circleci/config.yml

@@ -1,123 +0,0 @@
-version: 2.1
-jobs:
-  unit:
-    docker: &test_only
-      - image: fishtownanalytics/test-container:12
-        environment:
-          DBT_INVOCATION_ENV: circle
-          DOCKER_TEST_DATABASE_HOST: "database"
-          TOX_PARALLEL_NO_SPINNER: 1
-    steps:
-      - checkout
-      - run: tox -p -e py36,py37,py38
-  lint:
-    docker: *test_only
-    steps:
-      - checkout
-      - run: tox -e mypy,flake8 -- -v
-  build-wheels:
-    docker: *test_only
-    steps:
-      - checkout
-      - run:
-          name: Build wheels
-          command: |
-            python3.8 -m venv "${PYTHON_ENV}"
-            export PYTHON_BIN="${PYTHON_ENV}/bin/python"
-            $PYTHON_BIN -m pip install -U pip setuptools
-            $PYTHON_BIN -m pip install -r requirements.txt
-            $PYTHON_BIN -m pip install -r dev-requirements.txt
-            /bin/bash ./scripts/build-wheels.sh
-            $PYTHON_BIN ./scripts/collect-dbt-contexts.py > ./dist/context_metadata.json
-            $PYTHON_BIN ./scripts/collect-artifact-schema.py > ./dist/artifact_schemas.json
-          environment:
-            PYTHON_ENV: /home/tox/build_venv/
-      - store_artifacts:
-          path: ./dist
-          destination: dist
-  integration-postgres:
-    docker:
-      - image: fishtownanalytics/test-container:12
-        environment:
-          DBT_INVOCATION_ENV: circle
-          DOCKER_TEST_DATABASE_HOST: "database"
-          TOX_PARALLEL_NO_SPINNER: 1
-      - image: postgres
-        name: database
-        environment:
-          POSTGRES_USER: "root"
-          POSTGRES_PASSWORD: "password"
-          POSTGRES_DB: "dbt"
-    steps:
-      - checkout
-      - run:
-          name: Setup postgres
-          command: bash test/setup_db.sh
-          environment:
-            PGHOST: database
-            PGUSER: root
-            PGPASSWORD: password
-            PGDATABASE: postgres
-      - run:
-          name: Postgres integration tests
-          command: tox -p -e py36-postgres,py38-postgres -- -v -n4
-          no_output_timeout: 30m
-      - store_artifacts:
-          path: ./logs
-  integration-snowflake:
-    docker: *test_only
-    steps:
-      - checkout
-      - run:
-          name: Snowflake integration tests
-          command: tox -p -e py36-snowflake,py38-snowflake -- -v -n4
-          no_output_timeout: 30m
-      - store_artifacts:
-          path: ./logs
-  integration-redshift:
-    docker: *test_only
-    steps:
-      - checkout
-      - run:
-          name: Redshift integration tests
-          command: tox -p -e py36-redshift,py38-redshift -- -v -n4
-          no_output_timeout: 30m
-      - store_artifacts:
-          path: ./logs
-  integration-bigquery:
-    docker: *test_only
-    steps:
-      - checkout
-      - run:
-          name: Bigquery integration test
-          command: tox -p -e py36-bigquery,py38-bigquery -- -v -n4
-          no_output_timeout: 30m
-      - store_artifacts:
-          path: ./logs
-
-workflows:
-  version: 2
-  test-everything:
-    jobs:
-      - lint
-      - unit
-      - integration-postgres:
-          requires:
-            - unit
-      - integration-redshift:
-          requires:
-            - unit
-      - integration-bigquery:
-          requires:
-            - unit
-      - integration-snowflake:
-          requires:
-            - unit
-      - build-wheels:
-          requires:
-            - lint
-            - unit
-            - integration-postgres
-            - integration-redshift
-            - integration-bigquery
-            - integration-snowflake

.github/ISSUE_TEMPLATE/beta-minor-version-release.md

@@ -0,0 +1,27 @@
+---
+name: Beta minor version release
+about: Creates a tracking checklist of items for a Beta minor version release
+title: "[Tracking] v#.##.#B# release "
+labels: 'release'
+assignees: ''
+
+---
+
+### Release Core
+- [ ] [Engineering] Follow [dbt-release workflow](https://www.notion.so/dbtlabs/Releasing-b97c5ea9a02949e79e81db3566bbc8ef#03ff37da697d4d8ba63d24fae1bfa817) 
+- [ ] [Engineering] Verify new release branch is created in the repo
+- [ ] [Product] Finalize migration guide (next.docs.getdbt.com)
+
+### Release Cloud
+- [ ] [Engineering] Create a platform issue to update dbt Cloud and verify it is completed. [Example issue](https://github.com/dbt-labs/dbt-cloud/issues/3481)
+- [ ] [Engineering] Determine if schemas have changed. If so, generate new schemas and push to schemas.getdbt.com
+
+### Announce
+- [ ] [Product] Announce in dbt Slack
+
+### Post-release
+- [ ] [Engineering] [Bump plugin versions](https://www.notion.so/dbtlabs/Releasing-b97c5ea9a02949e79e81db3566bbc8ef#f01854e8da3641179fbcbe505bdf515c) (dbt-spark + dbt-presto), add compatibility as needed
+   - [ ]  [Spark](https://github.com/dbt-labs/dbt-spark) 
+   - [ ]  [Presto](https://github.com/dbt-labs/dbt-presto)
+- [ ] [Engineering] Create a platform issue to update dbt-spark versions to dbt Cloud. [Example issue](https://github.com/dbt-labs/dbt-cloud/issues/3481)
+- [ ] [Engineering] Create an epic for the RC release

.github/ISSUE_TEMPLATE/final-minor-version-release.md

@@ -0,0 +1,28 @@
+---
+name: Final minor version release
+about: Creates a tracking checklist of items for a final minor version release
+title: "[Tracking] v#.##.# final release "
+labels: 'release'
+assignees: ''
+
+---
+
+### Release Core
+- [ ] [Engineering] Verify all necessary changes exist on the release branch
+- [ ] [Engineering] Follow [dbt-release workflow](https://www.notion.so/dbtlabs/Releasing-b97c5ea9a02949e79e81db3566bbc8ef#03ff37da697d4d8ba63d24fae1bfa817) 
+- [ ] [Product] Merge `next` into `current` for docs.getdbt.com
+
+### Release Cloud
+- [ ] [Engineering] Create a platform issue to update dbt Cloud and verify it is completed. [Example issue](https://github.com/dbt-labs/dbt-cloud/issues/3481)
+- [ ] [Engineering] Determine if schemas have changed. If so, generate new schemas and push to schemas.getdbt.com
+
+### Announce
+- [ ] [Product] Update discourse
+- [ ] [Product] Announce in dbt Slack
+
+### Post-release
+- [ ] [Engineering] [Bump plugin versions](https://www.notion.so/dbtlabs/Releasing-b97c5ea9a02949e79e81db3566bbc8ef#f01854e8da3641179fbcbe505bdf515c) (dbt-spark + dbt-presto), add compatibility as needed
+   - [ ]  [Spark](https://github.com/dbt-labs/dbt-spark) 
+   - [ ]  [Presto](https://github.com/dbt-labs/dbt-presto)
+- [ ] [Engineering] Create a platform issue to update dbt-spark versions to dbt Cloud. [Example issue](https://github.com/dbt-labs/dbt-cloud/issues/3481)
+- [ ] [Product] Release new version of dbt-utils with new dbt version compatibility. If there are breaking changes requiring a minor version, plan upgrades of other packages that depend on dbt-utils.

.github/ISSUE_TEMPLATE/minor-version-release.md

@@ -1,29 +0,0 @@
----
-name: Minor version release
-about: Creates a tracking checklist of items for a minor version release
-title: "[Tracking] v#.##.# release "
-labels: ''
-assignees: ''
-
----
-
-### Release Core
-- [ ] [Engineering] dbt-release workflow 
-- [ ] [Engineering] Create new protected `x.latest` branch 
-- [ ] [Product] Finalize migration guide (next.docs.getdbt.com)
-
-### Release Cloud
-- [ ] [Engineering] Create a platform issue to update dbt Cloud and verify it is completed
-- [ ] [Engineering] Determine if schemas have changed. If so, generate new schemas and push to schemas.getdbt.com
-
-### Announce
-- [ ] [Product] Publish discourse
-- [ ] [Product] Announce in dbt Slack
-
-### Post-release
-- [ ] [Engineering] [Bump plugin versions](https://www.notion.so/fishtownanalytics/Releasing-b97c5ea9a02949e79e81db3566bbc8ef#59571f5bc1a040d9a8fd096e23d2c7db) (dbt-spark + dbt-presto), add compatibility as needed
-   - [ ]  Spark 
-   - [ ]  Presto
-- [ ] [Engineering] Create a platform issue to update dbt-spark versions to dbt Cloud 
-- [ ] [Product] Release new version of dbt-utils with new dbt version compatibility. If there are breaking changes requiring a minor version, plan upgrades of other packages that depend on dbt-utils.
-- [ ] [Engineering] If this isn't a final release, create an epic for the next release

.github/ISSUE_TEMPLATE/rc-minor-version-release copy.md

@@ -0,0 +1,29 @@
+---
+name: RC minor version release
+about: Creates a tracking checklist of items for a RC minor version release
+title: "[Tracking] v#.##.#RC# release "
+labels: 'release'
+assignees: ''
+
+---
+
+### Release Core
+- [ ] [Engineering] Verify all necessary changes exist on the release branch
+- [ ] [Engineering] Follow [dbt-release workflow](https://www.notion.so/dbtlabs/Releasing-b97c5ea9a02949e79e81db3566bbc8ef#03ff37da697d4d8ba63d24fae1bfa817) 
+- [ ] [Product] Update migration guide (next.docs.getdbt.com)
+
+### Release Cloud
+- [ ] [Engineering] Create a platform issue to update dbt Cloud and verify it is completed. [Example issue](https://github.com/dbt-labs/dbt-cloud/issues/3481)
+- [ ] [Engineering] Determine if schemas have changed. If so, generate new schemas and push to schemas.getdbt.com
+
+### Announce
+- [ ] [Product] Publish discourse
+- [ ] [Product] Announce in dbt Slack
+
+### Post-release
+- [ ] [Engineering] [Bump plugin versions](https://www.notion.so/dbtlabs/Releasing-b97c5ea9a02949e79e81db3566bbc8ef#f01854e8da3641179fbcbe505bdf515c) (dbt-spark + dbt-presto), add compatibility as needed
+   - [ ]  [Spark](https://github.com/dbt-labs/dbt-spark) 
+   - [ ]  [Presto](https://github.com/dbt-labs/dbt-presto)
+- [ ] [Engineering] Create a platform issue to update dbt-spark versions to dbt Cloud. [Example issue](https://github.com/dbt-labs/dbt-cloud/issues/3481)
+- [ ] [Product] Release new version of dbt-utils with new dbt version compatibility. If there are breaking changes requiring a minor version, plan upgrades of other packages that depend on dbt-utils.
+- [ ] [Engineering] Create an epic for the final release

.github/actions/setup-postgres-linux/action.yml

@@ -0,0 +1,10 @@
+name: "Set up postgres (linux)"
+description: "Set up postgres service on linux vm for dbt integration tests"
+runs:
+  using: "composite"
+  steps:
+    - shell: bash
+      run: |
+        sudo systemctl start postgresql.service
+        pg_isready
+        sudo -u postgres bash ${{ github.action_path }}/setup_db.sh

.github/actions/setup-postgres-linux/setup_db.sh

@@ -0,0 +1 @@
+../../../test/setup_db.sh

.github/actions/setup-postgres-macos/action.yml

@@ -0,0 +1,24 @@
+name: "Set up postgres (macos)"
+description: "Set up postgres service on macos vm for dbt integration tests"
+runs:
+  using: "composite"
+  steps:
+    - shell: bash
+      run: |
+        brew services start postgresql
+        echo "Check PostgreSQL service is running"
+        i=10
+        COMMAND='pg_isready'
+        while [ $i -gt -1 ]; do
+            if [ $i == 0 ]; then
+                echo "PostgreSQL service not ready, all attempts exhausted"
+                exit 1
+            fi
+            echo "Check PostgreSQL service status"
+            eval $COMMAND && break
+            echo "PostgreSQL service not ready, wait 10 more sec, attempts left: $i"
+            sleep 10
+            ((i--))
+        done
+        createuser -s postgres
+        bash ${{ github.action_path }}/setup_db.sh

.github/actions/setup-postgres-macos/setup_db.sh

@@ -0,0 +1 @@
+../../../test/setup_db.sh

.github/actions/setup-postgres-windows/action.yml

@@ -0,0 +1,12 @@
+name: "Set up postgres (windows)"
+description: "Set up postgres service on windows vm for dbt integration tests"
+runs:
+  using: "composite"
+  steps:
+    - shell: pwsh
+      run: |
+        $pgService = Get-Service -Name postgresql*
+        Set-Service -InputObject $pgService -Status running -StartupType automatic
+        Start-Process -FilePath "$env:PGBIN\pg_isready" -Wait -PassThru
+        $env:Path += ";$env:PGBIN"
+        bash ${{ github.action_path }}/setup_db.sh

.github/actions/setup-postgres-windows/setup_db.sh

@@ -0,0 +1 @@
+../../../test/setup_db.sh

.github/pull_request_template.md

@@ -9,14 +9,13 @@ resolves #
     resolves #1234
 -->
 
-
 ### Description
 
 <!--- Describe the Pull Request here -->
 
-
 ### Checklist
- - [ ] I have signed the [CLA](https://docs.getdbt.com/docs/contributor-license-agreements)
- - [ ] I have run this code in development and it appears to resolve the stated issue
- - [ ] This PR includes tests, or tests are not required/relevant for this PR
- - [ ] I have updated the `CHANGELOG.md` and added information about my change to the "dbt next" section.
+
+- [ ] I have signed the [CLA](https://docs.getdbt.com/docs/contributor-license-agreements)
+- [ ] I have run this code in development and it appears to resolve the stated issue
+- [ ] This PR includes tests, or tests are not required/relevant for this PR
+- [ ] I have updated the `CHANGELOG.md` and added information about my change to the "dbt next" section.

.github/scripts/integration-test-matrix.js

@@ -0,0 +1,95 @@
+module.exports = ({ context }) => {
+  const defaultPythonVersion = "3.8";
+  const supportedPythonVersions = ["3.6", "3.7", "3.8", "3.9"];
+  const supportedAdapters = ["snowflake", "postgres", "bigquery", "redshift"];
+
+  // if PR, generate matrix based on files changed and PR labels
+  if (context.eventName.includes("pull_request")) {
+    // `changes` is a list of adapter names that have related
+    // file changes in the PR
+    // ex: ['postgres', 'snowflake']
+    const changes = JSON.parse(process.env.CHANGES);
+    const labels = context.payload.pull_request.labels.map(({ name }) => name);
+    console.log("labels", labels);
+    console.log("changes", changes);
+    const testAllLabel = labels.includes("test all");
+    const include = [];
+
+    for (const adapter of supportedAdapters) {
+      if (
+        changes.includes(adapter) ||
+        testAllLabel ||
+        labels.includes(`test ${adapter}`)
+      ) {
+        for (const pythonVersion of supportedPythonVersions) {
+          if (
+            pythonVersion === defaultPythonVersion ||
+            labels.includes(`test python${pythonVersion}`) ||
+            testAllLabel
+          ) {
+            // always run tests on ubuntu by default
+            include.push({
+              os: "ubuntu-latest",
+              adapter,
+              "python-version": pythonVersion,
+            });
+
+            if (labels.includes("test windows") || testAllLabel) {
+              include.push({
+                os: "windows-latest",
+                adapter,
+                "python-version": pythonVersion,
+              });
+            }
+
+            if (labels.includes("test macos") || testAllLabel) {
+              include.push({
+                os: "macos-latest",
+                adapter,
+                "python-version": pythonVersion,
+              });
+            }
+          }
+        }
+      }
+    }
+
+    console.log("matrix", { include });
+
+    return {
+      include,
+    };
+  }
+  // if not PR, generate matrix of python version, adapter, and operating
+  // system to run integration tests on
+
+  const include = [];
+  // run for all adapters and python versions on ubuntu
+  for (const adapter of supportedAdapters) {
+    for (const pythonVersion of supportedPythonVersions) {
+      include.push({
+        os: 'ubuntu-latest',
+        adapter: adapter,
+        "python-version": pythonVersion,
+      });
+    }
+  }
+
+  // additionally include runs for all adapters, on macos and windows,
+  // but only for the default python version
+  for (const adapter of supportedAdapters) {
+    for (const operatingSystem of ["windows-latest", "macos-latest"]) {
+      include.push({
+        os: operatingSystem,
+        adapter: adapter,
+        "python-version": defaultPythonVersion,
+      });
+    }
+  }
+
+  console.log("matrix", { include });
+
+  return {
+    include,
+  };
+};

.github/workflows/integration.yml

@@ -0,0 +1,266 @@
+# **what?**
+# This workflow runs all integration tests for supported OS
+# and python versions and core adapters. If triggered by PR,
+# the workflow will only run tests for adapters related
+# to code changes. Use the `test all` and `test ${adapter}`
+# label to run all or additional tests. Use `ok to test`
+# label to mark PRs from forked repositories that are safe
+# to run integration tests for. Requires secrets to run
+# against different warehouses.
+
+# **why?**
+# This checks the functionality of dbt from a user's perspective
+# and attempts to catch functional regressions.
+
+# **when?**
+# This workflow will run on every push to a protected branch
+# and when manually triggered. It will also run for all PRs, including
+# PRs from forks. The workflow will be skipped until there is a label
+# to mark the PR as safe to run.
+
+name: Adapter Integration Tests
+
+on:
+  # pushes to release branches
+  push:
+    branches:
+      - "main"
+      - "develop"
+      - "*.latest"
+      - "releases/*"
+  # all PRs, important to note that `pull_request_target` workflows
+  # will run in the context of the target branch of a PR
+  pull_request_target:
+  # manual tigger
+  workflow_dispatch:
+
+# explicitly turn off permissions for `GITHUB_TOKEN`
+permissions: read-all
+
+# will cancel previous workflows triggered by the same event and for the same ref for PRs or same SHA otherwise
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event_name }}-${{ contains(github.event_name, 'pull_request') && github.event.pull_request.head.ref || github.sha }}
+  cancel-in-progress: true
+
+# sets default shell to bash, for all operating systems
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  # generate test metadata about what files changed and the testing matrix to use
+  test-metadata:
+    # run if not a PR from a forked repository or has a label to mark as safe to test
+    if: >-
+      github.event_name != 'pull_request_target' ||
+      github.event.pull_request.head.repo.full_name == github.repository ||
+      contains(github.event.pull_request.labels.*.name, 'ok to test')
+
+    runs-on: ubuntu-latest
+
+    outputs:
+      matrix: ${{ steps.generate-matrix.outputs.result }}
+
+    steps:
+      - name: Check out the repository (non-PR)
+        if: github.event_name != 'pull_request_target'
+        uses: actions/checkout@v2
+        with:
+          persist-credentials: false
+
+      - name: Check out the repository (PR)
+        if: github.event_name == 'pull_request_target'
+        uses: actions/checkout@v2
+        with:
+          persist-credentials: false
+          ref: ${{ github.event.pull_request.head.sha }}
+
+      - name: Check if relevant files changed
+        # https://github.com/marketplace/actions/paths-changes-filter
+        # For each filter, it sets output variable named by the filter to the text:
+        #  'true' - if any of changed files matches any of filter rules
+        #  'false' - if none of changed files matches any of filter rules
+        # also, returns:
+        #  `changes` - JSON array with names of all filters matching any of the changed files
+        uses: dorny/paths-filter@v2
+        id: get-changes
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          filters: |
+            postgres:
+              - 'core/**'
+              - 'plugins/postgres/**'
+              - 'dev-requirements.txt'
+            snowflake:
+              - 'core/**'
+              - 'plugins/snowflake/**'
+            bigquery:
+              - 'core/**'
+              - 'plugins/bigquery/**'
+            redshift:
+              - 'core/**'
+              - 'plugins/redshift/**'
+              - 'plugins/postgres/**'
+
+      - name: Generate integration test matrix
+        id: generate-matrix
+        uses: actions/github-script@v4
+        env:
+          CHANGES: ${{ steps.get-changes.outputs.changes }}
+        with:
+          script: |
+            const script = require('./.github/scripts/integration-test-matrix.js')
+            const matrix = script({ context })
+            console.log(matrix)
+            return matrix
+
+  test:
+    name: ${{ matrix.adapter }} / python ${{ matrix.python-version }} / ${{ matrix.os }}
+
+    # run if not a PR from a forked repository or has a label to mark as safe to test
+    # also checks that the matrix generated is not empty
+    if: >-
+      needs.test-metadata.outputs.matrix &&
+      fromJSON( needs.test-metadata.outputs.matrix ).include[0] &&
+      (
+        github.event_name != 'pull_request_target' ||
+        github.event.pull_request.head.repo.full_name == github.repository ||
+        contains(github.event.pull_request.labels.*.name, 'ok to test')
+      )
+
+    runs-on: ${{ matrix.os }}
+
+    needs: test-metadata
+
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJSON(needs.test-metadata.outputs.matrix) }}
+
+    env:
+      TOXENV: integration-${{ matrix.adapter }}
+      PYTEST_ADDOPTS: "-v --color=yes -n4 --csv integration_results.csv"
+      DBT_INVOCATION_ENV: github-actions
+
+    steps:
+      - name: Check out the repository
+        if: github.event_name != 'pull_request_target'
+        uses: actions/checkout@v2
+        with:
+          persist-credentials: false
+
+      # explicity checkout the branch for the PR,
+      # this is necessary for the `pull_request_target` event
+      - name: Check out the repository (PR)
+        if: github.event_name == 'pull_request_target'
+        uses: actions/checkout@v2
+        with:
+          persist-credentials: false
+          ref: ${{ github.event.pull_request.head.sha }}
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Set up postgres (linux)
+        if: |
+          matrix.adapter == 'postgres' &&
+          runner.os == 'Linux'
+        uses: ./.github/actions/setup-postgres-linux
+
+      - name: Set up postgres (macos)
+        if: |
+          matrix.adapter == 'postgres' &&
+          runner.os == 'macOS'
+        uses: ./.github/actions/setup-postgres-macos
+
+      - name: Set up postgres (windows)
+        if: |
+          matrix.adapter == 'postgres' &&
+          runner.os == 'Windows'
+        uses: ./.github/actions/setup-postgres-windows
+
+      - name: Install python dependencies
+        run: |
+          pip install --upgrade pip
+          pip install tox
+          pip --version
+          tox --version
+
+      - name: Run tox (postgres)
+        if: matrix.adapter == 'postgres'
+        run: tox
+
+      - name: Run tox (redshift)
+        if: matrix.adapter == 'redshift'
+        env:
+          REDSHIFT_TEST_DBNAME: ${{ secrets.REDSHIFT_TEST_DBNAME }}
+          REDSHIFT_TEST_PASS: ${{ secrets.REDSHIFT_TEST_PASS }}
+          REDSHIFT_TEST_USER: ${{ secrets.REDSHIFT_TEST_USER }}
+          REDSHIFT_TEST_PORT: ${{ secrets.REDSHIFT_TEST_PORT }}
+          REDSHIFT_TEST_HOST: ${{ secrets.REDSHIFT_TEST_HOST }}
+        run: tox
+
+      - name: Run tox (snowflake)
+        if: matrix.adapter == 'snowflake'
+        env:
+          SNOWFLAKE_TEST_ACCOUNT: ${{ secrets.SNOWFLAKE_TEST_ACCOUNT }}
+          SNOWFLAKE_TEST_PASSWORD: ${{ secrets.SNOWFLAKE_TEST_PASSWORD }}
+          SNOWFLAKE_TEST_USER: ${{ secrets.SNOWFLAKE_TEST_USER }}
+          SNOWFLAKE_TEST_WAREHOUSE: ${{ secrets.SNOWFLAKE_TEST_WAREHOUSE }}
+          SNOWFLAKE_TEST_OAUTH_REFRESH_TOKEN: ${{ secrets.SNOWFLAKE_TEST_OAUTH_REFRESH_TOKEN }}
+          SNOWFLAKE_TEST_OAUTH_CLIENT_ID: ${{ secrets.SNOWFLAKE_TEST_OAUTH_CLIENT_ID }}
+          SNOWFLAKE_TEST_OAUTH_CLIENT_SECRET: ${{ secrets.SNOWFLAKE_TEST_OAUTH_CLIENT_SECRET }}
+          SNOWFLAKE_TEST_ALT_DATABASE: ${{ secrets.SNOWFLAKE_TEST_ALT_DATABASE }}
+          SNOWFLAKE_TEST_ALT_WAREHOUSE: ${{ secrets.SNOWFLAKE_TEST_ALT_WAREHOUSE }}
+          SNOWFLAKE_TEST_DATABASE: ${{ secrets.SNOWFLAKE_TEST_DATABASE }}
+          SNOWFLAKE_TEST_QUOTED_DATABASE: ${{ secrets.SNOWFLAKE_TEST_QUOTED_DATABASE }}
+          SNOWFLAKE_TEST_ROLE: ${{ secrets.SNOWFLAKE_TEST_ROLE }}
+        run: tox
+
+      - name: Run tox (bigquery)
+        if: matrix.adapter == 'bigquery'
+        env:
+          BIGQUERY_TEST_SERVICE_ACCOUNT_JSON: ${{ secrets.BIGQUERY_TEST_SERVICE_ACCOUNT_JSON }}
+          BIGQUERY_TEST_ALT_DATABASE: ${{ secrets.BIGQUERY_TEST_ALT_DATABASE }}
+        run: tox
+
+      - uses: actions/upload-artifact@v2
+        if: always()
+        with:
+          name: logs
+          path: ./logs
+
+      - name: Get current date
+        if: always()
+        id: date
+        run: echo "::set-output name=date::$(date +'%Y-%m-%dT%H_%M_%S')" #no colons allowed for artifacts
+
+      - uses: actions/upload-artifact@v2
+        if: always()
+        with:
+          name: integration_results_${{ matrix.python-version }}_${{ matrix.os }}_${{ matrix.adapter }}-${{ steps.date.outputs.date }}.csv
+          path: integration_results.csv
+
+  require-label-comment:
+    runs-on: ubuntu-latest
+
+    needs: test
+
+    permissions:
+      pull-requests: write
+
+    steps:
+      - name: Needs permission PR comment
+        if: >-
+          needs.test.result == 'skipped' &&
+          github.event_name == 'pull_request_target' &&
+          github.event.pull_request.head.repo.full_name != github.repository
+        uses: unsplash/comment-on-pr@master
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          msg: |
+            "You do not have permissions to run integration tests, @dbt-labs/core "\
+            "needs to label this PR with `ok to test` in order to run integration tests!"
+          check_for_duplicate_msg: true

.github/workflows/main.yml

@@ -0,0 +1,206 @@
+# **what?**
+# Runs code quality checks, unit tests, and verifies python build on
+# all code commited to the repository. This workflow should not
+# require any secrets since it runs for PRs from forked repos.
+# By default, secrets are not passed to workflows running from
+# a forked repo.
+
+# **why?**
+# Ensure code for dbt meets a certain quality standard.
+
+# **when?**
+# This will run for all PRs, when code is pushed to a release
+# branch, and when manually triggered.
+
+name: Tests and Code Checks
+
+on:
+  push:
+    branches:
+      - "main"
+      - "develop"
+      - "*.latest"
+      - "releases/*"
+  pull_request:
+  workflow_dispatch:
+
+permissions: read-all
+
+# will cancel previous workflows triggered by the same event and for the same ref for PRs or same SHA otherwise
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event_name }}-${{ contains(github.event_name, 'pull_request') && github.event.pull_request.head.ref || github.sha }}
+  cancel-in-progress: true
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  code-quality:
+    name: ${{ matrix.toxenv }}
+
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        toxenv: [flake8, mypy]
+
+    env:
+      TOXENV: ${{ matrix.toxenv }}
+      PYTEST_ADDOPTS: "-v --color=yes"
+
+    steps:
+      - name: Check out the repository
+        uses: actions/checkout@v2
+        with:
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@v2
+
+      - name: Install python dependencies
+        run: |
+          pip install --upgrade pip
+          pip install tox
+          pip --version
+          tox --version
+
+      - name: Run tox
+        run: tox
+
+  unit:
+    name: unit test / python ${{ matrix.python-version }}
+
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: [3.6, 3.7, 3.8] # TODO: support unit testing for python 3.9 (https://github.com/dbt-labs/dbt/issues/3689)
+
+    env:
+      TOXENV: "unit"
+      PYTEST_ADDOPTS: "-v --color=yes --csv unit_results.csv"
+
+    steps:
+      - name: Check out the repository
+        uses: actions/checkout@v2
+        with:
+          persist-credentials: false
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install python dependencies
+        run: |
+          pip install --upgrade pip
+          pip install tox
+          pip --version
+          tox --version
+
+      - name: Run tox
+        run: tox
+
+      - name: Get current date
+        if: always()
+        id: date
+        run: echo "::set-output name=date::$(date +'%Y-%m-%dT%H_%M_%S')" #no colons allowed for artifacts
+
+      - uses: actions/upload-artifact@v2
+        if: always()
+        with:
+          name: unit_results_${{ matrix.python-version }}-${{ steps.date.outputs.date }}.csv
+          path: unit_results.csv
+
+  build:
+    name: build packages
+
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Check out the repository
+        uses: actions/checkout@v2
+        with:
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+
+      - name: Install python dependencies
+        run: |
+          pip install --upgrade pip
+          pip install --upgrade setuptools wheel twine check-wheel-contents
+          pip --version
+
+      - name: Build distributions
+        run: ./scripts/build-dist.sh
+
+      - name: Show distributions
+        run: ls -lh dist/
+
+      - name: Check distribution descriptions
+        run: |
+          twine check dist/*
+
+      - name: Check wheel contents
+        run: |
+          check-wheel-contents dist/*.whl --ignore W007,W008
+
+      - uses: actions/upload-artifact@v2
+        with:
+          name: dist
+          path: dist/
+
+  test-build:
+    name: verify packages / python ${{ matrix.python-version }} / ${{ matrix.os }}
+
+    needs: build
+
+    runs-on: ${{ matrix.os }}
+
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+        python-version: [3.6, 3.7, 3.8, 3.9]
+
+    steps:
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install python dependencies
+        run: |
+          pip install --upgrade pip
+          pip install --upgrade wheel
+          pip --version
+
+      - uses: actions/download-artifact@v2
+        with:
+          name: dist
+          path: dist/
+
+      - name: Show distributions
+        run: ls -lh dist/
+
+      - name: Install wheel distributions
+        run: |
+          find ./dist/*.whl -maxdepth 1 -type f | xargs pip install --force-reinstall --find-links=dist/
+
+      - name: Check wheel distributions
+        run: |
+          dbt --version
+
+      - name: Install source distributions
+        run: |
+          find ./dist/*.gz -maxdepth 1 -type f | xargs pip install --force-reinstall --find-links=dist/
+
+      - name: Check source distributions
+        run: |
+          dbt --version

.github/workflows/performance.yml

@@ -0,0 +1,174 @@
+name: Performance Regression Tests
+# Schedule triggers
+on:
+  # runs twice a day at 10:05am and 10:05pm
+  schedule:
+    - cron: "5 10,22 * * *"
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+jobs:
+  # checks fmt of runner code
+  # purposefully not a dependency of any other job
+  # will block merging, but not prevent developing
+  fmt:
+    name: Cargo fmt
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: stable
+          override: true
+      - run: rustup component add rustfmt
+      - uses: actions-rs/cargo@v1
+        with:
+          command: fmt
+          args: --manifest-path performance/runner/Cargo.toml --all -- --check
+
+  # runs any tests associated with the runner
+  # these tests make sure the runner logic is correct
+  test-runner:
+    name: Test Runner
+    runs-on: ubuntu-latest
+    env:
+      # turns errors into warnings
+      RUSTFLAGS: "-D warnings"
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: stable
+          override: true
+      - uses: actions-rs/cargo@v1
+        with:
+          command: test
+          args: --manifest-path performance/runner/Cargo.toml
+
+  # build an optimized binary to be used as the runner in later steps
+  build-runner:
+    needs: [test-runner]
+    name: Build Runner
+    runs-on: ubuntu-latest
+    env:
+      RUSTFLAGS: "-D warnings"
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: stable
+          override: true
+      - uses: actions-rs/cargo@v1
+        with:
+          command: build
+          args: --release --manifest-path performance/runner/Cargo.toml
+      - uses: actions/upload-artifact@v2
+        with:
+          name: runner
+          path: performance/runner/target/release/runner
+
+  # run the performance measurements on the current or default branch
+  measure-dev:
+    needs: [build-runner]
+    name: Measure Dev Branch
+    runs-on: ubuntu-latest
+    steps:
+      - name: checkout dev
+        uses: actions/checkout@v2
+      - name: Setup Python
+        uses: actions/setup-python@v2.2.2
+        with:
+          python-version: "3.8"
+      - name: install dbt
+        run: pip install -r dev-requirements.txt -r editable-requirements.txt
+      - name: install hyperfine
+        run: wget https://github.com/sharkdp/hyperfine/releases/download/v1.11.0/hyperfine_1.11.0_amd64.deb && sudo dpkg -i hyperfine_1.11.0_amd64.deb
+      - uses: actions/download-artifact@v2
+        with:
+          name: runner
+      - name: change permissions
+        run: chmod +x ./runner
+      - name: run
+        run: ./runner measure -b dev -p ${{ github.workspace }}/performance/projects/
+      - uses: actions/upload-artifact@v2
+        with:
+          name: dev-results
+          path: performance/results/
+
+  # run the performance measurements on the release branch which we use
+  # as a performance baseline. This part takes by far the longest, so
+  # we do everything we can first so the job fails fast.
+  # -----
+  # we need to checkout dbt twice in this job: once for the baseline dbt
+  # version, and once to get the latest regression testing projects,
+  # metrics, and runner code from the develop or current branch so that
+  # the calculations match for both versions of dbt we are comparing.
+  measure-baseline:
+    needs: [build-runner]
+    name: Measure Baseline Branch
+    runs-on: ubuntu-latest
+    steps:
+      - name: checkout latest
+        uses: actions/checkout@v2
+        with:
+          ref: "0.20.latest"
+      - name: Setup Python
+        uses: actions/setup-python@v2.2.2
+        with:
+          python-version: "3.8"
+      - name: move repo up a level
+        run: mkdir ${{ github.workspace }}/../baseline/ && cp -r ${{ github.workspace }} ${{ github.workspace }}/../baseline
+      - name: "[debug] ls new dbt location"
+        run: ls ${{ github.workspace }}/../baseline/dbt/
+      # installation creates egg-links so we have to preserve source
+      - name: install dbt from new location
+        run: cd ${{ github.workspace }}/../baseline/dbt/ && pip install -r dev-requirements.txt -r editable-requirements.txt
+      # checkout the current branch to get all the target projects
+      # this deletes the old checked out code which is why we had to copy before
+      - name: checkout dev
+        uses: actions/checkout@v2
+      - name: install hyperfine
+        run: wget https://github.com/sharkdp/hyperfine/releases/download/v1.11.0/hyperfine_1.11.0_amd64.deb && sudo dpkg -i hyperfine_1.11.0_amd64.deb
+      - uses: actions/download-artifact@v2
+        with:
+          name: runner
+      - name: change permissions
+        run: chmod +x ./runner
+      - name: run runner
+        run: ./runner measure -b baseline -p ${{ github.workspace }}/performance/projects/
+      - uses: actions/upload-artifact@v2
+        with:
+          name: baseline-results
+          path: performance/results/
+
+  # detect regressions on the output generated from measuring
+  # the two branches. Exits with non-zero code if a regression is detected.
+  calculate-regressions:
+    needs: [measure-dev, measure-baseline]
+    name: Compare Results
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/download-artifact@v2
+        with:
+          name: dev-results
+      - uses: actions/download-artifact@v2
+        with:
+          name: baseline-results
+      - name: "[debug] ls result files"
+        run: ls
+      - uses: actions/download-artifact@v2
+        with:
+          name: runner
+      - name: change permissions
+        run: chmod +x ./runner
+      - name: run calculation
+        run: ./runner calculate -r ./
+        # always attempt to upload the results even if there were regressions found
+      - uses: actions/upload-artifact@v2
+        if: ${{ always() }}
+        with:
+          name: final-calculations
+          path: ./final_calculations.json

.github/workflows/tests.yml

@@ -1,178 +0,0 @@
-# This is a workflow to run our unit and integration tests for windows and mac
-
-name: dbt Tests
-
-# Triggers
-on:
-  # Triggers the workflow on push or pull request events and also adds a manual trigger
-  push:
-    branches: 
-      - 'develop' 
-      - '*.latest' 
-      - 'releases/*'
-  pull_request_target:
-    branches: 
-      - 'develop' 
-      - '*.latest' 
-      - 'pr/*'
-      - 'releases/*'
-  # Allows you to run this workflow manually from the Actions tab
-  workflow_dispatch:
-
-jobs:
-  Linting:
-    runs-on: ubuntu-latest #no need to run on every OS
-    steps:
-    - uses: actions/checkout@v2
-    - name: Setup Python
-      uses: actions/setup-python@v2.2.2
-      with:
-        python-version: '3.8'
-        architecture: 'x64'
-
-    - name: 'Install dependencies'
-      run: python -m pip install --upgrade pip && pip install tox
-      
-    - name: 'Linting'
-      run: tox -e mypy,flake8 -- -v
-      
-  UnitTest:
-    strategy:
-      matrix:
-        os: [windows-latest, ubuntu-latest, macos-latest]
-    runs-on: ${{ matrix.os }}
-    steps:
-    - uses: actions/checkout@v2
-    - name: Setup Python
-      uses: actions/setup-python@v2.2.2
-      with:
-        python-version: '3.8'
-        architecture: 'x64'
-
-    - name: 'Install dependencies'
-      run: python -m pip install --upgrade pip && pip install tox
-
-    - name: 'Run unit tests'
-      run: python -m tox -e py -- -v
-      
-  PostgresIntegrationTest:
-    runs-on: 'windows-latest' #TODO: Add Mac support
-    environment: 'Postgres'
-    needs: UnitTest
-    steps:
-    - uses: actions/checkout@v2
-    - name: 'Install postgresql and set up database'
-      shell: pwsh
-      run: |
-          $serviceName = Get-Service -Name postgresql*
-          Set-Service -InputObject $serviceName -StartupType Automatic
-          Start-Service -InputObject $serviceName
-          & $env:PGBIN\createdb.exe -U postgres dbt
-          & $env:PGBIN\psql.exe  -U postgres -c "CREATE ROLE root WITH PASSWORD '$env:ROOT_PASSWORD';"
-          & $env:PGBIN\psql.exe  -U postgres -c "ALTER ROLE root WITH LOGIN;"
-          & $env:PGBIN\psql.exe  -U postgres -c "GRANT CREATE, CONNECT ON DATABASE dbt TO root WITH GRANT OPTION;"
-          & $env:PGBIN\psql.exe  -U postgres -c "CREATE ROLE noaccess WITH PASSWORD '$env:NOACCESS_PASSWORD' NOSUPERUSER;"
-          & $env:PGBIN\psql.exe  -U postgres -c "ALTER ROLE noaccess WITH LOGIN;"
-          & $env:PGBIN\psql.exe  -U postgres -c "GRANT CONNECT ON DATABASE dbt TO noaccess;"
-      env:
-          ROOT_PASSWORD: ${{ secrets.ROOT_PASSWORD }}
-          NOACCESS_PASSWORD: ${{ secrets.NOACCESS_PASSWORD }}
-      
-    - name: Setup Python
-      uses: actions/setup-python@v2.2.2
-      with:
-        python-version: '3.7'
-        architecture: 'x64'
-        
-    - name: 'Install dependencies'
-      run: python -m pip install --upgrade pip && pip install tox
-      
-    - name: 'Run integration tests'
-      run: python -m tox -e py-postgres -- -v -n4
-      
-  # These three are all similar except secure environment variables, which MUST be passed along to their tasks,
-  # but there's probably a better way to do this!
-  SnowflakeIntegrationTest:
-    strategy:
-      matrix:
-        os: [windows-latest, macos-latest]
-    runs-on: ${{ matrix.os }}
-    environment: 'Snowflake'
-    needs: UnitTest
-    steps:
-      - uses: actions/checkout@v2
-      - name: Setup Python
-        uses: actions/setup-python@v2.2.2
-        with:
-          python-version: '3.7'
-          architecture: 'x64'
-
-      - name: 'Install dependencies'
-        run: python -m pip install --upgrade pip && pip install tox
-
-      - name: 'Run integration tests'
-        run: python -m tox -e py-snowflake -- -v -n4
-        env:
-          SNOWFLAKE_TEST_ACCOUNT: ${{ secrets.SNOWFLAKE_TEST_ACCOUNT }}
-          SNOWFLAKE_TEST_PASSWORD: ${{ secrets.SNOWFLAKE_TEST_PASSWORD }}
-          SNOWFLAKE_TEST_USER: ${{ secrets.SNOWFLAKE_TEST_USER }}
-          SNOWFLAKE_TEST_WAREHOUSE: ${{ secrets.SNOWFLAKE_TEST_WAREHOUSE }}
-          SNOWFLAKE_TEST_OAUTH_REFRESH_TOKEN: ${{ secrets.SNOWFLAKE_TEST_OAUTH_REFRESH_TOKEN }}
-          SNOWFLAKE_TEST_OAUTH_CLIENT_ID: ${{ secrets.SNOWFLAKE_TEST_OAUTH_CLIENT_ID }}
-          SNOWFLAKE_TEST_OAUTH_CLIENT_SECRET: ${{ secrets.SNOWFLAKE_TEST_OAUTH_CLIENT_SECRET }}
-          SNOWFLAKE_TEST_ALT_DATABASE: ${{ secrets.SNOWFLAKE_TEST_ALT_DATABASE }}
-          SNOWFLAKE_TEST_ALT_WAREHOUSE: ${{ secrets.SNOWFLAKE_TEST_ALT_WAREHOUSE }}
-          SNOWFLAKE_TEST_DATABASE: ${{ secrets.SNOWFLAKE_TEST_DATABASE }}
-          SNOWFLAKE_TEST_QUOTED_DATABASE: ${{ secrets.SNOWFLAKE_TEST_QUOTED_DATABASE }}
-          SNOWFLAKE_TEST_ROLE: ${{ secrets.SNOWFLAKE_TEST_ROLE }}
-
-  BigQueryIntegrationTest:
-    strategy:
-      matrix:
-        os: [windows-latest, macos-latest]
-    runs-on: ${{ matrix.os }}
-    environment: 'Bigquery'
-    needs: UnitTest
-    steps:
-      - uses: actions/checkout@v2
-      - name: Setup Python
-        uses: actions/setup-python@v2.2.2
-        with:
-          python-version: '3.7'
-          architecture: 'x64'
-        
-      - name: 'Install dependencies'
-        run: python -m pip install --upgrade pip && pip install tox
-      
-      - name: 'Run integration tests'
-        run: python -m tox -e py-bigquery -- -v -n4
-        env:
-          BIGQUERY_SERVICE_ACCOUNT_JSON: ${{ secrets.BIGQUERY_SERVICE_ACCOUNT_JSON }}
-          BIGQUERY_TEST_ALT_DATABASE: ${{ secrets.BIGQUERY_TEST_ALT_DATABASE }}
-
-  RedshiftIntegrationTest:
-    strategy:
-      matrix:
-        os: [windows-latest, macos-latest]
-    runs-on: ${{ matrix.os }}
-    environment: 'Redshift'
-    needs: UnitTest
-    steps:
-      - uses: actions/checkout@v2
-      - name: Setup Python
-        uses: actions/setup-python@v2.2.2
-        with:
-          python-version: '3.7'
-          architecture: 'x64'
-
-      - name: 'Install dependencies'
-        run: python -m pip install --upgrade pip && pip install tox
-
-      - name: 'Run integration tests' 
-        run: python -m tox -e py-redshift -- -v -n4
-        env:
-          REDSHIFT_TEST_DBNAME: ${{ secrets.REDSHIFT_TEST_DBNAME }}
-          REDSHIFT_TEST_PASS: ${{ secrets.REDSHIFT_TEST_PASS }}
-          REDSHIFT_TEST_USER: ${{ secrets.REDSHIFT_TEST_USER }}
-          REDSHIFT_TEST_PORT: ${{ secrets.REDSHIFT_TEST_PORT }}
-          REDSHIFT_TEST_HOST: ${{ secrets.REDSHIFT_TEST_HOST }}

ARCHITECTURE.md

@@ -26,7 +26,7 @@ This is the docs website code. It comes from the dbt-docs repository, and is gen
 
 ## Adapters
 
-dbt uses an adapter-plugin pattern to extend support to different databases, warehouses, query engines, etc. The four core adapters that are in the main repository, contained within the [`plugins`](plugins) subdirectory, are: Postgres Redshift, Snowflake and BigQuery. Other warehouses use adapter plugins defined in separate repositories (e.g. [dbt-spark](https://github.com/fishtown-analytics/dbt-spark), [dbt-presto](https://github.com/fishtown-analytics/dbt-presto)).
+dbt uses an adapter-plugin pattern to extend support to different databases, warehouses, query engines, etc. The four core adapters that are in the main repository, contained within the [`plugins`](plugins) subdirectory, are: Postgres Redshift, Snowflake and BigQuery. Other warehouses use adapter plugins defined in separate repositories (e.g. [dbt-spark](https://github.com/dbt-labs/dbt-spark), [dbt-presto](https://github.com/dbt-labs/dbt-presto)).
 
 Each adapter is a mix of python, Jinja2, and SQL. The adapter code also makes heavy use of Jinja2 to wrap modular chunks of SQL functionality, define default implementations, and allow plugins to override it. 
 

CONTRIBUTING.md

@@ -24,7 +24,7 @@ Please note that all contributors to `dbt` must sign the [Contributor License Ag
 
 ### Defining the problem
 
-If you have an idea for a new feature or if you've discovered a bug in `dbt`, the first step is to open an issue. Please check the list of [open issues](https://github.com/fishtown-analytics/dbt/issues) before creating a new one. If you find a relevant issue, please add a comment to the open issue instead of creating a new one. There are hundreds of open issues in this repository and it can be hard to know where to look for a relevant open issue. **The `dbt` maintainers are always happy to point contributors in the right direction**, so please err on the side of documenting your idea in a new issue if you are unsure where a problem statement belongs.
+If you have an idea for a new feature or if you've discovered a bug in `dbt`, the first step is to open an issue. Please check the list of [open issues](https://github.com/dbt-labs/dbt/issues) before creating a new one. If you find a relevant issue, please add a comment to the open issue instead of creating a new one. There are hundreds of open issues in this repository and it can be hard to know where to look for a relevant open issue. **The `dbt` maintainers are always happy to point contributors in the right direction**, so please err on the side of documenting your idea in a new issue if you are unsure where a problem statement belongs.
 
 > **Note:** All community-contributed Pull Requests _must_ be associated with an open issue. If you submit a Pull Request that does not pertain to an open issue, you will be asked to create an issue describing the problem before the Pull Request can be reviewed.
 
@@ -36,7 +36,7 @@ After you open an issue, a `dbt` maintainer will follow up by commenting on your
 
 If an issue is appropriately well scoped and describes a beneficial change to the `dbt` codebase, then anyone may submit a Pull Request to implement the functionality described in the issue. See the sections below on how to do this.
 
-The `dbt` maintainers will add a `good first issue` label if an issue is suitable for a first-time contributor. This label often means that the required code change is small, limited to one database adapter, or a net-new addition that does not impact existing functionality. You can see the list of currently open issues on the [Contribute](https://github.com/fishtown-analytics/dbt/contribute) page.
+The `dbt` maintainers will add a `good first issue` label if an issue is suitable for a first-time contributor. This label often means that the required code change is small, limited to one database adapter, or a net-new addition that does not impact existing functionality. You can see the list of currently open issues on the [Contribute](https://github.com/dbt-labs/dbt/contribute) page.
 
 Here's a good workflow:
 - Comment on the open issue, expressing your interest in contributing the required code change
@@ -52,15 +52,15 @@ The `dbt` maintainers use labels to categorize open issues. Some labels indicate
 
 | tag | description |
 | --- | ----------- |
-| [triage](https://github.com/fishtown-analytics/dbt/labels/triage) | This is a new issue which has not yet been reviewed by a `dbt` maintainer. This label is removed when a maintainer reviews and responds to the issue. |
-| [bug](https://github.com/fishtown-analytics/dbt/labels/bug) | This issue represents a defect or regression in `dbt` |
-| [enhancement](https://github.com/fishtown-analytics/dbt/labels/enhancement) | This issue represents net-new functionality in `dbt` |
-| [good first issue](https://github.com/fishtown-analytics/dbt/labels/good%20first%20issue) | This issue does not require deep knowledge of the `dbt` codebase to implement. This issue is appropriate for a first-time contributor. |
-| [help wanted](https://github.com/fishtown-analytics/`dbt`/labels/help%20wanted) / [discussion](https://github.com/fishtown-analytics/dbt/labels/discussion) | Conversation around this issue in ongoing, and there isn't yet a clear path forward. Input from community members is most welcome. |
-| [duplicate](https://github.com/fishtown-analytics/dbt/issues/duplicate) | This issue is functionally identical to another open issue. The `dbt` maintainers will close this issue and encourage community members to focus conversation on the other one. |
-| [snoozed](https://github.com/fishtown-analytics/dbt/labels/snoozed) | This issue describes a good idea, but one which will probably not be addressed in a six-month time horizon. The `dbt` maintainers will revist these issues periodically and re-prioritize them accordingly. |
-| [stale](https://github.com/fishtown-analytics/dbt/labels/stale) | This is an old issue which has not recently been updated. Stale issues will periodically be closed by `dbt` maintainers, but they can be re-opened if the discussion is restarted. |
-| [wontfix](https://github.com/fishtown-analytics/dbt/labels/wontfix) | This issue does not require a code change in the `dbt` repository, or the maintainers are unwilling/unable to merge a Pull Request which implements the behavior described in the issue. |
+| [triage](https://github.com/dbt-labs/dbt/labels/triage) | This is a new issue which has not yet been reviewed by a `dbt` maintainer. This label is removed when a maintainer reviews and responds to the issue. |
+| [bug](https://github.com/dbt-labs/dbt/labels/bug) | This issue represents a defect or regression in `dbt` |
+| [enhancement](https://github.com/dbt-labs/dbt/labels/enhancement) | This issue represents net-new functionality in `dbt` |
+| [good first issue](https://github.com/dbt-labs/dbt/labels/good%20first%20issue) | This issue does not require deep knowledge of the `dbt` codebase to implement. This issue is appropriate for a first-time contributor. |
+| [help wanted](https://github.com/dbt-labs/dbt/labels/help%20wanted) / [discussion](https://github.com/dbt-labs/dbt/labels/discussion) | Conversation around this issue in ongoing, and there isn't yet a clear path forward. Input from community members is most welcome. |
+| [duplicate](https://github.com/dbt-labs/dbt/issues/duplicate) | This issue is functionally identical to another open issue. The `dbt` maintainers will close this issue and encourage community members to focus conversation on the other one. |
+| [snoozed](https://github.com/dbt-labs/dbt/labels/snoozed) | This issue describes a good idea, but one which will probably not be addressed in a six-month time horizon. The `dbt` maintainers will revist these issues periodically and re-prioritize them accordingly. |
+| [stale](https://github.com/dbt-labs/dbt/labels/stale) | This is an old issue which has not recently been updated. Stale issues will periodically be closed by `dbt` maintainers, but they can be re-opened if the discussion is restarted. |
+| [wontfix](https://github.com/dbt-labs/dbt/labels/wontfix) | This issue does not require a code change in the `dbt` repository, or the maintainers are unwilling/unable to merge a Pull Request which implements the behavior described in the issue. |
 
 #### Branching Strategy
 
@@ -78,17 +78,17 @@ You will need `git` in order to download and modify the `dbt` source code. On ma
 
 ### External contributors
 
-If you are not a member of the `fishtown-analytics` GitHub organization, you can contribute to `dbt` by forking the `dbt` repository. For a detailed overview on forking, check out the [GitHub docs on forking](https://help.github.com/en/articles/fork-a-repo). In short, you will need to:
+If you are not a member of the `dbt-labs` GitHub organization, you can contribute to `dbt` by forking the `dbt` repository. For a detailed overview on forking, check out the [GitHub docs on forking](https://help.github.com/en/articles/fork-a-repo). In short, you will need to:
 
 1. fork the `dbt` repository
 2. clone your fork locally
 3. check out a new branch for your proposed changes
 4. push changes to your fork
-5. open a pull request against `fishtown-analytics/dbt` from your forked repository
+5. open a pull request against `dbt-labs/dbt` from your forked repository
 
 ### Core contributors
 
-If you are a member of the `fishtown-analytics` GitHub organization, you will have push access to the `dbt` repo. Rather than forking `dbt` to make your changes, just clone the repository, check out a new branch, and push directly to that branch.
+If you are a member of the `dbt-labs` GitHub organization, you will have push access to the `dbt` repo. Rather than forking `dbt` to make your changes, just clone the repository, check out a new branch, and push directly to that branch.
 
 ## Setting up an environment
 
@@ -155,7 +155,7 @@ Configure your [profile](https://docs.getdbt.com/docs/configure-your-profile) as
 
 Getting the `dbt` integration tests set up in your local environment will be very helpful as you start to make changes to your local version of `dbt`. The section that follows outlines some helpful tips for setting up the test environment.
 
-Since `dbt` works with a number of different databases, you will need to supply credentials for one or more of these databases in your test environment. Most organizations don't have access to each of a BigQuery, Redshift, Snowflake, and Postgres database, so it's likely that you will be unable to run every integration test locally. Fortunately, Fishtown Analytics provides a CI environment with access to sandboxed Redshift, Snowflake, BigQuery, and Postgres databases. See the section on [_Submitting a Pull Request_](#submitting-a-pull-request) below for more information on this CI setup.
+Since `dbt` works with a number of different databases, you will need to supply credentials for one or more of these databases in your test environment. Most organizations don't have access to each of a BigQuery, Redshift, Snowflake, and Postgres database, so it's likely that you will be unable to run every integration test locally. Fortunately, dbt Labs provides a CI environment with access to sandboxed Redshift, Snowflake, BigQuery, and Postgres databases. See the section on [_Submitting a Pull Request_](#submitting-a-pull-request) below for more information on this CI setup.
 
 ### Initial setup
 
@@ -224,7 +224,7 @@ python -m pytest test/unit/test_graph.py::GraphTest::test__dependency_list
 > is a list of useful command-line options for `pytest` to use while developing.
 ## Submitting a Pull Request
 
-Fishtown Analytics provides a sandboxed Redshift, Snowflake, and BigQuery database for use in a CI environment. When pull requests are submitted to the `fishtown-analytics/dbt` repo, GitHub will trigger automated tests in CircleCI and Azure Pipelines.
+dbt Labs provides a sandboxed Redshift, Snowflake, and BigQuery database for use in a CI environment. When pull requests are submitted to the `dbt-labs/dbt` repo, GitHub will trigger automated tests in CircleCI and Azure Pipelines.
 
 A `dbt` maintainer will review your PR. They may suggest code revision for style or clarity, or request that you add unit or integration test(s). These are good things! We believe that, with a little bit of help, anyone can contribute high-quality code.
 

Dockerfile.test

@@ -1,4 +1,4 @@
-FROM ubuntu:18.04
+FROM ubuntu:20.04
 
 ENV DEBIAN_FRONTEND noninteractive
 

README.md

@@ -2,20 +2,17 @@
   <img src="https://raw.githubusercontent.com/dbt-labs/dbt/ec7dee39f793aa4f7dd3dae37282cc87664813e4/etc/dbt-logo-full.svg" alt="dbt logo" width="500"/>
 </p>
 <p align="center">
-  <a href="https://github.com/dbt-labs/dbt/actions/workflows/tests.yml?query=branch%3Adevelop">
-    <img src="https://github.com/dbt-labs/dbt/actions/workflows/tests.yml/badge.svg" alt="GitHub Actions"/>
+  <a href="https://github.com/dbt-labs/dbt/actions/workflows/main.yml">
+    <img src="https://github.com/dbt-labs/dbt/actions/workflows/main.yml/badge.svg?event=push" alt="Unit Tests Badge"/>
   </a>
-  <a href="https://circleci.com/gh/dbt-labs/dbt/tree/develop">
-    <img src="https://circleci.com/gh/dbt-labs/dbt/tree/develop.svg?style=svg"  alt="CircleCI" />
-  </a>
-  <a href="https://dev.azure.com/fishtown-analytics/dbt/_build?definitionId=1&_a=summary&repositoryFilter=1&branchFilter=789%2C789%2C789%2C789">
-    <img src="https://dev.azure.com/fishtown-analytics/dbt/_apis/build/status/fishtown-analytics.dbt?branchName=develop" alt="Azure Pipelines" />
+  <a href="https://github.com/dbt-labs/dbt/actions/workflows/integration.yml">
+    <img src="https://github.com/dbt-labs/dbt/actions/workflows/integration.yml/badge.svg?event=push" alt="Integration Tests Badge"/>
   </a>
 </p>
 
 **[dbt](https://www.getdbt.com/)** enables data analysts and engineers to transform their data using the same practices that software engineers use to build applications.
 
-![dbt architecture](https://raw.githubusercontent.com/dbt-labs/dbt/6c6649f9129d5d108aa3b0526f634cd8f3a9d1ed/etc/dbt-arch.png)
+![architecture](https://raw.githubusercontent.com/dbt-labs/dbt/6c6649f9129d5d108aa3b0526f634cd8f3a9d1ed/etc/dbt-arch.png)
 
 ## Understanding dbt
 

azure-pipelines.yml

@@ -1,154 +0,0 @@
-# Python package
-# Create and test a Python package on multiple Python versions.
-# Add steps that analyze code, save the dist with the build record, publish to a PyPI-compatible index, and more:
-# https://docs.microsoft.com/azure/devops/pipelines/languages/python
-
-trigger:
-  branches:
-    include:
-    - develop
-    - '*.latest'
-    - pr/*
-
-jobs:
-- job: UnitTest
-  pool:
-    vmImage: 'vs2017-win2016'
-  steps:
-  - task: UsePythonVersion@0
-    inputs:
-      versionSpec: '3.7'
-      architecture: 'x64'
-
-  - script: python -m pip install --upgrade pip && pip install tox
-    displayName: 'Install dependencies'
-
-  - script: python -m tox -e py -- -v
-    displayName: Run unit tests
-
-- job: PostgresIntegrationTest
-  pool:
-    vmImage: 'vs2017-win2016'
-  dependsOn: UnitTest
-
-  steps:
-  - pwsh: |
-      $serviceName = Get-Service -Name postgresql*
-      Set-Service -InputObject $serviceName -StartupType Automatic
-      Start-Service -InputObject $serviceName
-
-      & $env:PGBIN\createdb.exe -U postgres dbt
-      & $env:PGBIN\psql.exe  -U postgres -c "CREATE ROLE root WITH PASSWORD 'password';"
-      & $env:PGBIN\psql.exe  -U postgres -c "ALTER ROLE root WITH LOGIN;"
-      & $env:PGBIN\psql.exe  -U postgres -c "GRANT CREATE, CONNECT ON DATABASE dbt TO root WITH GRANT OPTION;"
-      & $env:PGBIN\psql.exe  -U postgres -c "CREATE ROLE noaccess WITH PASSWORD 'password' NOSUPERUSER;"
-      & $env:PGBIN\psql.exe  -U postgres -c "ALTER ROLE noaccess WITH LOGIN;"
-      & $env:PGBIN\psql.exe  -U postgres -c "GRANT CONNECT ON DATABASE dbt TO noaccess;"
-    displayName: Install postgresql and set up database
-
-  - task: UsePythonVersion@0
-    inputs:
-      versionSpec: '3.7'
-      architecture: 'x64'
-
-  - script: python -m pip install --upgrade pip && pip install tox
-    displayName: 'Install dependencies'
-
-  - script: python -m tox -e py-postgres -- -v -n4
-    displayName: Run integration tests
-
-# These three are all similar except secure environment variables, which MUST be passed along to their tasks,
-# but there's probably a better way to do this!
-- job: SnowflakeIntegrationTest
-  pool:
-    vmImage: 'vs2017-win2016'
-  dependsOn: UnitTest
-  condition: succeeded()
-  steps:
-  - task: UsePythonVersion@0
-    inputs:
-      versionSpec: '3.7'
-      architecture: 'x64'
-
-  - script: python -m pip install --upgrade pip && pip install tox
-    displayName: 'Install dependencies'
-
-  - script: python -m tox -e py-snowflake -- -v -n4
-    env:
-      SNOWFLAKE_TEST_ACCOUNT: $(SNOWFLAKE_TEST_ACCOUNT)
-      SNOWFLAKE_TEST_PASSWORD: $(SNOWFLAKE_TEST_PASSWORD)
-      SNOWFLAKE_TEST_USER: $(SNOWFLAKE_TEST_USER)
-      SNOWFLAKE_TEST_WAREHOUSE: $(SNOWFLAKE_TEST_WAREHOUSE)
-      SNOWFLAKE_TEST_OAUTH_REFRESH_TOKEN: $(SNOWFLAKE_TEST_OAUTH_REFRESH_TOKEN)
-      SNOWFLAKE_TEST_OAUTH_CLIENT_ID: $(SNOWFLAKE_TEST_OAUTH_CLIENT_ID)
-      SNOWFLAKE_TEST_OAUTH_CLIENT_SECRET: $(SNOWFLAKE_TEST_OAUTH_CLIENT_SECRET)
-    displayName: Run integration tests
-
-- job: BigQueryIntegrationTest
-  pool:
-    vmImage: 'vs2017-win2016'
-  dependsOn: UnitTest
-  condition: succeeded()
-  steps:
-  - task: UsePythonVersion@0
-    inputs:
-      versionSpec: '3.7'
-      architecture: 'x64'
-  - script: python -m pip install --upgrade pip && pip install tox
-    displayName: 'Install dependencies'
-  - script: python -m tox -e py-bigquery -- -v -n4
-    env:
-      BIGQUERY_SERVICE_ACCOUNT_JSON: $(BIGQUERY_SERVICE_ACCOUNT_JSON)
-    displayName: Run integration tests
-
-- job: RedshiftIntegrationTest
-  pool:
-    vmImage: 'vs2017-win2016'
-  dependsOn: UnitTest
-  condition: succeeded()
-  steps:
-  - task: UsePythonVersion@0
-    inputs:
-      versionSpec: '3.7'
-      architecture: 'x64'
-
-  - script: python -m pip install --upgrade pip && pip install tox
-    displayName: 'Install dependencies'
-
-  - script: python -m tox -e py-redshift -- -v -n4
-    env:
-      REDSHIFT_TEST_DBNAME: $(REDSHIFT_TEST_DBNAME)
-      REDSHIFT_TEST_PASS: $(REDSHIFT_TEST_PASS)
-      REDSHIFT_TEST_USER: $(REDSHIFT_TEST_USER)
-      REDSHIFT_TEST_PORT: $(REDSHIFT_TEST_PORT)
-      REDSHIFT_TEST_HOST: $(REDSHIFT_TEST_HOST)
-    displayName: Run integration tests
-
-- job: BuildWheel
-  pool:
-    vmImage: 'vs2017-win2016'
-  dependsOn:
-    - UnitTest
-    - PostgresIntegrationTest
-    - RedshiftIntegrationTest
-    - SnowflakeIntegrationTest
-    - BigQueryIntegrationTest
-  condition: succeeded()
-  steps:
-    - task: UsePythonVersion@0
-      inputs:
-        versionSpec: '3.7'
-        architecture: 'x64'
-    - script: python -m pip install --upgrade pip setuptools && python -m pip install -r requirements.txt && python -m pip install -r dev-requirements.txt
-      displayName: Install dependencies
-    - task: ShellScript@2
-      inputs:
-        scriptPath: scripts/build-wheels.sh
-    - task: CopyFiles@2
-      inputs:
-        contents: 'dist\?(*.whl|*.tar.gz)'
-        TargetFolder: '$(Build.ArtifactStagingDirectory)'
-    - task: PublishBuildArtifacts@1
-      inputs:
-        pathtoPublish: '$(Build.ArtifactStagingDirectory)'
-        artifactName: dists

converter.py

@@ -1,73 +0,0 @@
-#!/usr/bin/env python
-import json
-import yaml
-import sys
-import argparse
-from datetime import datetime, timezone
-import dbt.clients.registry as registry
-
-
-def yaml_type(fname):
-    with open(fname) as f:
-        return yaml.load(f)
-
-
-def parse_args():
-    parser = argparse.ArgumentParser()
-    parser.add_argument("--project", type=yaml_type, default="dbt_project.yml")
-    parser.add_argument("--namespace", required=True)
-    return parser.parse_args()
-
-
-def get_full_name(args):
-    return "{}/{}".format(args.namespace, args.project["name"])
-
-
-def init_project_in_packages(args, packages):
-    full_name = get_full_name(args)
-    if full_name not in packages:
-        packages[full_name] = {
-            "name": args.project["name"],
-            "namespace": args.namespace,
-            "latest": args.project["version"],
-            "assets": {},
-            "versions": {},
-        }
-    return packages[full_name]
-
-
-def add_version_to_package(args, project_json):
-    project_json["versions"][args.project["version"]] = {
-        "id": "{}/{}".format(get_full_name(args), args.project["version"]),
-        "name": args.project["name"],
-        "version": args.project["version"],
-        "description": "",
-        "published_at": datetime.now(timezone.utc).astimezone().isoformat(),
-        "packages": args.project.get("packages") or [],
-        "works_with": [],
-        "_source": {
-            "type": "github",
-            "url": "",
-            "readme": "",
-        },
-        "downloads": {
-            "tarball": "",
-            "format": "tgz",
-            "sha1": "",
-        },
-    }
-
-
-def main():
-    args = parse_args()
-    packages = registry.packages()
-    project_json = init_project_in_packages(args, packages)
-    if args.project["version"] in project_json["versions"]:
-        raise Exception("Version {} already in packages JSON"
-                        .format(args.project["version"]),
-                        file=sys.stderr)
-    add_version_to_package(args, project_json)
-    print(json.dumps(packages, indent=2))
-
-if __name__ == "__main__":
-    main()

core/dbt/adapters/base/impl.py

@@ -31,7 +31,6 @@ from dbt.contracts.graph.compiled import (
 from dbt.contracts.graph.manifest import Manifest, MacroManifest
 from dbt.contracts.graph.parsed import ParsedSeedNode
 from dbt.exceptions import warn_or_error
-from dbt.node_types import NodeType
 from dbt.logger import GLOBAL_LOGGER as logger
 from dbt.utils import filter_null_values, executor
 
@@ -310,8 +309,7 @@ class BaseAdapter(metaclass=AdapterMeta):
             self.Relation.create_from(self.config, node).without_identifier()
             for node in manifest.nodes.values()
             if (
-                node.resource_type in NodeType.executable() and
-                not node.is_ephemeral_model
+                node.is_relational and not node.is_ephemeral_model
             )
         }
 
@@ -513,7 +511,7 @@ class BaseAdapter(metaclass=AdapterMeta):
     def get_columns_in_relation(
         self, relation: BaseRelation
     ) -> List[BaseColumn]:
-        """Get a list of the columns in the given Relation."""
+        """Get a list of the columns in the given Relation. """
         raise NotImplementedException(
             '`get_columns_in_relation` is not implemented for this adapter!'
         )

core/dbt/clients/jinja_static.py

@@ -153,7 +153,7 @@ def statically_parse_adapter_dispatch(func_call, ctx, db_wrapper):
                 package_name = packages_arg.node.node.name
                 macro_name = packages_arg.node.attr
                 if (macro_name.startswith('_get') and 'namespaces' in macro_name):
-                    # noqa: https://github.com/fishtown-analytics/dbt-utils/blob/9e9407b/macros/cross_db_utils/_get_utils_namespaces.sql
+                    # noqa: https://github.com/dbt-labs/dbt-utils/blob/9e9407b/macros/cross_db_utils/_get_utils_namespaces.sql
                     var_name = f'{package_name}_dispatch_list'
                     # hard code compatibility for fivetran_utils, just a teensy bit different
                     # noqa: https://github.com/fivetran/dbt_fivetran_utils/blob/0978ba2/macros/_get_utils_namespaces.sql

core/dbt/clients/registry.py

@@ -1,10 +1,9 @@
-from functools import wraps
+import functools
 import requests
-from dbt.exceptions import RegistryException
-from dbt.utils import memoized
+from dbt.utils import memoized, _connection_exception_retry as connection_exception_retry
 from dbt.logger import GLOBAL_LOGGER as logger
+from dbt import deprecations
 import os
-import time
 
 if os.getenv('DBT_PACKAGE_HUB_URL'):
     DEFAULT_REGISTRY_BASE_URL = os.getenv('DBT_PACKAGE_HUB_URL')
@@ -19,26 +18,11 @@ def _get_url(url, registry_base_url=None):
     return '{}{}'.format(registry_base_url, url)
 
 
-def _wrap_exceptions(fn):
-    @wraps(fn)
-    def wrapper(*args, **kwargs):
-        max_attempts = 5
-        attempt = 0
-        while True:
-            attempt += 1
-            try:
-                return fn(*args, **kwargs)
-            except (requests.exceptions.ConnectionError, requests.exceptions.Timeout) as exc:
-                if attempt < max_attempts:
-                    time.sleep(1)
-                    continue
-                raise RegistryException(
-                    'Unable to connect to registry hub'
-                ) from exc
-    return wrapper
+def _get_with_retries(path, registry_base_url=None):
+    get_fn = functools.partial(_get, path, registry_base_url)
+    return connection_exception_retry(get_fn, 5)
 
 
-@_wrap_exceptions
 def _get(path, registry_base_url=None):
     url = _get_url(path, registry_base_url)
     logger.debug('Making package registry request: GET {}'.format(url))
@@ -50,22 +34,44 @@ def _get(path, registry_base_url=None):
 
 
 def index(registry_base_url=None):
-    return _get('api/v1/index.json', registry_base_url)
+    return _get_with_retries('api/v1/index.json', registry_base_url)
 
 
 index_cached = memoized(index)
 
 
 def packages(registry_base_url=None):
-    return _get('api/v1/packages.json', registry_base_url)
+    return _get_with_retries('api/v1/packages.json', registry_base_url)
 
 
 def package(name, registry_base_url=None):
-    return _get('api/v1/{}.json'.format(name), registry_base_url)
+    response = _get_with_retries('api/v1/{}.json'.format(name), registry_base_url)
+
+    # Either redirectnamespace or redirectname in the JSON response indicate a redirect
+    # redirectnamespace redirects based on package ownership
+    # redirectname redirects based on package name
+    # Both can be present at the same time, or neither. Fails gracefully to old name
+
+    if ('redirectnamespace' in response) or ('redirectname' in response):
+
+        if ('redirectnamespace' in response) and response['redirectnamespace'] is not None:
+            use_namespace = response['redirectnamespace']
+        else:
+            use_namespace = response['namespace']
+
+        if ('redirectname' in response) and response['redirectname'] is not None:
+            use_name = response['redirectname']
+        else:
+            use_name = response['name']
+
+        new_nwo = use_namespace + "/" + use_name
+        deprecations.warn('package-redirect', old_name=name, new_name=new_nwo)
+
+    return response
 
 
 def package_version(name, version, registry_base_url=None):
-    return _get('api/v1/{}/{}.json'.format(name, version), registry_base_url)
+    return _get_with_retries('api/v1/{}/{}.json'.format(name, version), registry_base_url)
 
 
 def get_available_versions(name):

core/dbt/clients/system.py

@@ -1,4 +1,5 @@
 import errno
+import functools
 import fnmatch
 import json
 import os
@@ -15,9 +16,8 @@ from typing import (
 )
 
 import dbt.exceptions
-import dbt.utils
-
 from dbt.logger import GLOBAL_LOGGER as logger
+from dbt.utils import _connection_exception_retry as connection_exception_retry
 
 if sys.platform == 'win32':
     from ctypes import WinDLL, c_bool
@@ -441,6 +441,13 @@ def run_cmd(
     return out, err
 
 
+def download_with_retries(
+    url: str, path: str, timeout: Optional[Union[float, tuple]] = None
+) -> None:
+    download_fn = functools.partial(download, url, path, timeout)
+    connection_exception_retry(download_fn, 5)
+
+
 def download(
     url: str, path: str, timeout: Optional[Union[float, tuple]] = None
 ) -> None:

core/dbt/clients/yaml_helper.py

@@ -1,5 +1,5 @@
 import dbt.exceptions
-
+from typing import Any, Dict, Optional
 import yaml
 import yaml.scanner
 
@@ -56,7 +56,7 @@ def contextualized_yaml_error(raw_contents, error):
                                      raw_error=error)
 
 
-def safe_load(contents):
+def safe_load(contents) -> Optional[Dict[str, Any]]:
     return yaml.load(contents, Loader=SafeLoader)
 
 

core/dbt/config/profile.py

@@ -84,7 +84,8 @@ def read_user_config(directory: str) -> UserConfig:
 
 # The Profile class is included in RuntimeConfig, so any attribute
 # additions must also be set where the RuntimeConfig class is created
-@dataclass
+# `init=False` is a workaround for https://bugs.python.org/issue45081
+@dataclass(init=False)
 class Profile(HasCredentials):
     profile_name: str
     target_name: str
@@ -92,6 +93,23 @@ class Profile(HasCredentials):
     threads: int
     credentials: Credentials
 
+    def __init__(
+        self,
+        profile_name: str,
+        target_name: str,
+        config: UserConfig,
+        threads: int,
+        credentials: Credentials
+    ):
+        """Explicitly defining `__init__` to work around bug in Python 3.9.7
+        https://bugs.python.org/issue45081
+        """
+        self.profile_name = profile_name
+        self.target_name = target_name
+        self.config = config
+        self.threads = threads
+        self.credentials = credentials
+
     def to_profile_info(
         self, serialize_credentials: bool = False
     ) -> Dict[str, Any]:

core/dbt/config/runtime.py

@@ -391,6 +391,10 @@ class UnsetCredentials(Credentials):
     def type(self):
         return None
 
+    @property
+    def unique_field(self):
+        return None
+
     def connection_info(self, *args, **kwargs):
         return {}
 

core/dbt/context/context_config.py

@@ -120,11 +120,12 @@ class BaseContextConfigGenerator(Generic[T]):
 
     def calculate_node_config(
         self,
-        config_calls: List[Dict[str, Any]],
+        config_call_dict: Dict[str, Any],
         fqn: List[str],
         resource_type: NodeType,
         project_name: str,
         base: bool,
+        patch_config_dict: Dict[str, Any] = None
     ) -> BaseConfig:
         own_config = self.get_node_project(project_name)
 
@@ -134,8 +135,15 @@ class BaseContextConfigGenerator(Generic[T]):
         for fqn_config in project_configs:
             result = self._update_from_config(result, fqn_config)
 
-        for config_call in config_calls:
-            result = self._update_from_config(result, config_call)
+        # When schema files patch config, it has lower precedence than
+        # config in the models (config_call_dict), so we add the patch_config_dict
+        # before the config_call_dict
+        if patch_config_dict:
+            result = self._update_from_config(result, patch_config_dict)
+
+        # config_calls are created in the 'experimental' model parser and
+        # the ParseConfigObject (via add_config_call)
+        result = self._update_from_config(result, config_call_dict)
 
         if own_config.project_name != self._active_project.project_name:
             for fqn_config in self._active_project_configs(fqn, resource_type):
@@ -147,11 +155,12 @@ class BaseContextConfigGenerator(Generic[T]):
     @abstractmethod
     def calculate_node_config_dict(
         self,
-        config_calls: List[Dict[str, Any]],
+        config_call_dict: Dict[str, Any],
         fqn: List[str],
         resource_type: NodeType,
         project_name: str,
         base: bool,
+        patch_config_dict: Dict[str, Any],
     ) -> Dict[str, Any]:
         ...
 
@@ -186,18 +195,20 @@ class ContextConfigGenerator(BaseContextConfigGenerator[C]):
 
     def calculate_node_config_dict(
         self,
-        config_calls: List[Dict[str, Any]],
+        config_call_dict: Dict[str, Any],
         fqn: List[str],
         resource_type: NodeType,
         project_name: str,
         base: bool,
+        patch_config_dict: dict = None
     ) -> Dict[str, Any]:
         config = self.calculate_node_config(
-            config_calls=config_calls,
+            config_call_dict=config_call_dict,
             fqn=fqn,
             resource_type=resource_type,
             project_name=project_name,
             base=base,
+            patch_config_dict=patch_config_dict
         )
         finalized = config.finalize_and_validate()
         return finalized.to_dict(omit_none=True)
@@ -209,18 +220,20 @@ class UnrenderedConfigGenerator(BaseContextConfigGenerator[Dict[str, Any]]):
 
     def calculate_node_config_dict(
         self,
-        config_calls: List[Dict[str, Any]],
+        config_call_dict: Dict[str, Any],
         fqn: List[str],
         resource_type: NodeType,
         project_name: str,
         base: bool,
+        patch_config_dict: dict = None
     ) -> Dict[str, Any]:
         return self.calculate_node_config(
-            config_calls=config_calls,
+            config_call_dict=config_call_dict,
             fqn=fqn,
             resource_type=resource_type,
             project_name=project_name,
             base=base,
+            patch_config_dict=patch_config_dict
         )
 
     def initial_result(
@@ -251,20 +264,39 @@ class ContextConfig:
         resource_type: NodeType,
         project_name: str,
     ) -> None:
-        self._config_calls: List[Dict[str, Any]] = []
+        self._config_call_dict: Dict[str, Any] = {}
         self._active_project = active_project
         self._fqn = fqn
         self._resource_type = resource_type
         self._project_name = project_name
 
-    def update_in_model_config(self, opts: Dict[str, Any]) -> None:
-        self._config_calls.append(opts)
+    def add_config_call(self, opts: Dict[str, Any]) -> None:
+        dct = self._config_call_dict
+        self._add_config_call(dct, opts)
+
+    @classmethod
+    def _add_config_call(cls, config_call_dict, opts: Dict[str, Any]) -> None:
+        for k, v in opts.items():
+            # MergeBehavior for post-hook and pre-hook is to collect all
+            # values, instead of overwriting
+            if k in BaseConfig.mergebehavior['append']:
+                if not isinstance(v, list):
+                    v = [v]
+            if k in BaseConfig.mergebehavior['update'] and not isinstance(v, dict):
+                raise InternalException(f'expected dict, got {v}')
+            if k in config_call_dict and isinstance(config_call_dict[k], list):
+                config_call_dict[k].extend(v)
+            elif k in config_call_dict and isinstance(config_call_dict[k], dict):
+                config_call_dict[k].update(v)
+            else:
+                config_call_dict[k] = v
 
     def build_config_dict(
         self,
         base: bool = False,
         *,
         rendered: bool = True,
+        patch_config_dict: dict = None
     ) -> Dict[str, Any]:
         if rendered:
             src = ContextConfigGenerator(self._active_project)
@@ -272,9 +304,10 @@ class ContextConfig:
             src = UnrenderedConfigGenerator(self._active_project)
 
         return src.calculate_node_config_dict(
-            config_calls=self._config_calls,
+            config_call_dict=self._config_call_dict,
             fqn=self._fqn,
             resource_type=self._resource_type,
             project_name=self._project_name,
             base=base,
+            patch_config_dict=patch_config_dict
         )

core/dbt/context/providers.py

@@ -279,7 +279,7 @@ class Config(Protocol):
         ...
 
 
-# `config` implementations
+# Implementation of "config(..)" calls in models
 class ParseConfigObject(Config):
     def __init__(self, model, context_config: Optional[ContextConfig]):
         self.model = model
@@ -316,7 +316,7 @@ class ParseConfigObject(Config):
             raise RuntimeException(
                 'At parse time, did not receive a context config'
             )
-        self.context_config.update_in_model_config(opts)
+        self.context_config.add_config_call(opts)
         return ''
 
     def set(self, name, value):
@@ -1243,7 +1243,7 @@ class ModelContext(ProviderContext):
 
     @contextproperty
     def pre_hooks(self) -> List[Dict[str, Any]]:
-        if isinstance(self.model, ParsedSourceDefinition):
+        if self.model.resource_type in [NodeType.Source, NodeType.Test]:
             return []
         return [
             h.to_dict(omit_none=True) for h in self.model.config.pre_hook
@@ -1251,7 +1251,7 @@ class ModelContext(ProviderContext):
 
     @contextproperty
     def post_hooks(self) -> List[Dict[str, Any]]:
-        if isinstance(self.model, ParsedSourceDefinition):
+        if self.model.resource_type in [NodeType.Source, NodeType.Test]:
             return []
         return [
             h.to_dict(omit_none=True) for h in self.model.config.post_hook

core/dbt/contracts/connection.py

@@ -1,5 +1,6 @@
 import abc
 import itertools
+import hashlib
 from dataclasses import dataclass, field
 from typing import (
     Any, ClassVar, Dict, Tuple, Iterable, Optional, List, Callable,
@@ -127,6 +128,15 @@ class Credentials(
             'type not implemented for base credentials class'
         )
 
+    @abc.abstractproperty
+    def unique_field(self) -> str:
+        raise NotImplementedError(
+            'type not implemented for base credentials class'
+        )
+
+    def hashed_unique_field(self) -> str:
+        return hashlib.md5(self.unique_field.encode('utf-8')).hexdigest()
+
     def connection_info(
         self, *, with_aliases: bool = False
     ) -> Iterable[Tuple[str, Any]]:

core/dbt/contracts/files.py

@@ -220,7 +220,7 @@ class SchemaSourceFile(BaseSourceFile):
     # node patches contain models, seeds, snapshots, analyses
     ndp: List[str] = field(default_factory=list)
     # any macro patches in this file by macro unique_id.
-    mcp: List[str] = field(default_factory=list)
+    mcp: Dict[str, str] = field(default_factory=dict)
     # any source patches in this file. The entries are package, name pairs
     # Patches are only against external sources. Sources can be
     # created too, but those are in 'sources'

core/dbt/contracts/graph/compiled.py

@@ -109,7 +109,9 @@ class CompiledSnapshotNode(CompiledNode):
 @dataclass
 class CompiledDataTestNode(CompiledNode):
     resource_type: NodeType = field(metadata={'restrict': [NodeType.Test]})
-    config: TestConfig = field(default_factory=TestConfig)
+    # Was not able to make mypy happy and keep the code working. We need to
+    # refactor the various configs.
+    config: TestConfig = field(default_factory=TestConfig)  # type:ignore
 
 
 @dataclass
@@ -117,7 +119,9 @@ class CompiledSchemaTestNode(CompiledNode, HasTestMetadata):
     # keep this in sync with ParsedSchemaTestNode!
     resource_type: NodeType = field(metadata={'restrict': [NodeType.Test]})
     column_name: Optional[str] = None
-    config: TestConfig = field(default_factory=TestConfig)
+    # Was not able to make mypy happy and keep the code working. We need to
+    # refactor the various configs.
+    config: TestConfig = field(default_factory=TestConfig)  # type:ignore
 
     def same_contents(self, other) -> bool:
         if other is None:

core/dbt/contracts/graph/manifest.py

@@ -14,7 +14,7 @@ from dbt.contracts.graph.compiled import (
     CompileResultNode, ManifestNode, NonSourceCompiledNode, GraphMemberNode
 )
 from dbt.contracts.graph.parsed import (
-    ParsedMacro, ParsedDocumentation, ParsedNodePatch, ParsedMacroPatch,
+    ParsedMacro, ParsedDocumentation,
     ParsedSourceDefinition, ParsedExposure, HasUniqueID,
     UnpatchedSourceDefinition, ManifestNodes
 )
@@ -26,9 +26,7 @@ from dbt.contracts.util import (
 from dbt.dataclass_schema import dbtClassMixin
 from dbt.exceptions import (
     CompilationException,
-    raise_duplicate_resource_name, raise_compiler_error, warn_or_error,
-    raise_duplicate_patch_name,
-    raise_duplicate_macro_patch_name, raise_duplicate_source_patch_name,
+    raise_duplicate_resource_name, raise_compiler_error,
 )
 from dbt.helper_types import PathSet
 from dbt.logger import GLOBAL_LOGGER as logger
@@ -172,7 +170,7 @@ class RefableLookup(dbtClassMixin):
 
 
 class AnalysisLookup(RefableLookup):
-    _lookup_types: ClassVar[set] = set(NodeType.Analysis)
+    _lookup_types: ClassVar[set] = set([NodeType.Analysis])
 
 
 def _search_packages(
@@ -718,60 +716,6 @@ class Manifest(MacroMethods, DataClassMessagePackMixin, dbtClassMixin):
             resource_fqns[resource_type_plural].add(tuple(resource.fqn))
         return resource_fqns
 
-    # This is called by 'parse_patch' in the NodePatchParser
-    def add_patch(
-        self, source_file: SchemaSourceFile, patch: ParsedNodePatch,
-    ) -> None:
-        if patch.yaml_key in ['models', 'seeds', 'snapshots']:
-            unique_id = self.ref_lookup.get_unique_id(patch.name, None)
-        elif patch.yaml_key == 'analyses':
-            unique_id = self.analysis_lookup.get_unique_id(patch.name, None)
-        else:
-            raise dbt.exceptions.InternalException(
-                f'Unexpected yaml_key {patch.yaml_key} for patch in '
-                f'file {source_file.path.original_file_path}'
-            )
-        if unique_id is None:
-            # This will usually happen when a node is disabled
-            return
-
-        # patches can't be overwritten
-        node = self.nodes.get(unique_id)
-        if node:
-            if node.patch_path:
-                package_name, existing_file_path = node.patch_path.split('://')
-                raise_duplicate_patch_name(patch, existing_file_path)
-            source_file.append_patch(patch.yaml_key, unique_id)
-            node.patch(patch)
-
-    def add_macro_patch(
-        self, source_file: SchemaSourceFile, patch: ParsedMacroPatch,
-    ) -> None:
-        # macros are fully namespaced
-        unique_id = f'macro.{patch.package_name}.{patch.name}'
-        macro = self.macros.get(unique_id)
-        if not macro:
-            warn_or_error(
-                f'WARNING: Found documentation for macro "{patch.name}" '
-                f'which was not found'
-            )
-            return
-        if macro.patch_path:
-            package_name, existing_file_path = macro.patch_path.split('://')
-            raise_duplicate_macro_patch_name(patch, existing_file_path)
-        source_file.macro_patches.append(unique_id)
-        macro.patch(patch)
-
-    def add_source_patch(
-        self, source_file: SchemaSourceFile, patch: SourcePatch,
-    ) -> None:
-        # source patches must be unique
-        key = (patch.overrides, patch.name)
-        if key in self.source_patches:
-            raise_duplicate_source_patch_name(patch, self.source_patches[key])
-        self.source_patches[key] = patch
-        source_file.source_patches.append(key)
-
     def get_used_schemas(self, resource_types=None):
         return frozenset({
             (node.database, node.schema) for node in

core/dbt/contracts/graph/model_config.py

@@ -2,13 +2,13 @@ from dataclasses import field, Field, dataclass
 from enum import Enum
 from itertools import chain
 from typing import (
-    Any, List, Optional, Dict, Union, Type, TypeVar
+    Any, List, Optional, Dict, Union, Type, TypeVar, Callable
 )
 from dbt.dataclass_schema import (
     dbtClassMixin, ValidationError, register_pattern,
 )
 from dbt.contracts.graph.unparsed import AdditionalPropertiesAllowed
-from dbt.exceptions import InternalException
+from dbt.exceptions import InternalException, CompilationException
 from dbt.contracts.util import Replaceable, list_str
 from dbt import hooks
 from dbt.node_types import NodeType
@@ -204,6 +204,34 @@ class BaseConfig(
         else:
             self._extra[key] = value
 
+    def __delitem__(self, key):
+        if hasattr(self, key):
+            msg = (
+                'Error, tried to delete config key "{}": Cannot delete '
+                'built-in keys'
+            ).format(key)
+            raise CompilationException(msg)
+        else:
+            del self._extra[key]
+
+    def _content_iterator(self, include_condition: Callable[[Field], bool]):
+        seen = set()
+        for fld, _ in self._get_fields():
+            seen.add(fld.name)
+            if include_condition(fld):
+                yield fld.name
+
+        for key in self._extra:
+            if key not in seen:
+                seen.add(key)
+                yield key
+
+    def __iter__(self):
+        yield from self._content_iterator(include_condition=lambda f: True)
+
+    def __len__(self):
+        return len(self._get_fields()) + len(self._extra)
+
     @staticmethod
     def compare_key(
         unrendered: Dict[str, Any],
@@ -239,8 +267,15 @@ class BaseConfig(
                     return False
         return True
 
+    # This is used in 'add_config_call' to created the combined config_call_dict.
+    # 'meta' moved here from node
+    mergebehavior = {
+        "append": ['pre-hook', 'pre_hook', 'post-hook', 'post_hook', 'tags'],
+        "update": ['quoting', 'column_types', 'meta'],
+    }
+
     @classmethod
-    def _extract_dict(
+    def _merge_dicts(
         cls, src: Dict[str, Any], data: Dict[str, Any]
     ) -> Dict[str, Any]:
         """Find all the items in data that match a target_field on this class,
@@ -286,10 +321,10 @@ class BaseConfig(
 
         adapter_config_cls = get_config_class_by_name(adapter_type)
 
-        self_merged = self._extract_dict(dct, data)
+        self_merged = self._merge_dicts(dct, data)
         dct.update(self_merged)
 
-        adapter_merged = adapter_config_cls._extract_dict(dct, data)
+        adapter_merged = adapter_config_cls._merge_dicts(dct, data)
         dct.update(adapter_merged)
 
         # any remaining fields must be "clobber"
@@ -321,33 +356,8 @@ class SourceConfig(BaseConfig):
 
 
 @dataclass
-class NodeConfig(BaseConfig):
+class NodeAndTestConfig(BaseConfig):
     enabled: bool = True
-    materialized: str = 'view'
-    persist_docs: Dict[str, Any] = field(default_factory=dict)
-    post_hook: List[Hook] = field(
-        default_factory=list,
-        metadata=MergeBehavior.Append.meta(),
-    )
-    pre_hook: List[Hook] = field(
-        default_factory=list,
-        metadata=MergeBehavior.Append.meta(),
-    )
-    # this only applies for config v1, so it doesn't participate in comparison
-    vars: Dict[str, Any] = field(
-        default_factory=dict,
-        metadata=metas(CompareBehavior.Exclude, MergeBehavior.Update),
-    )
-    quoting: Dict[str, Any] = field(
-        default_factory=dict,
-        metadata=MergeBehavior.Update.meta(),
-    )
-    # This is actually only used by seeds. Should it be available to others?
-    # That would be a breaking change!
-    column_types: Dict[str, Any] = field(
-        default_factory=dict,
-        metadata=MergeBehavior.Update.meta(),
-    )
     # these fields are included in serialized output, but are not part of
     # config comparison (they are part of database_representation)
     alias: Optional[str] = field(
@@ -368,7 +378,38 @@ class NodeConfig(BaseConfig):
                        MergeBehavior.Append,
                        CompareBehavior.Exclude),
     )
+    meta: Dict[str, Any] = field(
+        default_factory=dict,
+        metadata=MergeBehavior.Update.meta(),
+    )
+
+
+@dataclass
+class NodeConfig(NodeAndTestConfig):
+    # Note: if any new fields are added with MergeBehavior, also update the
+    # 'mergebehavior' dictionary
+    materialized: str = 'view'
+    persist_docs: Dict[str, Any] = field(default_factory=dict)
+    post_hook: List[Hook] = field(
+        default_factory=list,
+        metadata=MergeBehavior.Append.meta(),
+    )
+    pre_hook: List[Hook] = field(
+        default_factory=list,
+        metadata=MergeBehavior.Append.meta(),
+    )
+    quoting: Dict[str, Any] = field(
+        default_factory=dict,
+        metadata=MergeBehavior.Update.meta(),
+    )
+    # This is actually only used by seeds. Should it be available to others?
+    # That would be a breaking change!
+    column_types: Dict[str, Any] = field(
+        default_factory=dict,
+        metadata=MergeBehavior.Update.meta(),
+    )
     full_refresh: Optional[bool] = None
+    on_schema_change: Optional[str] = 'ignore'
 
     @classmethod
     def __pre_deserialize__(cls, data):
@@ -410,7 +451,8 @@ class SeedConfig(NodeConfig):
 
 
 @dataclass
-class TestConfig(NodeConfig):
+class TestConfig(NodeAndTestConfig):
+    # this is repeated because of a different default
     schema: Optional[str] = field(
         default='dbt_test__audit',
         metadata=CompareBehavior.Exclude.meta(),

core/dbt/contracts/graph/parsed.py

@@ -148,6 +148,7 @@ class ParsedNodeMixins(dbtClassMixin):
         """Given a ParsedNodePatch, add the new information to the node."""
         # explicitly pick out the parts to update so we don't inadvertently
         # step on the model name or anything
+        # Note: config should already be updated
         self.patch_path: Optional[str] = patch.file_id
         # update created_at so process_docs will run in partial parsing
         self.created_at = int(time.time())
@@ -166,9 +167,6 @@ class ParsedNodeMixins(dbtClassMixin):
     def get_materialization(self):
         return self.config.materialized
 
-    def local_vars(self):
-        return self.config.vars
-
 
 @dataclass
 class ParsedNodeMandatory(
@@ -203,6 +201,7 @@ class ParsedNodeDefaults(ParsedNodeMandatory):
     deferred: bool = False
     unrendered_config: Dict[str, Any] = field(default_factory=dict)
     created_at: int = field(default_factory=lambda: int(time.time()))
+    config_call_dict: Dict[str, Any] = field(default_factory=dict)
 
     def write_node(self, target_path: str, subdirectory: str, payload: str):
         if (os.path.basename(self.path) ==
@@ -229,6 +228,11 @@ class ParsedNode(ParsedNodeDefaults, ParsedNodeMixins, SerializableType):
     def _serialize(self):
         return self.to_dict()
 
+    def __post_serialize__(self, dct):
+        if 'config_call_dict' in dct:
+            del dct['config_call_dict']
+        return dct
+
     @classmethod
     def _deserialize(cls, dct: Dict[str, int]):
         # The serialized ParsedNodes do not differ from each other
@@ -258,10 +262,16 @@ class ParsedNode(ParsedNodeDefaults, ParsedNodeMixins, SerializableType):
             return cls.from_dict(dct)
 
     def _persist_column_docs(self) -> bool:
-        return bool(self.config.persist_docs.get('columns'))
+        if hasattr(self.config, 'persist_docs'):
+            assert isinstance(self.config, NodeConfig)
+            return bool(self.config.persist_docs.get('columns'))
+        return False
 
     def _persist_relation_docs(self) -> bool:
-        return bool(self.config.persist_docs.get('relation'))
+        if hasattr(self.config, 'persist_docs'):
+            assert isinstance(self.config, NodeConfig)
+            return bool(self.config.persist_docs.get('relation'))
+        return False
 
     def same_body(self: T, other: T) -> bool:
         return self.raw_sql == other.raw_sql
@@ -411,7 +421,9 @@ class HasTestMetadata(dbtClassMixin):
 @dataclass
 class ParsedDataTestNode(ParsedNode):
     resource_type: NodeType = field(metadata={'restrict': [NodeType.Test]})
-    config: TestConfig = field(default_factory=TestConfig)
+    # Was not able to make mypy happy and keep the code working. We need to
+    # refactor the various configs.
+    config: TestConfig = field(default_factory=TestConfig)  # type: ignore
 
 
 @dataclass
@@ -419,7 +431,9 @@ class ParsedSchemaTestNode(ParsedNode, HasTestMetadata):
     # keep this in sync with CompiledSchemaTestNode!
     resource_type: NodeType = field(metadata={'restrict': [NodeType.Test]})
     column_name: Optional[str] = None
-    config: TestConfig = field(default_factory=TestConfig)
+    # Was not able to make mypy happy and keep the code working. We need to
+    # refactor the various configs.
+    config: TestConfig = field(default_factory=TestConfig)  # type: ignore
 
     def same_contents(self, other) -> bool:
         if other is None:
@@ -456,6 +470,7 @@ class ParsedPatch(HasYamlMetadata, Replaceable):
     description: str
     meta: Dict[str, Any]
     docs: Docs
+    config: Dict[str, Any]
 
 
 # The parsed node update is only the 'patch', not the test. The test became a
@@ -487,9 +502,6 @@ class ParsedMacro(UnparsedBaseNode, HasUniqueID):
     arguments: List[MacroArgument] = field(default_factory=list)
     created_at: int = field(default_factory=lambda: int(time.time()))
 
-    def local_vars(self):
-        return {}
-
     def patch(self, patch: ParsedMacroPatch):
         self.patch_path: Optional[str] = patch.file_id
         self.description = patch.description
@@ -592,7 +604,8 @@ class ParsedSourceDefinition(
     UnparsedBaseNode,
     HasUniqueID,
     HasRelationMetadata,
-    HasFqn
+    HasFqn,
+
 ):
     name: str
     source_name: str
@@ -689,6 +702,10 @@ class ParsedSourceDefinition(
     def depends_on_nodes(self):
         return []
 
+    @property
+    def depends_on(self):
+        return DependsOn(macros=[], nodes=[])
+
     @property
     def refs(self):
         return []

core/dbt/contracts/graph/unparsed.py

@@ -126,12 +126,17 @@ class HasYamlMetadata(dbtClassMixin):
 
 
 @dataclass
-class UnparsedAnalysisUpdate(HasColumnDocs, HasDocs, HasYamlMetadata):
+class HasConfig():
+    config: Dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass
+class UnparsedAnalysisUpdate(HasConfig, HasColumnDocs, HasDocs, HasYamlMetadata):
     pass
 
 
 @dataclass
-class UnparsedNodeUpdate(HasColumnTests, HasTests, HasYamlMetadata):
+class UnparsedNodeUpdate(HasConfig, HasColumnTests, HasTests, HasYamlMetadata):
     quote_columns: Optional[bool] = None
 
 
@@ -143,7 +148,7 @@ class MacroArgument(dbtClassMixin):
 
 
 @dataclass
-class UnparsedMacroUpdate(HasDocs, HasYamlMetadata):
+class UnparsedMacroUpdate(HasConfig, HasDocs, HasYamlMetadata):
     arguments: List[MacroArgument] = field(default_factory=list)
 
 
@@ -261,6 +266,7 @@ class UnparsedSourceDefinition(dbtClassMixin, Replaceable):
     loaded_at_field: Optional[str] = None
     tables: List[UnparsedSourceTableDefinition] = field(default_factory=list)
     tags: List[str] = field(default_factory=list)
+    config: Dict[str, Any] = field(default_factory=dict)
 
     @property
     def yaml_key(self) -> 'str':

core/dbt/contracts/project.py

@@ -83,6 +83,7 @@ class GitPackage(Package):
 class RegistryPackage(Package):
     package: str
     version: Union[RawVersion, List[RawVersion]]
+    install_prerelease: Optional[bool] = False
 
     def get_versions(self) -> List[str]:
         if isinstance(self.version, list):

core/dbt/contracts/results.py

@@ -78,6 +78,7 @@ class TestStatus(StrEnum):
     Error = NodeStatus.Error
     Fail = NodeStatus.Fail
     Warn = NodeStatus.Warn
+    Skipped = NodeStatus.Skipped
 
 
 class FreshnessStatus(StrEnum):

core/dbt/contracts/rpc.py

@@ -58,6 +58,7 @@ class RPCExecParameters(RPCParameters):
 class RPCCompileParameters(RPCParameters):
     threads: Optional[int] = None
     models: Union[None, str, List[str]] = None
+    select: Union[None, str, List[str]] = None
     exclude: Union[None, str, List[str]] = None
     selector: Optional[str] = None
     state: Optional[str] = None
@@ -71,12 +72,14 @@ class RPCListParameters(RPCParameters):
     select: Union[None, str, List[str]] = None
     selector: Optional[str] = None
     output: Optional[str] = 'json'
+    output_keys: Optional[List[str]] = None
 
 
 @dataclass
 class RPCRunParameters(RPCParameters):
     threads: Optional[int] = None
     models: Union[None, str, List[str]] = None
+    select: Union[None, str, List[str]] = None
     exclude: Union[None, str, List[str]] = None
     selector: Optional[str] = None
     state: Optional[str] = None
@@ -116,6 +119,17 @@ class RPCDocsGenerateParameters(RPCParameters):
     state: Optional[str] = None
 
 
+@dataclass
+class RPCBuildParameters(RPCParameters):
+    threads: Optional[int] = None
+    models: Union[None, str, List[str]] = None
+    select: Union[None, str, List[str]] = None
+    exclude: Union[None, str, List[str]] = None
+    selector: Optional[str] = None
+    state: Optional[str] = None
+    defer: Optional[bool] = None
+
+
 @dataclass
 class RPCCliParameters(RPCParameters):
     cli: str
@@ -186,6 +200,8 @@ class RPCRunOperationParameters(RPCParameters):
 class RPCSourceFreshnessParameters(RPCParameters):
     threads: Optional[int] = None
     select: Union[None, str, List[str]] = None
+    exclude: Union[None, str, List[str]] = None
+    selector: Optional[str] = None
 
 
 @dataclass

core/dbt/deprecations.py

@@ -131,6 +131,14 @@ class AdapterMacroDeprecation(DBTDeprecation):
     '''
 
 
+class PackageRedirectDeprecation(DBTDeprecation):
+    _name = 'package-redirect'
+    _description = '''\
+    The `{old_name}` package is deprecated in favor of `{new_name}`. Please update
+    your `packages.yml` configuration to use `{new_name}` instead.
+    '''
+
+
 _adapter_renamed_description = """\
 The adapter function `adapter.{old_name}` is deprecated and will be removed in
 a future release of dbt. Please use `adapter.{new_name}` instead.
@@ -176,6 +184,7 @@ deprecations_list: List[DBTDeprecation] = [
     ModelsKeyNonModelDeprecation(),
     ExecuteMacrosReleaseDeprecation(),
     AdapterMacroDeprecation(),
+    PackageRedirectDeprecation()
 ]
 
 deprecations: Dict[str, DBTDeprecation] = {

core/dbt/deps/registry.py

@@ -30,9 +30,13 @@ class RegistryPackageMixin:
 
 
 class RegistryPinnedPackage(RegistryPackageMixin, PinnedPackage):
-    def __init__(self, package: str, version: str) -> None:
+    def __init__(self,
+                 package: str,
+                 version: str,
+                 version_latest: str) -> None:
         super().__init__(package)
         self.version = version
+        self.version_latest = version_latest
 
     @property
     def name(self):
@@ -44,6 +48,9 @@ class RegistryPinnedPackage(RegistryPackageMixin, PinnedPackage):
     def get_version(self):
         return self.version
 
+    def get_version_latest(self):
+        return self.version_latest
+
     def nice_version_name(self):
         return 'version {}'.format(self.version)
 
@@ -61,7 +68,7 @@ class RegistryPinnedPackage(RegistryPackageMixin, PinnedPackage):
         system.make_directory(os.path.dirname(tar_path))
 
         download_url = metadata.downloads.tarball
-        system.download(download_url, tar_path)
+        system.download_with_retries(download_url, tar_path)
         deps_path = project.modules_path
         package_name = self.get_project_name(project, renderer)
         system.untar_package(tar_path, deps_path, package_name)
@@ -71,10 +78,14 @@ class RegistryUnpinnedPackage(
     RegistryPackageMixin, UnpinnedPackage[RegistryPinnedPackage]
 ):
     def __init__(
-        self, package: str, versions: List[semver.VersionSpecifier]
+        self,
+        package: str,
+        versions: List[semver.VersionSpecifier],
+        install_prerelease: bool
     ) -> None:
         super().__init__(package)
         self.versions = versions
+        self.install_prerelease = install_prerelease
 
     def _check_in_index(self):
         index = registry.index_cached()
@@ -91,13 +102,18 @@ class RegistryUnpinnedPackage(
             semver.VersionSpecifier.from_version_string(v)
             for v in raw_version
         ]
-        return cls(package=contract.package, versions=versions)
+        return cls(
+            package=contract.package,
+            versions=versions,
+            install_prerelease=contract.install_prerelease
+        )
 
     def incorporate(
         self, other: 'RegistryUnpinnedPackage'
     ) -> 'RegistryUnpinnedPackage':
         return RegistryUnpinnedPackage(
             package=self.package,
+            install_prerelease=self.install_prerelease,
             versions=self.versions + other.versions,
         )
 
@@ -111,12 +127,18 @@ class RegistryUnpinnedPackage(
             raise DependencyException(new_msg) from e
 
         available = registry.get_available_versions(self.package)
+        installable = semver.filter_installable(
+            available,
+            self.install_prerelease
+        )
+        available_latest = installable[-1]
 
         # for now, pick a version and then recurse. later on,
         # we'll probably want to traverse multiple options
         # so we can match packages. not going to make a difference
         # right now.
-        target = semver.resolve_to_specific_version(range_, available)
+        target = semver.resolve_to_specific_version(range_, installable)
         if not target:
-            package_version_not_found(self.package, range_, available)
-        return RegistryPinnedPackage(package=self.package, version=target)
+            package_version_not_found(self.package, range_, installable)
+        return RegistryPinnedPackage(package=self.package, version=target,
+                                     version_latest=available_latest)

core/dbt/exceptions.py

@@ -710,11 +710,11 @@ def system_error(operation_name):
     raise_compiler_error(
         "dbt encountered an error when attempting to {}. "
         "If this error persists, please create an issue at: \n\n"
-        "https://github.com/fishtown-analytics/dbt"
+        "https://github.com/dbt-labs/dbt"
         .format(operation_name))
 
 
-class RegistryException(Exception):
+class ConnectionException(Exception):
     pass
 
 

core/dbt/graph/queue.py

@@ -1,10 +1,8 @@
-import threading
-from queue import PriorityQueue
-from typing import (
-    Dict, Set, Optional
-)
-
 import networkx as nx  # type: ignore
+import threading
+
+from queue import PriorityQueue
+from typing import Dict, Set, List, Generator, Optional
 
 from .graph import UniqueId
 from dbt.contracts.graph.parsed import ParsedSourceDefinition, ParsedExposure
@@ -21,9 +19,8 @@ class GraphQueue:
     that separate threads do not call `.empty()` or `__len__()` and `.get()` at
     the same time, as there is an unlocked race!
     """
-    def __init__(
-        self, graph: nx.DiGraph, manifest: Manifest, selected: Set[UniqueId]
-    ):
+
+    def __init__(self, graph: nx.DiGraph, manifest: Manifest, selected: Set[UniqueId]):
         self.graph = graph
         self.manifest = manifest
         self._selected = selected
@@ -37,7 +34,7 @@ class GraphQueue:
         # this lock controls most things
         self.lock = threading.Lock()
         # store the 'score' of each node as a number. Lower is higher priority.
-        self._scores = self._calculate_scores()
+        self._scores = self._get_scores(self.graph)
         # populate the initial queue
         self._find_new_additions()
         # awaits after task end
@@ -56,30 +53,59 @@ class GraphQueue:
             return False
         return True
 
-    def _calculate_scores(self) -> Dict[UniqueId, int]:
-        """Calculate the 'value' of each node in the graph based on how many
-        blocking descendants it has. We use this score for the internal
-        priority queue's ordering, so the quality of this metric is important.
+    @staticmethod
+    def _grouped_topological_sort(
+        graph: nx.DiGraph,
+    ) -> Generator[List[str], None, None]:
+        """Topological sort of given graph that groups ties.
 
-        The score is stored as a negative number because the internal
-        PriorityQueue picks lowest values first.
+        Adapted from `nx.topological_sort`, this function returns a topo sort of a graph however
+        instead of arbitrarily ordering ties in the sort order, ties are grouped together in
+        lists.
 
-        We could do this in one pass over the graph instead of len(self.graph)
-        passes but this is easy. For large graphs this may hurt performance.
+        Args:
+            graph: The graph to be sorted.
 
-        This operates on the graph, so it would require a lock if called from
-        outside __init__.
-
-        :return Dict[str, int]: The score dict, mapping unique IDs to integer
-            scores. Lower scores are higher priority.
+        Returns:
+            A generator that yields lists of nodes, one list per graph depth level.
         """
+        indegree_map = {v: d for v, d in graph.in_degree() if d > 0}
+        zero_indegree = [v for v, d in graph.in_degree() if d == 0]
+
+        while zero_indegree:
+            yield zero_indegree
+            new_zero_indegree = []
+            for v in zero_indegree:
+                for _, child in graph.edges(v):
+                    indegree_map[child] -= 1
+                    if not indegree_map[child]:
+                        new_zero_indegree.append(child)
+            zero_indegree = new_zero_indegree
+
+    def _get_scores(self, graph: nx.DiGraph) -> Dict[str, int]:
+        """Scoring nodes for processing order.
+
+        Scores are calculated by the graph depth level. Lowest score (0) should be processed first.
+
+        Args:
+            graph: The graph to be scored.
+
+        Returns:
+            A dictionary consisting of `node name`:`score` pairs.
+        """
+        # split graph by connected subgraphs
+        subgraphs = (
+            graph.subgraph(x) for x in nx.connected_components(nx.Graph(graph))
+        )
+
+        # score all nodes in all subgraphs
         scores = {}
-        for node in self.graph.nodes():
-            score = -1 * len([
-                d for d in nx.descendants(self.graph, node)
-                if self._include_in_cost(d)
-            ])
-            scores[node] = score
+        for subgraph in subgraphs:
+            grouped_nodes = self._grouped_topological_sort(subgraph)
+            for level, group in enumerate(grouped_nodes):
+                for node in group:
+                    scores[node] = level
+
         return scores
 
     def get(
@@ -133,8 +159,6 @@ class GraphQueue:
     def _find_new_additions(self) -> None:
         """Find any nodes in the graph that need to be added to the internal
         queue and add them.
-
-        Callers must hold the lock.
         """
         for node, in_degree in self.graph.in_degree():
             if not self._already_known(node) and in_degree == 0:

core/dbt/graph/selector_methods.py

@@ -22,13 +22,11 @@ from dbt.contracts.graph.parsed import (
     ParsedSourceDefinition,
 )
 from dbt.contracts.state import PreviousState
-from dbt.logger import GLOBAL_LOGGER as logger
 from dbt.exceptions import (
     InternalException,
     RuntimeException,
 )
 from dbt.node_types import NodeType
-from dbt.ui import warning_tag
 
 
 SELECTOR_GLOB = '*'
@@ -381,7 +379,7 @@ class TestTypeSelectorMethod(SelectorMethod):
 class StateSelectorMethod(SelectorMethod):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
-        self.macros_were_modified: Optional[List[str]] = None
+        self.modified_macros: Optional[List[str]] = None
 
     def _macros_modified(self) -> List[str]:
         # we checked in the caller!
@@ -394,44 +392,74 @@ class StateSelectorMethod(SelectorMethod):
 
         modified = []
         for uid, macro in new_macros.items():
-            name = f'{macro.package_name}.{macro.name}'
             if uid in old_macros:
                 old_macro = old_macros[uid]
                 if macro.macro_sql != old_macro.macro_sql:
-                    modified.append(f'{name} changed')
+                    modified.append(uid)
             else:
-                modified.append(f'{name} added')
+                modified.append(uid)
 
         for uid, macro in old_macros.items():
             if uid not in new_macros:
-                modified.append(f'{macro.package_name}.{macro.name} removed')
+                modified.append(uid)
 
-        return modified[:3]
+        return modified
 
-    def check_modified(
-        self,
-        old: Optional[SelectorTarget],
-        new: SelectorTarget,
+    def recursively_check_macros_modified(self, node):
+        # check if there are any changes in macros the first time
+        if self.modified_macros is None:
+            self.modified_macros = self._macros_modified()
+
+        # loop through all macros that this node depends on
+        for macro_uid in node.depends_on.macros:
+            # is this macro one of the modified macros?
+            if macro_uid in self.modified_macros:
+                return True
+            # if not, and this macro depends on other macros, keep looping
+            macro = self.manifest.macros[macro_uid]
+            if len(macro.depends_on.macros) > 0:
+                return self.recursively_check_macros_modified(macro)
+            else:
+                return False
+        return False
+
+    def check_modified(self, old: Optional[SelectorTarget], new: SelectorTarget) -> bool:
+        different_contents = not new.same_contents(old)  # type: ignore
+        upstream_macro_change = self.recursively_check_macros_modified(new)
+        return different_contents or upstream_macro_change
+
+    def check_modified_body(self, old: Optional[SelectorTarget], new: SelectorTarget) -> bool:
+        if hasattr(new, "same_body"):
+            return not new.same_body(old)  # type: ignore
+        else:
+            return False
+
+    def check_modified_configs(self, old: Optional[SelectorTarget], new: SelectorTarget) -> bool:
+        if hasattr(new, "same_config"):
+            return not new.same_config(old)  # type: ignore
+        else:
+            return False
+
+    def check_modified_persisted_descriptions(
+        self, old: Optional[SelectorTarget], new: SelectorTarget
     ) -> bool:
-        # check if there are any changes in macros, if so, log a warning the
-        # first time
-        if self.macros_were_modified is None:
-            self.macros_were_modified = self._macros_modified()
-            if self.macros_were_modified:
-                log_str = ', '.join(self.macros_were_modified)
-                logger.warning(warning_tag(
-                    f'During a state comparison, dbt detected a change in '
-                    f'macros. This will not be marked as a modification. Some '
-                    f'macros: {log_str}'
-                ))
+        if hasattr(new, "same_persisted_description"):
+            return not new.same_persisted_description(old)  # type: ignore
+        else:
+            return False
 
-        return not new.same_contents(old)  # type: ignore
-
-    def check_new(
-        self,
-        old: Optional[SelectorTarget],
-        new: SelectorTarget,
+    def check_modified_relation(
+        self, old: Optional[SelectorTarget], new: SelectorTarget
     ) -> bool:
+        if hasattr(new, "same_database_representation"):
+            return not new.same_database_representation(old)  # type: ignore
+        else:
+            return False
+
+    def check_modified_macros(self, _, new: SelectorTarget) -> bool:
+        return self.recursively_check_macros_modified(new)
+
+    def check_new(self, old: Optional[SelectorTarget], new: SelectorTarget) -> bool:
         return old is None
 
     def search(
@@ -443,8 +471,15 @@ class StateSelectorMethod(SelectorMethod):
             )
 
         state_checks = {
+            # it's new if there is no old version
+            'new': lambda old, _: old is None,
+            # use methods defined above to compare properties of old + new
             'modified': self.check_modified,
-            'new': self.check_new,
+            'modified.body': self.check_modified_body,
+            'modified.configs': self.check_modified_configs,
+            'modified.persisted_descriptions': self.check_modified_persisted_descriptions,
+            'modified.relation': self.check_modified_relation,
+            'modified.macros': self.check_modified_macros,
         }
         if selector in state_checks:
             checker = state_checks[selector]

core/dbt/include/global_project/macros/adapters/common.sql

@@ -1,5 +1,5 @@
 {% macro get_columns_in_query(select_sql) -%}
-  {{ return(adapter.dispatch('get_columns_in_query')(select_sql)) }}
+  {{ return(adapter.dispatch('get_columns_in_query', 'dbt')(select_sql)) }}
 {% endmacro %}
 
 {% macro default__get_columns_in_query(select_sql) %}
@@ -15,7 +15,7 @@
 {% endmacro %}
 
 {% macro create_schema(relation) -%}
-  {{ adapter.dispatch('create_schema')(relation) }}
+  {{ adapter.dispatch('create_schema', 'dbt')(relation) }}
 {% endmacro %}
 
 {% macro default__create_schema(relation) -%}
@@ -25,7 +25,7 @@
 {% endmacro %}
 
 {% macro drop_schema(relation) -%}
-  {{ adapter.dispatch('drop_schema')(relation) }}
+  {{ adapter.dispatch('drop_schema', 'dbt')(relation) }}
 {% endmacro %}
 
 {% macro default__drop_schema(relation) -%}
@@ -35,7 +35,7 @@
 {% endmacro %}
 
 {% macro create_table_as(temporary, relation, sql) -%}
-  {{ adapter.dispatch('create_table_as')(temporary, relation, sql) }}
+  {{ adapter.dispatch('create_table_as', 'dbt')(temporary, relation, sql) }}
 {%- endmacro %}
 
 {% macro default__create_table_as(temporary, relation, sql) -%}
@@ -52,7 +52,7 @@
 {% endmacro %}
 
 {% macro get_create_index_sql(relation, index_dict) -%}
-  {{ return(adapter.dispatch('get_create_index_sql')(relation, index_dict)) }}
+  {{ return(adapter.dispatch('get_create_index_sql', 'dbt')(relation, index_dict)) }}
 {% endmacro %}
 
 {% macro default__get_create_index_sql(relation, index_dict) -%}
@@ -60,7 +60,7 @@
 {% endmacro %}
 
 {% macro create_indexes(relation) -%}
-  {{ adapter.dispatch('create_indexes')(relation) }}
+  {{ adapter.dispatch('create_indexes', 'dbt')(relation) }}
 {%- endmacro %}
 
 {% macro default__create_indexes(relation) -%}
@@ -75,7 +75,7 @@
 {% endmacro %}
 
 {% macro create_view_as(relation, sql) -%}
-  {{ adapter.dispatch('create_view_as')(relation, sql) }}
+  {{ adapter.dispatch('create_view_as', 'dbt')(relation, sql) }}
 {%- endmacro %}
 
 {% macro default__create_view_as(relation, sql) -%}
@@ -89,7 +89,7 @@
 
 
 {% macro get_catalog(information_schema, schemas) -%}
-  {{ return(adapter.dispatch('get_catalog')(information_schema, schemas)) }}
+  {{ return(adapter.dispatch('get_catalog', 'dbt')(information_schema, schemas)) }}
 {%- endmacro %}
 
 {% macro default__get_catalog(information_schema, schemas) -%}
@@ -104,7 +104,7 @@
 
 
 {% macro get_columns_in_relation(relation) -%}
-  {{ return(adapter.dispatch('get_columns_in_relation')(relation)) }}
+  {{ return(adapter.dispatch('get_columns_in_relation', 'dbt')(relation)) }}
 {% endmacro %}
 
 {% macro sql_convert_columns_in_relation(table) -%}
@@ -121,13 +121,13 @@
 {% endmacro %}
 
 {% macro alter_column_type(relation, column_name, new_column_type) -%}
-  {{ return(adapter.dispatch('alter_column_type')(relation, column_name, new_column_type)) }}
+  {{ return(adapter.dispatch('alter_column_type', 'dbt')(relation, column_name, new_column_type)) }}
 {% endmacro %}
 
 
 
 {% macro alter_column_comment(relation, column_dict) -%}
-  {{ return(adapter.dispatch('alter_column_comment')(relation, column_dict)) }}
+  {{ return(adapter.dispatch('alter_column_comment', 'dbt')(relation, column_dict)) }}
 {% endmacro %}
 
 {% macro default__alter_column_comment(relation, column_dict) -%}
@@ -136,7 +136,7 @@
 {% endmacro %}
 
 {% macro alter_relation_comment(relation, relation_comment) -%}
-  {{ return(adapter.dispatch('alter_relation_comment')(relation, relation_comment)) }}
+  {{ return(adapter.dispatch('alter_relation_comment', 'dbt')(relation, relation_comment)) }}
 {% endmacro %}
 
 {% macro default__alter_relation_comment(relation, relation_comment) -%}
@@ -145,7 +145,7 @@
 {% endmacro %}
 
 {% macro persist_docs(relation, model, for_relation=true, for_columns=true) -%}
-  {{ return(adapter.dispatch('persist_docs')(relation, model, for_relation, for_columns)) }}
+  {{ return(adapter.dispatch('persist_docs', 'dbt')(relation, model, for_relation, for_columns)) }}
 {% endmacro %}
 
 {% macro default__persist_docs(relation, model, for_relation, for_columns) -%}
@@ -180,7 +180,7 @@
 
 
 {% macro drop_relation(relation) -%}
-  {{ return(adapter.dispatch('drop_relation')(relation)) }}
+  {{ return(adapter.dispatch('drop_relation', 'dbt')(relation)) }}
 {% endmacro %}
 
 
@@ -191,7 +191,7 @@
 {% endmacro %}
 
 {% macro truncate_relation(relation) -%}
-  {{ return(adapter.dispatch('truncate_relation')(relation)) }}
+  {{ return(adapter.dispatch('truncate_relation', 'dbt')(relation)) }}
 {% endmacro %}
 
 
@@ -202,7 +202,7 @@
 {% endmacro %}
 
 {% macro rename_relation(from_relation, to_relation) -%}
-  {{ return(adapter.dispatch('rename_relation')(from_relation, to_relation)) }}
+  {{ return(adapter.dispatch('rename_relation', 'dbt')(from_relation, to_relation)) }}
 {% endmacro %}
 
 {% macro default__rename_relation(from_relation, to_relation) -%}
@@ -214,7 +214,7 @@
 
 
 {% macro information_schema_name(database) %}
-  {{ return(adapter.dispatch('information_schema_name')(database)) }}
+  {{ return(adapter.dispatch('information_schema_name', 'dbt')(database)) }}
 {% endmacro %}
 
 {% macro default__information_schema_name(database) -%}
@@ -227,7 +227,7 @@
 
 
 {% macro list_schemas(database) -%}
-  {{ return(adapter.dispatch('list_schemas')(database)) }}
+  {{ return(adapter.dispatch('list_schemas', 'dbt')(database)) }}
 {% endmacro %}
 
 {% macro default__list_schemas(database) -%}
@@ -241,7 +241,7 @@
 
 
 {% macro check_schema_exists(information_schema, schema) -%}
-  {{ return(adapter.dispatch('check_schema_exists')(information_schema, schema)) }}
+  {{ return(adapter.dispatch('check_schema_exists', 'dbt')(information_schema, schema)) }}
 {% endmacro %}
 
 {% macro default__check_schema_exists(information_schema, schema) -%}
@@ -256,7 +256,7 @@
 
 
 {% macro list_relations_without_caching(schema_relation) %}
-  {{ return(adapter.dispatch('list_relations_without_caching')(schema_relation)) }}
+  {{ return(adapter.dispatch('list_relations_without_caching', 'dbt')(schema_relation)) }}
 {% endmacro %}
 
 
@@ -267,7 +267,7 @@
 
 
 {% macro current_timestamp() -%}
-  {{ adapter.dispatch('current_timestamp')() }}
+  {{ adapter.dispatch('current_timestamp', 'dbt')() }}
 {%- endmacro %}
 
 
@@ -278,7 +278,7 @@
 
 
 {% macro collect_freshness(source, loaded_at_field, filter) %}
-  {{ return(adapter.dispatch('collect_freshness')(source, loaded_at_field, filter))}}
+  {{ return(adapter.dispatch('collect_freshness', 'dbt')(source, loaded_at_field, filter))}}
 {% endmacro %}
 
 
@@ -296,7 +296,7 @@
 {% endmacro %}
 
 {% macro make_temp_relation(base_relation, suffix='__dbt_tmp') %}
-  {{ return(adapter.dispatch('make_temp_relation')(base_relation, suffix))}}
+  {{ return(adapter.dispatch('make_temp_relation', 'dbt')(base_relation, suffix))}}
 {% endmacro %}
 
 {% macro default__make_temp_relation(base_relation, suffix) %}
@@ -311,3 +311,34 @@
   {{ config.set('sql_header', caller()) }}
 {%- endmacro %}
 
+
+{% macro alter_relation_add_remove_columns(relation, add_columns = none, remove_columns = none) -%}
+  {{ return(adapter.dispatch('alter_relation_add_remove_columns', 'dbt')(relation, add_columns, remove_columns)) }}
+{% endmacro %}
+
+{% macro default__alter_relation_add_remove_columns(relation, add_columns, remove_columns) %}
+  
+  {% if add_columns is none %}
+    {% set add_columns = [] %}
+  {% endif %}
+  {% if remove_columns is none %}
+    {% set remove_columns = [] %}
+  {% endif %}
+  
+  {% set sql -%}
+     
+     alter {{ relation.type }} {{ relation }}
+       
+            {% for column in add_columns %}
+               add column {{ column.name }} {{ column.data_type }}{{ ',' if not loop.last }}
+            {% endfor %}{{ ',' if remove_columns | length > 0 }}
+            
+            {% for column in remove_columns %}
+                drop column {{ column.name }}{{ ',' if not loop.last }}
+            {% endfor %}
+  
+  {%- endset -%}
+
+  {% do run_query(sql) %}
+
+{% endmacro %}

core/dbt/include/global_project/macros/etc/get_custom_alias.sql

@@ -13,6 +13,10 @@
 
 #}
 {% macro generate_alias_name(custom_alias_name=none, node=none) -%}
+    {% do return(adapter.dispatch('generate_alias_name', 'dbt')(custom_alias_name, node)) %}
+{%- endmacro %}
+
+{% macro default__generate_alias_name(custom_alias_name=none, node=none) -%}
 
     {%- if custom_alias_name is none -%}
 

core/dbt/include/global_project/macros/etc/get_custom_database.sql

@@ -14,7 +14,7 @@
 
 #}
 {% macro generate_database_name(custom_database_name=none, node=none) -%}
-    {% do return(adapter.dispatch('generate_database_name')(custom_database_name, node)) %}
+    {% do return(adapter.dispatch('generate_database_name', 'dbt')(custom_database_name, node)) %}
 {%- endmacro %}
 
 {% macro default__generate_database_name(custom_database_name=none, node=none) -%}

core/dbt/include/global_project/macros/etc/get_custom_schema.sql

@@ -15,6 +15,10 @@
 
 #}
 {% macro generate_schema_name(custom_schema_name, node) -%}
+    {{ return(adapter.dispatch('generate_schema_name', 'dbt')(custom_schema_name, node)) }}
+{% endmacro %}
+
+{% macro default__generate_schema_name(custom_schema_name, node) -%}
 
     {%- set default_schema = target.schema -%}
     {%- if custom_schema_name is none -%}

core/dbt/include/global_project/macros/etc/where_subquery.sql

@@ -0,0 +1,15 @@
+{% macro get_where_subquery(relation) -%}
+    {% do return(adapter.dispatch('get_where_subquery')(relation)) %}
+{%- endmacro %}
+
+{% macro default__get_where_subquery(relation) -%}
+    {% set where = config.get('where', '') %}
+    {% if where %}
+        {%- set filtered -%}
+            (select * from {{ relation }} where {{ where }}) dbt_subquery
+        {%- endset -%}
+        {% do return(filtered) %}
+    {%- else -%}
+        {% do return(relation) %}
+    {%- endif -%}
+{%- endmacro %}

core/dbt/include/global_project/macros/materializations/common/merge.sql

@@ -1,17 +1,17 @@
 
 
 {% macro get_merge_sql(target, source, unique_key, dest_columns, predicates=none) -%}
-  {{ adapter.dispatch('get_merge_sql')(target, source, unique_key, dest_columns, predicates) }}
+  {{ adapter.dispatch('get_merge_sql', 'dbt')(target, source, unique_key, dest_columns, predicates) }}
 {%- endmacro %}
 
 
 {% macro get_delete_insert_merge_sql(target, source, unique_key, dest_columns) -%}
-  {{ adapter.dispatch('get_delete_insert_merge_sql')(target, source, unique_key, dest_columns) }}
+  {{ adapter.dispatch('get_delete_insert_merge_sql', 'dbt')(target, source, unique_key, dest_columns) }}
 {%- endmacro %}
 
 
 {% macro get_insert_overwrite_merge_sql(target, source, dest_columns, predicates, include_sql_header=false) -%}
-  {{ adapter.dispatch('get_insert_overwrite_merge_sql')(target, source, dest_columns, predicates, include_sql_header) }}
+  {{ adapter.dispatch('get_insert_overwrite_merge_sql', 'dbt')(target, source, dest_columns, predicates, include_sql_header) }}
 {%- endmacro %}
 
 
@@ -79,7 +79,7 @@
     (
         select {{ dest_cols_csv }}
         from {{ source }}
-    );
+    )
 
 {%- endmacro %}
 

core/dbt/include/global_project/macros/materializations/incremental/helpers.sql

@@ -1,5 +1,6 @@
 
 {% macro incremental_upsert(tmp_relation, target_relation, unique_key=none, statement_name="main") %}
+    
     {%- set dest_columns = adapter.get_columns_in_relation(target_relation) -%}
     {%- set dest_cols_csv = dest_columns | map(attribute='quoted') | join(', ') -%}
 

core/dbt/include/global_project/macros/materializations/incremental/incremental.sql

@@ -5,6 +5,10 @@
 
   {% set target_relation = this.incorporate(type='table') %}
   {% set existing_relation = load_relation(this) %}
+  {% set tmp_relation = make_temp_relation(target_relation) %}
+  {%- set full_refresh_mode = (should_full_refresh()) -%}
+
+  {% set on_schema_change = incremental_validate_on_schema_change(config.get('on_schema_change'), default='ignore') %}
 
   {% set tmp_identifier = model['name'] + '__dbt_tmp' %}
   {% set backup_identifier = model['name'] + "__dbt_backup" %}
@@ -28,9 +32,16 @@
   {{ run_hooks(pre_hooks, inside_transaction=True) }}
 
   {% set to_drop = [] %}
+
+  {# -- first check whether we want to full refresh for source view or config reasons #}
+  {% set trigger_full_refresh = (full_refresh_mode or existing_relation.is_view) %}
+
   {% if existing_relation is none %}
       {% set build_sql = create_table_as(False, target_relation, sql) %}
-  {% elif existing_relation.is_view or should_full_refresh() %}
+{% elif trigger_full_refresh %}
+      {#-- Make sure the backup doesn't exist so we don't encounter issues with the rename below #}
+      {% set tmp_identifier = model['name'] + '__dbt_tmp' %}
+      {% set backup_identifier = model['name'] + '__dbt_backup' %}
       {% set intermediate_relation = existing_relation.incorporate(path={"identifier": tmp_identifier}) %}
       {% set backup_relation = existing_relation.incorporate(path={"identifier": backup_identifier}) %}
 
@@ -38,12 +49,13 @@
       {% set need_swap = true %}
       {% do to_drop.append(backup_relation) %}
   {% else %}
-      {% set tmp_relation = make_temp_relation(target_relation) %}
-      {% do run_query(create_table_as(True, tmp_relation, sql)) %}
-      {% do adapter.expand_target_column_types(
+    {% do run_query(create_table_as(True, tmp_relation, sql)) %}
+    {% do adapter.expand_target_column_types(
              from_relation=tmp_relation,
              to_relation=target_relation) %}
-      {% set build_sql = incremental_upsert(tmp_relation, target_relation, unique_key=unique_key) %}
+    {% do process_schema_changes(on_schema_change, tmp_relation, existing_relation) %}
+    {% set build_sql = incremental_upsert(tmp_relation, target_relation, unique_key=unique_key) %}
+  
   {% endif %}
 
   {% call statement("main") %}

core/dbt/include/global_project/macros/materializations/incremental/on_schema_change.sql

@@ -0,0 +1,164 @@
+{% macro incremental_validate_on_schema_change(on_schema_change, default='ignore') %}
+   
+   {% if on_schema_change not in ['sync_all_columns', 'append_new_columns', 'fail', 'ignore'] %}
+     
+     {% set log_message = 'Invalid value for on_schema_change (%s) specified. Setting default value of %s.' % (on_schema_change, default) %}
+     {% do log(log_message) %}
+     
+     {{ return(default) }}
+
+   {% else %}
+
+     {{ return(on_schema_change) }}
+   
+   {% endif %}
+
+{% endmacro %}
+
+{% macro diff_columns(source_columns, target_columns) %}
+
+  {% set result = [] %}
+  {% set source_names = source_columns | map(attribute = 'column') | list %}
+  {% set target_names = target_columns | map(attribute = 'column') | list %}
+   
+   {# --check whether the name attribute exists in the target - this does not perform a data type check #}
+   {% for sc in source_columns %}
+     {% if sc.name not in target_names %}
+        {{ result.append(sc) }}
+     {% endif %}
+   {% endfor %}
+  
+  {{ return(result) }}
+
+{% endmacro %}
+
+{% macro diff_column_data_types(source_columns, target_columns) %}
+  
+  {% set result = [] %}
+  {% for sc in source_columns %}
+    {% set tc = target_columns | selectattr("name", "equalto", sc.name) | list | first %}
+    {% if tc %}
+      {% if sc.data_type != tc.data_type %}
+        {{ result.append( { 'column_name': tc.name, 'new_type': sc.data_type } ) }} 
+      {% endif %}
+    {% endif %}
+  {% endfor %}
+
+  {{ return(result) }}
+
+{% endmacro %}
+
+
+{% macro check_for_schema_changes(source_relation, target_relation) %}
+  
+  {% set schema_changed = False %}
+  
+  {%- set source_columns = adapter.get_columns_in_relation(source_relation) -%}
+  {%- set target_columns = adapter.get_columns_in_relation(target_relation) -%}
+  {%- set source_not_in_target = diff_columns(source_columns, target_columns) -%}
+  {%- set target_not_in_source = diff_columns(target_columns, source_columns) -%}
+  
+  {% set new_target_types = diff_column_data_types(source_columns, target_columns) %}
+
+  {% if source_not_in_target != [] %}
+    {% set schema_changed = True %}
+  {% elif target_not_in_source != [] or new_target_types != [] %}
+    {% set schema_changed = True %}
+  {% elif new_target_types != [] %}
+    {% set schema_changed = True %}
+  {% endif %}
+  
+  {% set changes_dict = {
+    'schema_changed': schema_changed,
+    'source_not_in_target': source_not_in_target,
+    'target_not_in_source': target_not_in_source,
+    'new_target_types': new_target_types
+  } %}
+
+  {% set msg %}
+    In {{ target_relation }}:
+        Schema changed: {{ schema_changed }}
+        Source columns not in target: {{ source_not_in_target }}
+        Target columns not in source: {{ target_not_in_source }}
+        New column types: {{ new_target_types }}
+  {% endset %}
+  
+  {% do log(msg) %}
+
+  {{ return(changes_dict) }}
+
+{% endmacro %}
+
+
+{% macro sync_column_schemas(on_schema_change, target_relation, schema_changes_dict) %}
+  
+  {%- set add_to_target_arr = schema_changes_dict['source_not_in_target'] -%}
+
+  {%- if on_schema_change == 'append_new_columns'-%}
+     {%- if add_to_target_arr | length > 0 -%}
+       {%- do alter_relation_add_remove_columns(target_relation, add_to_target_arr, none) -%}
+     {%- endif -%}
+  
+  {% elif on_schema_change == 'sync_all_columns' %}
+     {%- set remove_from_target_arr = schema_changes_dict['target_not_in_source'] -%}
+     {%- set new_target_types = schema_changes_dict['new_target_types'] -%}
+  
+     {% if add_to_target_arr | length > 0 or remove_from_target_arr | length > 0 %} 
+       {%- do alter_relation_add_remove_columns(target_relation, add_to_target_arr, remove_from_target_arr) -%}
+     {% endif %}
+
+     {% if new_target_types != [] %}
+       {% for ntt in new_target_types %}
+         {% set column_name = ntt['column_name'] %}
+         {% set new_type = ntt['new_type'] %}
+         {% do alter_column_type(target_relation, column_name, new_type) %}
+       {% endfor %}
+     {% endif %}
+  
+  {% endif %}
+
+  {% set schema_change_message %}
+    In {{ target_relation }}:
+        Schema change approach: {{ on_schema_change }}
+        Columns added: {{ add_to_target_arr }}
+        Columns removed: {{ remove_from_target_arr }}
+        Data types changed: {{ new_target_types }}
+  {% endset %}
+  
+  {% do log(schema_change_message) %}
+  
+{% endmacro %}
+
+
+{% macro process_schema_changes(on_schema_change, source_relation, target_relation) %}
+    
+    {% if on_schema_change != 'ignore' %}
+    
+      {% set schema_changes_dict = check_for_schema_changes(source_relation, target_relation) %}
+      
+      {% if schema_changes_dict['schema_changed'] %}
+    
+        {% if on_schema_change == 'fail' %}
+        
+          {% set fail_msg %}
+              The source and target schemas on this incremental model are out of sync!
+              They can be reconciled in several ways: 
+                - set the `on_schema_change` config to either append_new_columns or sync_all_columns, depending on your situation.
+                - Re-run the incremental model with `full_refresh: True` to update the target schema.
+                - update the schema manually and re-run the process.
+          {% endset %}
+          
+          {% do exceptions.raise_compiler_error(fail_msg) %}
+        
+        {# -- unless we ignore, run the sync operation per the config #}
+        {% else %}
+          
+          {% do sync_column_schemas(on_schema_change, target_relation, schema_changes_dict) %}
+        
+        {% endif %}
+      
+      {% endif %}
+    
+    {% endif %}
+
+{% endmacro %}

core/dbt/include/global_project/macros/materializations/seed/seed.sql

@@ -1,14 +1,6 @@
 
 {% macro create_csv_table(model, agate_table) -%}
-  {{ adapter.dispatch('create_csv_table')(model, agate_table) }}
-{%- endmacro %}
-
-{% macro reset_csv_table(model, full_refresh, old_relation, agate_table) -%}
-  {{ adapter.dispatch('reset_csv_table')(model, full_refresh, old_relation, agate_table) }}
-{%- endmacro %}
-
-{% macro load_csv_rows(model, agate_table) -%}
-  {{ adapter.dispatch('load_csv_rows')(model, agate_table) }}
+  {{ adapter.dispatch('create_csv_table', 'dbt')(model, agate_table) }}
 {%- endmacro %}
 
 {% macro default__create_csv_table(model, agate_table) %}
@@ -33,6 +25,9 @@
   {{ return(sql) }}
 {% endmacro %}
 
+{% macro reset_csv_table(model, full_refresh, old_relation, agate_table) -%}
+  {{ adapter.dispatch('reset_csv_table', 'dbt')(model, full_refresh, old_relation, agate_table) }}
+{%- endmacro %}
 
 {% macro default__reset_csv_table(model, full_refresh, old_relation, agate_table) %}
     {% set sql = "" %}
@@ -47,6 +42,21 @@
     {{ return(sql) }}
 {% endmacro %}
 
+{% macro get_binding_char() -%}
+  {{ adapter.dispatch('get_binding_char', 'dbt')() }}
+{%- endmacro %}
+
+{% macro default__get_binding_char() %}
+  {{ return('%s') }}
+{% endmacro %}
+
+{% macro get_batch_size() -%}
+  {{ adapter.dispatch('get_batch_size', 'dbt')() }}
+{%- endmacro %}
+
+{% macro default__get_batch_size() %}
+  {{ return(10000) }}
+{% endmacro %}
 
 {% macro get_seed_column_quoted_csv(model, column_names) %}
   {%- set quote_seed_column = model['config'].get('quote_columns', None) -%}
@@ -59,47 +69,47 @@
     {{ return(dest_cols_csv) }}
 {% endmacro %}
 
-
-{% macro basic_load_csv_rows(model, batch_size, agate_table) %}
-    {% set cols_sql = get_seed_column_quoted_csv(model, agate_table.column_names) %}
-    {% set bindings = [] %}
-
-    {% set statements = [] %}
-
-    {% for chunk in agate_table.rows | batch(batch_size) %}
-        {% set bindings = [] %}
-
-        {% for row in chunk %}
-            {% do bindings.extend(row) %}
-        {% endfor %}
-
-        {% set sql %}
-            insert into {{ this.render() }} ({{ cols_sql }}) values
-            {% for row in chunk -%}
-                ({%- for column in agate_table.column_names -%}
-                    %s
-                    {%- if not loop.last%},{%- endif %}
-                {%- endfor -%})
-                {%- if not loop.last%},{%- endif %}
-            {%- endfor %}
-        {% endset %}
-
-        {% do adapter.add_query(sql, bindings=bindings, abridge_sql_log=True) %}
-
-        {% if loop.index0 == 0 %}
-            {% do statements.append(sql) %}
-        {% endif %}
-    {% endfor %}
-
-    {# Return SQL so we can render it out into the compiled files #}
-    {{ return(statements[0]) }}
-{% endmacro %}
-
+{% macro load_csv_rows(model, agate_table) -%}
+  {{ adapter.dispatch('load_csv_rows', 'dbt')(model, agate_table) }}
+{%- endmacro %}
 
 {% macro default__load_csv_rows(model, agate_table) %}
-  {{ return(basic_load_csv_rows(model, 10000, agate_table) )}}
-{% endmacro %}
 
+  {% set batch_size = get_batch_size() %}
+
+  {% set cols_sql = get_seed_column_quoted_csv(model, agate_table.column_names) %}
+  {% set bindings = [] %}
+
+  {% set statements = [] %}
+
+  {% for chunk in agate_table.rows | batch(batch_size) %}
+      {% set bindings = [] %}
+
+      {% for row in chunk %}
+          {% do bindings.extend(row) %}
+      {% endfor %}
+
+      {% set sql %}
+          insert into {{ this.render() }} ({{ cols_sql }}) values
+          {% for row in chunk -%}
+              ({%- for column in agate_table.column_names -%}
+                  {{ get_binding_char() }}
+                  {%- if not loop.last%},{%- endif %}
+              {%- endfor -%})
+              {%- if not loop.last%},{%- endif %}
+          {%- endfor %}
+      {% endset %}
+
+      {% do adapter.add_query(sql, bindings=bindings, abridge_sql_log=True) %}
+
+      {% if loop.index0 == 0 %}
+          {% do statements.append(sql) %}
+      {% endif %}
+  {% endfor %}
+
+  {# Return SQL so we can render it out into the compiled files #}
+  {{ return(statements[0]) }}
+{% endmacro %}
 
 {% materialization seed, default %}
 

core/dbt/include/global_project/macros/materializations/snapshot/snapshot.sql

@@ -2,7 +2,7 @@
     Add new columns to the table if applicable
 #}
 {% macro create_columns(relation, columns) %}
-  {{ adapter.dispatch('create_columns')(relation, columns) }}
+  {{ adapter.dispatch('create_columns', 'dbt')(relation, columns) }}
 {% endmacro %}
 
 {% macro default__create_columns(relation, columns) %}
@@ -15,7 +15,7 @@
 
 
 {% macro post_snapshot(staging_relation) %}
-  {{ adapter.dispatch('post_snapshot')(staging_relation) }}
+  {{ adapter.dispatch('post_snapshot', 'dbt')(staging_relation) }}
 {% endmacro %}
 
 {% macro default__post_snapshot(staging_relation) %}

core/dbt/include/global_project/macros/materializations/snapshot/snapshot_merge.sql

@@ -1,6 +1,6 @@
 
 {% macro snapshot_merge_sql(target, source, insert_cols) -%}
-  {{ adapter.dispatch('snapshot_merge_sql')(target, source, insert_cols) }}
+  {{ adapter.dispatch('snapshot_merge_sql', 'dbt')(target, source, insert_cols) }}
 {%- endmacro %}
 
 
@@ -21,7 +21,6 @@
      and DBT_INTERNAL_SOURCE.dbt_change_type = 'insert'
         then insert ({{ insert_cols_csv }})
         values ({{ insert_cols_csv }})
-    ;
+
 {% endmacro %}
 
-

core/dbt/include/global_project/macros/materializations/snapshot/strategies.sql

@@ -36,7 +36,7 @@
     Create SCD Hash SQL fields cross-db
 #}
 {% macro snapshot_hash_arguments(args) -%}
-  {{ adapter.dispatch('snapshot_hash_arguments')(args) }}
+  {{ adapter.dispatch('snapshot_hash_arguments', 'dbt')(args) }}
 {%- endmacro %}
 
 
@@ -52,7 +52,7 @@
     Get the current time cross-db
 #}
 {% macro snapshot_get_time() -%}
-  {{ adapter.dispatch('snapshot_get_time')() }}
+  {{ adapter.dispatch('snapshot_get_time', 'dbt')() }}
 {%- endmacro %}
 
 {% macro default__snapshot_get_time() -%}
@@ -75,7 +75,7 @@
         table instead of assuming that the user-supplied {{ updated_at }}
         will be present in the historical data.
 
-        See https://github.com/fishtown-analytics/dbt/issues/2350
+        See https://github.com/dbt-labs/dbt/issues/2350
     */ #}
     {% set row_changed_expr -%}
         ({{ snapshotted_rel }}.dbt_valid_from < {{ current_rel }}.{{ updated_at }})
@@ -94,7 +94,7 @@
 
 
 {% macro snapshot_string_as_time(timestamp) -%}
-    {{ adapter.dispatch('snapshot_string_as_time')(timestamp) }}
+    {{ adapter.dispatch('snapshot_string_as_time', 'dbt')(timestamp) }}
 {%- endmacro %}
 
 

core/dbt/include/global_project/macros/materializations/table/table.sql

@@ -48,7 +48,7 @@
 
   -- cleanup
   {% if old_relation is not none %}
-      {{ adapter.rename_relation(target_relation, backup_relation) }}
+      {{ adapter.rename_relation(old_relation, backup_relation) }}
   {% endif %}
 
   {{ adapter.rename_relation(intermediate_relation, target_relation) }}

core/dbt/include/global_project/macros/materializations/test.sql

@@ -1,5 +1,5 @@
 {% macro get_test_sql(main_sql, fail_calc, warn_if, error_if, limit) -%}
-  {{ adapter.dispatch('get_test_sql')(main_sql, fail_calc, warn_if, error_if, limit) }}
+  {{ adapter.dispatch('get_test_sql', 'dbt')(main_sql, fail_calc, warn_if, error_if, limit) }}
 {%- endmacro %}
 
 

core/dbt/include/global_project/macros/materializations/view/create_or_replace_view.sql

@@ -1,9 +1,10 @@
 
 {% macro handle_existing_table(full_refresh, old_relation) %}
-    {{ adapter.dispatch('handle_existing_table', macro_namespace = 'dbt')(full_refresh, old_relation) }}
+    {{ adapter.dispatch('handle_existing_table', 'dbt')(full_refresh, old_relation) }}
 {% endmacro %}
 
 {% macro default__handle_existing_table(full_refresh, old_relation) %}
+    {{ log("Dropping relation " ~ old_relation ~ " because it is of type " ~ old_relation.type) }}
     {{ adapter.drop_relation(old_relation) }}
 {% endmacro %}
 
@@ -19,7 +20,7 @@
     */
 #}
 
-{% macro create_or_replace_view(run_outside_transaction_hooks=True) %}
+{% macro create_or_replace_view() %}
   {%- set identifier = model['alias'] -%}
 
   {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}
@@ -30,13 +31,7 @@
       identifier=identifier, schema=schema, database=database,
       type='view') -%}
 
-  {% if run_outside_transaction_hooks %}
-      -- no transactions on BigQuery
-      {{ run_hooks(pre_hooks, inside_transaction=False) }}
-  {% endif %}
-
-  -- `BEGIN` happens here on Snowflake
-  {{ run_hooks(pre_hooks, inside_transaction=True) }}
+  {{ run_hooks(pre_hooks) }}
 
   -- If there's a table with the same name and we weren't told to full refresh,
   -- that's an error. If we were told to full refresh, drop it. This behavior differs
@@ -50,14 +45,7 @@
     {{ create_view_as(target_relation, sql) }}
   {%- endcall %}
 
-  {{ run_hooks(post_hooks, inside_transaction=True) }}
-
-  {{ adapter.commit() }}
-
-  {% if run_outside_transaction_hooks %}
-      -- No transactions on BigQuery
-      {{ run_hooks(post_hooks, inside_transaction=False) }}
-  {% endif %}
+  {{ run_hooks(post_hooks) }}
 
   {{ return({'relations': [target_relation]}) }}
 

core/dbt/include/global_project/macros/materializations/view/view.sql

@@ -54,7 +54,7 @@
   -- cleanup
   -- move the existing view out of the way
   {% if old_relation is not none %}
-    {{ adapter.rename_relation(target_relation, backup_relation) }}
+    {{ adapter.rename_relation(old_relation, backup_relation) }}
   {% endif %}
   {{ adapter.rename_relation(intermediate_relation, target_relation) }}
 

core/dbt/include/global_project/macros/schema_tests/accepted_values.sql

@@ -28,6 +28,6 @@ where value_field not in (
 
 
 {% test accepted_values(model, column_name, values, quote=True) %}
-    {% set macro = adapter.dispatch('test_accepted_values') %}
+    {% set macro = adapter.dispatch('test_accepted_values', 'dbt') %}
     {{ macro(model, column_name, values, quote) }}
 {% endtest %}

core/dbt/include/global_project/macros/schema_tests/not_null.sql

@@ -8,6 +8,6 @@ where {{ column_name }} is null
 
 
 {% test not_null(model, column_name) %}
-    {% set macro = adapter.dispatch('test_not_null') %}
+    {% set macro = adapter.dispatch('test_not_null', 'dbt') %}
     {{ macro(model, column_name) }}
 {% endtest %}

core/dbt/include/global_project/macros/schema_tests/relationships.sql

@@ -1,21 +1,30 @@
 
 {% macro default__test_relationships(model, column_name, to, field) %}
 
+with child as (
+    select {{ column_name }} as from_field
+    from {{ model }}
+    where {{ column_name }} is not null
+),
+
+parent as (
+    select {{ field }} as to_field
+    from {{ to }}
+)
+
 select
-    child.{{ column_name }}
+    from_field
 
-from {{ model }} as child
+from child
+left join parent
+    on child.from_field = parent.to_field
 
-left join {{ to }} as parent
-    on child.{{ column_name }} = parent.{{ field }}
-
-where child.{{ column_name }} is not null
-  and parent.{{ field }} is null
+where parent.to_field is null
 
 {% endmacro %}
 
 
 {% test relationships(model, column_name, to, field) %}
-    {% set macro = adapter.dispatch('test_relationships') %}
+    {% set macro = adapter.dispatch('test_relationships', 'dbt') %}
     {{ macro(model, column_name, to, field) }}
 {% endtest %}

core/dbt/include/global_project/macros/schema_tests/unique.sql

@@ -1,7 +1,7 @@
 {% macro default__test_unique(model, column_name) %}
 
 select
-    {{ column_name }},
+    {{ column_name }} as unique_field,
     count(*) as n_records
 
 from {{ model }}
@@ -13,6 +13,6 @@ having count(*) > 1
 
 
 {% test unique(model, column_name) %}
-    {% set macro = adapter.dispatch('test_unique') %}
+    {% set macro = adapter.dispatch('test_unique', 'dbt') %}
     {{ macro(model, column_name) }}
 {% endtest %}

core/dbt/logger.py

@@ -43,6 +43,15 @@ DEBUG_LOG_FORMAT = (
     '{record.message}'
 )
 
+SECRET_ENV_PREFIX = 'DBT_ENV_SECRET_'
+
+
+def get_secret_env() -> List[str]:
+    return [
+        v for k, v in os.environ.items()
+        if k.startswith(SECRET_ENV_PREFIX)
+    ]
+
 
 ExceptionInformation = str
 
@@ -333,6 +342,12 @@ class TimestampNamed(logbook.Processor):
         record.extra[self.name] = datetime.utcnow().isoformat()
 
 
+class ScrubSecrets(logbook.Processor):
+    def process(self, record):
+        for secret in get_secret_env():
+            record.message = record.message.replace(secret, "*****")
+
+
 logger = logbook.Logger('dbt')
 # provide this for the cache, disabled by default
 CACHE_LOGGER = logbook.Logger('dbt.cache')
@@ -473,7 +488,8 @@ class LogManager(logbook.NestedSetup):
         self._file_handler = DelayedFileHandler()
         self._relevel_processor = Relevel(allowed=['dbt', 'werkzeug'])
         self._state_processor = DbtProcessState('internal')
-        # keep track of wheter we've already entered to decide if we should
+        self._scrub_processor = ScrubSecrets()
+        # keep track of whether we've already entered to decide if we should
         # be actually pushing. This allows us to log in main() and also
         # support entering dbt execution via handle_and_check.
         self._stack_depth = 0
@@ -483,6 +499,7 @@ class LogManager(logbook.NestedSetup):
             self._file_handler,
             self._relevel_processor,
             self._state_processor,
+            self._scrub_processor
         ])
 
     def push_application(self):

core/dbt/main.py

@@ -11,6 +11,7 @@ from pathlib import Path
 import dbt.version
 import dbt.flags as flags
 import dbt.task.run as run_task
+import dbt.task.build as build_task
 import dbt.task.compile as compile_task
 import dbt.task.debug as debug_task
 import dbt.task.clean as clean_task
@@ -40,6 +41,7 @@ class DBTVersion(argparse.Action):
     """This is very very similar to the builtin argparse._Version action,
     except it just calls dbt.version.get_version_information().
     """
+
     def __init__(self,
                  option_strings,
                  version=None,
@@ -376,6 +378,30 @@ def _build_init_subparser(subparsers, base_subparser):
     return sub
 
 
+def _build_build_subparser(subparsers, base_subparser):
+    sub = subparsers.add_parser(
+        'build',
+        parents=[base_subparser],
+        help='''
+        Run all Seeds, Models, Snapshots, and tests in DAG order
+        '''
+    )
+    sub.set_defaults(
+        cls=build_task.BuildTask,
+        which='build',
+        rpc_method='build'
+    )
+    sub.add_argument(
+        '-x',
+        '--fail-fast',
+        action='store_true',
+        help='''
+        Stop execution upon a first failure.
+        '''
+    )
+    return sub
+
+
 def _build_clean_subparser(subparsers, base_subparser):
     sub = subparsers.add_parser(
         'clean',
@@ -528,39 +554,6 @@ def _build_docs_generate_subparser(subparsers, base_subparser):
     return generate_sub
 
 
-def _add_models_argument(sub, help_override=None, **kwargs):
-    help_str = '''
-        Specify the models to include.
-    '''
-    if help_override is not None:
-        help_str = help_override
-    sub.add_argument(
-        '-m',
-        '--models',
-        dest='models',
-        nargs='+',
-        help=help_str,
-        **kwargs
-    )
-
-
-def _add_select_argument(sub, dest='models', help_override=None, **kwargs):
-    help_str = '''
-        Specify the nodes to include.
-    '''
-    if help_override is not None:
-        help_str = help_override
-
-    sub.add_argument(
-        '-s',
-        '--select',
-        dest=dest,
-        nargs='+',
-        help=help_str,
-        **kwargs
-    )
-
-
 def _add_common_selector_arguments(sub):
     sub.add_argument(
         '--exclude',
@@ -589,17 +582,26 @@ def _add_common_selector_arguments(sub):
     )
 
 
-def _add_selection_arguments(*subparsers, **kwargs):
-    models_name = kwargs.get('models_name', 'models')
+def _add_selection_arguments(*subparsers):
     for sub in subparsers:
-        if models_name == 'models':
-            _add_models_argument(sub)
-        elif models_name == 'select':
-            # these still get stored in 'models', so they present the same
-            # interface to the task
-            _add_select_argument(sub)
-        else:
-            raise InternalException(f'Unknown models style {models_name}')
+        sub.add_argument(
+            '-m',
+            '--models',
+            dest='select',
+            nargs='+',
+            help='''
+                Specify the nodes to include.
+            ''',
+        )
+        sub.add_argument(
+            '-s',
+            '--select',
+            dest='select',
+            nargs='+',
+            help='''
+                Specify the nodes to include.
+            ''',
+        )
         _add_common_selector_arguments(sub)
 
 
@@ -730,23 +732,14 @@ def _build_test_subparser(subparsers, base_subparser):
     return sub
 
 
-def _build_source_snapshot_freshness_subparser(subparsers, base_subparser):
+def _build_source_freshness_subparser(subparsers, base_subparser):
     sub = subparsers.add_parser(
-        'snapshot-freshness',
+        'freshness',
         parents=[base_subparser],
         help='''
         Snapshots the current freshness of the project's sources
         ''',
-    )
-    sub.add_argument(
-        '-s',
-        '--select',
-        required=False,
-        nargs='+',
-        help='''
-        Specify the sources to snapshot freshness
-        ''',
-        dest='selected'
+        aliases=['snapshot-freshness'],
     )
     sub.add_argument(
         '-o',
@@ -767,9 +760,19 @@ def _build_source_snapshot_freshness_subparser(subparsers, base_subparser):
     )
     sub.set_defaults(
         cls=freshness_task.FreshnessTask,
-        which='snapshot-freshness',
-        rpc_method='snapshot-freshness',
+        which='source-freshness',
+        rpc_method='source-freshness',
     )
+    sub.add_argument(
+        '-s',
+        '--select',
+        dest='select',
+        nargs='+',
+        help='''
+            Specify the nodes to include.
+        ''',
+    )
+    _add_common_selector_arguments(sub)
     return sub
 
 
@@ -824,19 +827,28 @@ def _build_list_subparser(subparsers, base_subparser):
     sub.add_argument('--output',
                      choices=['json', 'name', 'path', 'selector'],
                      default='selector')
+    sub.add_argument('--output-keys')
 
-    _add_models_argument(
-        sub,
-        help_override='''
+    sub.add_argument(
+        '-m',
+        '--models',
+        dest='models',
+        nargs='+',
+        help='''
         Specify the models to select and set the resource-type to 'model'.
         Mutually exclusive with '--select' (or '-s') and '--resource-type'
         ''',
         metavar='SELECTOR',
-        required=False
+        required=False,
     )
-    _add_select_argument(
-        sub,
+    sub.add_argument(
+        '-s',
+        '--select',
         dest='select',
+        nargs='+',
+        help='''
+            Specify the nodes to include.
+        ''',
         metavar='SELECTOR',
         required=False,
     )
@@ -982,8 +994,6 @@ def parse_args(args, cls=DBTArgumentParser):
         enable_help='''
         Allow for partial parsing by looking for and writing to a pickle file
         in the target directory. This overrides the user configuration file.
-
-        WARNING: This can result in unexpected behavior if you use env_var()!
         ''',
         disable_help='''
         Disallow partial parsing. This overrides the user configuration file.
@@ -1037,6 +1047,7 @@ def parse_args(args, cls=DBTArgumentParser):
     _build_deps_subparser(subs, base_subparser)
     _build_list_subparser(subs, base_subparser)
 
+    build_sub = _build_build_subparser(subs, base_subparser)
     snapshot_sub = _build_snapshot_subparser(subs, base_subparser)
     rpc_sub = _build_rpc_subparser(subs, base_subparser)
     run_sub = _build_run_subparser(subs, base_subparser)
@@ -1047,18 +1058,18 @@ def parse_args(args, cls=DBTArgumentParser):
     seed_sub = _build_seed_subparser(subs, base_subparser)
     # --threads, --no-version-check
     _add_common_arguments(run_sub, compile_sub, generate_sub, test_sub,
-                          rpc_sub, seed_sub, parse_sub)
-    # --models, --exclude
+                          rpc_sub, seed_sub, parse_sub, build_sub)
+    # --select, --exclude
     # list_sub sets up its own arguments.
-    _add_selection_arguments(run_sub, compile_sub, generate_sub, test_sub)
-    _add_selection_arguments(snapshot_sub, seed_sub, models_name='select')
+    _add_selection_arguments(
+        build_sub, run_sub, compile_sub, generate_sub, test_sub, snapshot_sub, seed_sub)
     # --defer
-    _add_defer_argument(run_sub, test_sub)
+    _add_defer_argument(run_sub, test_sub, build_sub)
     # --full-refresh
-    _add_table_mutability_arguments(run_sub, compile_sub)
+    _add_table_mutability_arguments(run_sub, compile_sub, build_sub)
 
     _build_docs_serve_subparser(docs_subs, base_subparser)
-    _build_source_snapshot_freshness_subparser(source_subs, base_subparser)
+    _build_source_freshness_subparser(source_subs, base_subparser)
     _build_run_operation_subparser(subs, base_subparser)
 
     if len(args) == 0:

core/dbt/parser/base.py

@@ -256,9 +256,7 @@ class ConfiguredParser(
             parsed_node, self.root_project, self.manifest, config
         )
 
-    def render_with_context(
-        self, parsed_node: IntermediateNode, config: ContextConfig
-    ) -> None:
+    def render_with_context(self, parsed_node: IntermediateNode, config: ContextConfig):
         # Given the parsed node and a ContextConfig to use during parsing,
         # render the node's sql wtih macro capture enabled.
         # Note: this mutates the config object when config calls are rendered.
@@ -273,11 +271,12 @@ class ConfiguredParser(
             get_rendered(
                 parsed_node.raw_sql, context, parsed_node, capture_macros=True
             )
+        return context
 
     # This is taking the original config for the node, converting it to a dict,
     # updating the config with new config passed in, then re-creating the
     # config from the dict in the node.
-    def update_parsed_node_config(
+    def update_parsed_node_config_dict(
         self, parsed_node: IntermediateNode, config_dict: Dict[str, Any]
     ) -> None:
         # Overwrite node config
@@ -294,28 +293,50 @@ class ConfiguredParser(
         self._update_node_schema(parsed_node, config_dict)
         self._update_node_alias(parsed_node, config_dict)
 
-    def update_parsed_node(
-        self, parsed_node: IntermediateNode, config: ContextConfig
+    def update_parsed_node_config(
+        self, parsed_node: IntermediateNode, config: ContextConfig,
+        context=None, patch_config_dict=None
     ) -> None:
         """Given the ContextConfig used for parsing and the parsed node,
         generate and set the true values to use, overriding the temporary parse
         values set in _build_intermediate_parsed_node.
         """
-        config_dict = config.build_config_dict()
 
-        # Set tags on node provided in config blocks
+        # build_config_dict takes the config_call_dict in the ContextConfig object
+        # and calls calculate_node_config to combine dbt_project configs and
+        # config calls from SQL files
+        config_dict = config.build_config_dict(patch_config_dict=patch_config_dict)
+
+        # Set tags on node provided in config blocks. Tags are additive, so even if
+        # config has been built before, we don't have to reset tags in the parsed_node.
         model_tags = config_dict.get('tags', [])
-        parsed_node.tags.extend(model_tags)
+        for tag in model_tags:
+            if tag not in parsed_node.tags:
+                parsed_node.tags.append(tag)
 
+        # If we have meta in the config, copy to node level, for backwards
+        # compatibility with earlier node-only config.
+        if 'meta' in config_dict and config_dict['meta']:
+            parsed_node.meta = config_dict['meta']
+
+        # unrendered_config is used to compare the original database/schema/alias
+        # values and to handle 'same_config' and 'same_contents' calls
         parsed_node.unrendered_config = config.build_config_dict(
             rendered=False
         )
 
+        parsed_node.config_call_dict = config._config_call_dict
+
         # do this once before we parse the node database/schema/alias, so
         # parsed_node.config is what it would be if they did nothing
-        self.update_parsed_node_config(parsed_node, config_dict)
+        self.update_parsed_node_config_dict(parsed_node, config_dict)
+        # This updates the node database/schema/alias
         self.update_parsed_node_name(parsed_node, config_dict)
 
+        # tests don't have hooks
+        if parsed_node.resource_type == NodeType.Test:
+            return
+
         # at this point, we've collected our hooks. Use the node context to
         # render each hook and collect refs/sources
         hooks = list(itertools.chain(parsed_node.config.pre_hook,
@@ -323,9 +344,8 @@ class ConfiguredParser(
         # skip context rebuilding if there aren't any hooks
         if not hooks:
             return
-        # we could cache the original context from parsing this node. Is that
-        # worth the cost in memory/complexity?
-        context = self._context_for(parsed_node, config)
+        if not context:
+            context = self._context_for(parsed_node, config)
         for hook in hooks:
             get_rendered(hook.sql, context, parsed_node, capture_macros=True)
 
@@ -357,8 +377,8 @@ class ConfiguredParser(
         self, node: IntermediateNode, config: ContextConfig
     ) -> None:
         try:
-            self.render_with_context(node, config)
-            self.update_parsed_node(node, config)
+            context = self.render_with_context(node, config)
+            self.update_parsed_node_config(node, config, context=context)
         except ValidationError as exc:
             # we got a ValidationError - probably bad types in config()
             msg = validator_error_message(exc)

core/dbt/parser/manifest.py

@@ -1,8 +1,9 @@
 from dataclasses import dataclass
 from dataclasses import field
 import os
+import traceback
 from typing import (
-    Dict, Optional, Mapping, Callable, Any, List, Type, Union
+    Dict, Optional, Mapping, Callable, Any, List, Type, Union, Tuple
 )
 import time
 
@@ -59,13 +60,24 @@ from dbt.parser.sources import SourcePatcher
 from dbt.ui import warning_tag
 from dbt.version import __version__
 
-from dbt.dataclass_schema import dbtClassMixin
+from dbt.dataclass_schema import StrEnum, dbtClassMixin
 
 PARTIAL_PARSE_FILE_NAME = 'partial_parse.msgpack'
 PARSING_STATE = DbtProcessState('parsing')
 DEFAULT_PARTIAL_PARSE = False
 
 
+class ReparseReason(StrEnum):
+    version_mismatch = '01_version_mismatch'
+    file_not_found = '02_file_not_found'
+    vars_changed = '03_vars_changed'
+    profile_changed = '04_profile_changed'
+    deps_changed = '05_deps_changed'
+    project_config_changed = '06_project_config_changed'
+    load_file_failure = '07_load_file_failure'
+    exception = '08_exception'
+
+
 # Part of saved performance info
 @dataclass
 class ParserInfo(dbtClassMixin):
@@ -189,10 +201,6 @@ class ManifestLoader:
         # Read files creates a dictionary of projects to a dictionary
         # of parsers to lists of file strings. The file strings are
         # used to get the SourceFiles from the manifest files.
-        # In the future the loaded files will be used to control
-        # partial parsing, but right now we're just moving the
-        # file loading out of the individual parsers and doing it
-        # all at once.
         start_read_files = time.perf_counter()
         project_parser_files = {}
         for project in self.all_projects.values():
@@ -204,15 +212,51 @@ class ManifestLoader:
         if self.saved_manifest is not None:
             partial_parsing = PartialParsing(self.saved_manifest, self.manifest.files)
             skip_parsing = partial_parsing.skip_parsing()
-            if not skip_parsing:
+            if skip_parsing:
+                # nothing changed, so we don't need to generate project_parser_files
+                self.manifest = self.saved_manifest
+            else:
                 # create child_map and parent_map
                 self.saved_manifest.build_parent_and_child_maps()
                 # files are different, we need to create a new set of
                 # project_parser_files.
-                project_parser_files = partial_parsing.get_parsing_files()
-                self.partially_parsing = True
+                try:
+                    project_parser_files = partial_parsing.get_parsing_files()
+                    self.partially_parsing = True
+                    self.manifest = self.saved_manifest
+                except Exception:
+                    # pp_files should still be the full set and manifest is new manifest,
+                    # since get_parsing_files failed
+                    logger.info("Partial parsing enabled but an error occurred. "
+                                "Switching to a full re-parse.")
 
-            self.manifest = self.saved_manifest
+                    # Get traceback info
+                    tb_info = traceback.format_exc()
+                    formatted_lines = tb_info.splitlines()
+                    (_, line, method) = formatted_lines[-3].split(', ')
+                    exc_info = {
+                        "traceback": tb_info,
+                        "exception": formatted_lines[-1],
+                        "code": formatted_lines[-2],
+                        "location": f"{line} {method}",
+                    }
+
+                    # get file info for local logs
+                    parse_file_type = None
+                    file_id = partial_parsing.processing_file
+                    if file_id and file_id in self.manifest.files:
+                        old_file = self.manifest.files[file_id]
+                        parse_file_type = old_file.parse_file_type
+                        logger.debug(f"Partial parsing exception processing file {file_id}")
+                        file_dict = old_file.to_dict()
+                        logger.debug(f"PP file: {file_dict}")
+                    exc_info['parse_file_type'] = parse_file_type
+                    logger.debug(f"PP exception info: {exc_info}")
+
+                    # Send event
+                    if dbt.tracking.active_user is not None:
+                        exc_info['full_reparse_reason'] = ReparseReason.exception
+                        dbt.tracking.track_partial_parser(exc_info)
 
         if self.manifest._parsing_info is None:
             self.manifest._parsing_info = ParsingInfo()
@@ -379,10 +423,10 @@ class ManifestLoader:
         if not self.partially_parsing and HookParser in parser_types:
             hook_parser = HookParser(project, self.manifest, self.root_project)
             path = hook_parser.get_path()
-            file_block = FileBlock(
-                load_source_file(path, ParseFileType.Hook, project.project_name)
-            )
-            hook_parser.parse_file(file_block)
+            file = load_source_file(path, ParseFileType.Hook, project.project_name)
+            if file:
+                file_block = FileBlock(file)
+                hook_parser.parse_file(file_block)
 
         # Store the performance info
         elapsed = time.perf_counter() - start_timer
@@ -434,6 +478,12 @@ class ManifestLoader:
         path = os.path.join(self.root_project.target_path,
                             PARTIAL_PARSE_FILE_NAME)
         try:
+            # This shouldn't be necessary, but we have gotten bug reports (#3757) of the
+            # saved manifest not matching the code version.
+            if self.manifest.metadata.dbt_version != __version__:
+                logger.debug("Manifest metadata did not contain correct version. "
+                             f"Contained '{self.manifest.metadata.dbt_version}' instead.")
+                self.manifest.metadata.dbt_version = __version__
             manifest_msgpack = self.manifest.to_msgpack()
             make_directory(os.path.dirname(path))
             with open(path, 'wb') as fp:
@@ -441,24 +491,31 @@ class ManifestLoader:
         except Exception:
             raise
 
-    def matching_parse_results(self, manifest: Manifest) -> bool:
+    def is_partial_parsable(self, manifest: Manifest) -> Tuple[bool, Optional[str]]:
         """Compare the global hashes of the read-in parse results' values to
         the known ones, and return if it is ok to re-use the results.
         """
         valid = True
+        reparse_reason = None
 
         if manifest.metadata.dbt_version != __version__:
-            logger.info("Unable to do partial parsing because of a dbt version mismatch")
-            return False  # If the version is wrong, the other checks might not work
+            # #3757 log both versions because of reports of invalid cases of mismatch.
+            logger.info("Unable to do partial parsing because of a dbt version mismatch. "
+                        f"Saved manifest version: {manifest.metadata.dbt_version}. "
+                        f"Current version: {__version__}.")
+            # If the version is wrong, the other checks might not work
+            return False, ReparseReason.version_mismatch
         if self.manifest.state_check.vars_hash != manifest.state_check.vars_hash:
             logger.info("Unable to do partial parsing because config vars, "
                         "config profile, or config target have changed")
             valid = False
+            reparse_reason = ReparseReason.vars_changed
         if self.manifest.state_check.profile_hash != manifest.state_check.profile_hash:
             # Note: This should be made more granular. We shouldn't need to invalidate
             # partial parsing if a non-used profile section has changed.
             logger.info("Unable to do partial parsing because profile has changed")
             valid = False
+            reparse_reason = ReparseReason.profile_changed
 
         missing_keys = {
             k for k in self.manifest.state_check.project_hashes
@@ -467,6 +524,7 @@ class ManifestLoader:
         if missing_keys:
             logger.info("Unable to do partial parsing because a project dependency has been added")
             valid = False
+            reparse_reason = ReparseReason.deps_changed
 
         for key, new_value in self.manifest.state_check.project_hashes.items():
             if key in manifest.state_check.project_hashes:
@@ -475,7 +533,8 @@ class ManifestLoader:
                     logger.info("Unable to do partial parsing because "
                                 "a project config has changed")
                     valid = False
-        return valid
+                    reparse_reason = ReparseReason.project_config_changed
+        return valid, reparse_reason
 
     def _partial_parse_enabled(self):
         # if the CLI is set, follow that
@@ -494,6 +553,8 @@ class ManifestLoader:
         path = os.path.join(self.root_project.target_path,
                             PARTIAL_PARSE_FILE_NAME)
 
+        reparse_reason = None
+
         if os.path.exists(path):
             try:
                 with open(path, 'rb') as fp:
@@ -502,7 +563,8 @@ class ManifestLoader:
                 # keep this check inside the try/except in case something about
                 # the file has changed in weird ways, perhaps due to being a
                 # different version of dbt
-                if self.matching_parse_results(manifest):
+                is_partial_parseable, reparse_reason = self.is_partial_parsable(manifest)
+                if is_partial_parseable:
                     return manifest
             except Exception as exc:
                 logger.debug(
@@ -510,8 +572,13 @@ class ManifestLoader:
                     .format(path, exc),
                     exc_info=True
                 )
+                reparse_reason = ReparseReason.load_file_failure
         else:
             logger.info(f"Unable to do partial parsing because {path} not found")
+            reparse_reason = ReparseReason.file_not_found
+
+        # this event is only fired if a full reparse is needed
+        dbt.tracking.track_partial_parser({'full_reparse_reason': reparse_reason})
 
         return None
 

core/dbt/parser/models.py

@@ -1,15 +1,15 @@
 from dbt.context.context_config import ContextConfig
 from dbt.contracts.graph.parsed import ParsedModelNode
 import dbt.flags as flags
+import dbt.tracking
 from dbt.node_types import NodeType
 from dbt.parser.base import SimpleSQLParser
 from dbt.parser.search import FileBlock
 import dbt.tracking as tracking
 from dbt import utils
 from dbt_extractor import ExtractionError, py_extract_from_source  # type: ignore
-import itertools
 import random
-from typing import Any, Dict, List, Tuple
+from typing import Any, Dict, List
 
 
 class ModelParser(SimpleSQLParser[ParsedModelNode]):
@@ -40,9 +40,9 @@ class ModelParser(SimpleSQLParser[ParsedModelNode]):
                 experimentally_parsed: Dict[str, List[Any]] = py_extract_from_source(node.raw_sql)
 
                 # second config format
-                config_calls: List[Dict[str, str]] = []
+                config_call_dict: Dict[str, Any] = {}
                 for c in experimentally_parsed['configs']:
-                    config_calls.append({c[0]: c[1]})
+                    ContextConfig._add_config_call(config_call_dict, {c[0]: c[1]})
 
                 # format sources TODO change extractor to match this type
                 source_calls: List[List[str]] = []
@@ -64,22 +64,15 @@ class ModelParser(SimpleSQLParser[ParsedModelNode]):
                 if isinstance(experimentally_parsed, Exception):
                     result += ["01_experimental_parser_cannot_parse"]
                 else:
-                    # rearrange existing configs to match:
-                    real_configs: List[Tuple[str, Any]] = list(
-                        itertools.chain.from_iterable(
-                            map(lambda x: x.items(), config._config_calls)
-                        )
-                    )
-
                     # look for false positive configs
-                    for c in experimentally_parsed['configs']:
-                        if c not in real_configs:
+                    for k in config_call_dict.keys():
+                        if k not in config._config_call_dict:
                             result += ["02_false_positive_config_value"]
                             break
 
                     # look for missed configs
-                    for c in real_configs:
-                        if c not in experimentally_parsed['configs']:
+                    for k in config._config_call_dict.keys():
+                        if k not in config_call_dict:
                             result += ["03_missed_config_value"]
                             break
 
@@ -116,33 +109,35 @@ class ModelParser(SimpleSQLParser[ParsedModelNode]):
                 # no false positives or misses, we can expect the number model
                 # files parseable by the experimental parser to match our internal
                 # testing.
-                tracking.track_experimental_parser_sample({
-                    "project_id": self.root_project.hashed_name(),
-                    "file_id": utils.get_hash(node),
-                    "status": result
-                })
+                if dbt.tracking.active_user is not None:  # None in some tests
+                    tracking.track_experimental_parser_sample({
+                        "project_id": self.root_project.hashed_name(),
+                        "file_id": utils.get_hash(node),
+                        "status": result
+                    })
 
         # if the --use-experimental-parser flag was set, and the experimental parser succeeded
         elif not isinstance(experimentally_parsed, Exception):
             # since it doesn't need python jinja, fit the refs, sources, and configs
             # into the node. Down the line the rest of the node will be updated with
             # this information. (e.g. depends_on etc.)
-            config._config_calls = config_calls
+            config._config_call_dict = config_call_dict
 
             # this uses the updated config to set all the right things in the node.
             # if there are hooks present, it WILL render jinja. Will need to change
             # when the experimental parser supports hooks
-            self.update_parsed_node(node, config)
+            self.update_parsed_node_config(node, config)
 
             # update the unrendered config with values from the file.
             # values from yaml files are in there already
             node.unrendered_config.update(dict(experimentally_parsed['configs']))
 
-            # set refs, sources, and configs on the node object
+            # set refs and sources on the node object
             node.refs += experimentally_parsed['refs']
             node.sources += experimentally_parsed['sources']
-            for configv in experimentally_parsed['configs']:
-                node.config[configv[0]] = configv[1]
+
+            # configs don't need to be merged into the node
+            # setting them in config._config_call_dict is sufficient
 
             self.manifest._parsing_info.static_analysis_parsed_path_count += 1
 

core/dbt/parser/partial.py

@@ -46,6 +46,7 @@ class PartialParsing:
         self.deleted_manifest = Manifest()
         self.macro_child_map: Dict[str, List[str]] = {}
         self.build_file_diff()
+        self.processing_file = None
 
     def skip_parsing(self):
         return (
@@ -118,16 +119,21 @@ class PartialParsing:
         # Need to add new files first, because changes in schema files
         # might refer to them
         for file_id in self.file_diff['added']:
+            self.processing_file = file_id
             self.add_to_saved(file_id)
         # Need to process schema files next, because the dictionaries
         # need to be in place for handling SQL file changes
         for file_id in self.file_diff['changed_schema_files']:
+            self.processing_file = file_id
             self.change_schema_file(file_id)
         for file_id in self.file_diff['deleted_schema_files']:
+            self.processing_file = file_id
             self.delete_schema_file(file_id)
         for file_id in self.file_diff['deleted']:
+            self.processing_file = file_id
             self.delete_from_saved(file_id)
         for file_id in self.file_diff['changed']:
+            self.processing_file = file_id
             self.update_in_saved(file_id)
         return self.project_parser_files
 
@@ -147,6 +153,18 @@ class PartialParsing:
                 file_id not in self.file_diff['deleted']):
             self.project_parser_files[project_name][parser_name].append(file_id)
 
+    def already_scheduled_for_parsing(self, source_file):
+        file_id = source_file.file_id
+        project_name = source_file.project_name
+        if project_name not in self.project_parser_files:
+            return False
+        parser_name = parse_file_type_to_parser[source_file.parse_file_type]
+        if parser_name not in self.project_parser_files[project_name]:
+            return False
+        if file_id not in self.project_parser_files[project_name][parser_name]:
+            return False
+        return True
+
     # Add new files, including schema files
     def add_to_saved(self, file_id):
         # add file object to saved manifest.files
@@ -211,6 +229,9 @@ class PartialParsing:
     # Updated schema files should have been processed already.
     def update_mssat_in_saved(self, new_source_file, old_source_file):
 
+        if self.already_scheduled_for_parsing(old_source_file):
+            return
+
         # These files only have one node.
         unique_id = old_source_file.nodes[0]
 
@@ -251,12 +272,16 @@ class PartialParsing:
                         schema_file.node_patches.remove(unique_id)
 
     def update_macro_in_saved(self, new_source_file, old_source_file):
+        if self.already_scheduled_for_parsing(old_source_file):
+            return
         self.handle_macro_file_links(old_source_file, follow_references=True)
         file_id = new_source_file.file_id
         self.saved_files[file_id] = new_source_file
         self.add_to_pp_files(new_source_file)
 
     def update_doc_in_saved(self, new_source_file, old_source_file):
+        if self.already_scheduled_for_parsing(old_source_file):
+            return
         self.delete_doc_node(old_source_file)
         self.saved_files[new_source_file.file_id] = new_source_file
         self.add_to_pp_files(new_source_file)
@@ -343,7 +368,8 @@ class PartialParsing:
         for unique_id in macros:
             if unique_id not in self.saved_manifest.macros:
                 # This happens when a macro has already been removed
-                source_file.macros.remove(unique_id)
+                if unique_id in source_file.macros:
+                    source_file.macros.remove(unique_id)
                 continue
 
             base_macro = self.saved_manifest.macros.pop(unique_id)
@@ -369,7 +395,9 @@ class PartialParsing:
                     macro_patch = self.get_schema_element(macro_patches, base_macro.name)
                     self.delete_schema_macro_patch(schema_file, macro_patch)
                     self.merge_patch(schema_file, 'macros', macro_patch)
-            source_file.macros.remove(unique_id)
+            # The macro may have already been removed by handling macro children
+            if unique_id in source_file.macros:
+                source_file.macros.remove(unique_id)
 
     # similar to schedule_nodes_for_parsing but doesn't do sources and exposures
     # and handles schema tests
@@ -385,12 +413,21 @@ class PartialParsing:
                         patch_list = []
                         if key in schema_file.dict_from_yaml:
                             patch_list = schema_file.dict_from_yaml[key]
-                        node_patch = self.get_schema_element(patch_list, name)
-                        if node_patch:
-                            self.delete_schema_mssa_links(schema_file, key, node_patch)
-                            self.merge_patch(schema_file, key, node_patch)
-                            if unique_id in schema_file.node_patches:
-                                schema_file.node_patches.remove(unique_id)
+                        patch = self.get_schema_element(patch_list, name)
+                        if patch:
+                            if key in ['models', 'seeds', 'snapshots']:
+                                self.delete_schema_mssa_links(schema_file, key, patch)
+                                self.merge_patch(schema_file, key, patch)
+                                if unique_id in schema_file.node_patches:
+                                    schema_file.node_patches.remove(unique_id)
+                            elif key == 'sources':
+                                # re-schedule source
+                                if 'overrides' in patch:
+                                    # This is a source patch; need to re-parse orig source
+                                    self.remove_source_override_target(patch)
+                                self.delete_schema_source(schema_file, patch)
+                                self.remove_tests(schema_file, 'sources', patch['name'])
+                                self.merge_patch(schema_file, 'sources', patch)
                 else:
                     file_id = node.file_id
                     if file_id in self.saved_files and file_id not in self.file_diff['deleted']:
@@ -426,7 +463,13 @@ class PartialParsing:
         new_schema_file = self.new_files[file_id]
         saved_yaml_dict = saved_schema_file.dict_from_yaml
         new_yaml_dict = new_schema_file.dict_from_yaml
-        saved_schema_file.pp_dict = {"version": saved_yaml_dict['version']}
+        if 'version' in new_yaml_dict:
+            # despite the fact that this goes in the saved_schema_file, it
+            # should represent the new yaml dictionary, and should produce
+            # an error if the updated yaml file doesn't have a version
+            saved_schema_file.pp_dict = {"version": new_yaml_dict['version']}
+        else:
+            saved_schema_file.pp_dict = {}
         self.handle_schema_file_changes(saved_schema_file, saved_yaml_dict, new_yaml_dict)
 
         # copy from new schema_file to saved_schema_file to preserve references
@@ -611,8 +654,9 @@ class PartialParsing:
     def remove_tests(self, schema_file, dict_key, name):
         tests = schema_file.get_tests(dict_key, name)
         for test_unique_id in tests:
-            node = self.saved_manifest.nodes.pop(test_unique_id)
-            self.deleted_manifest.nodes[test_unique_id] = node
+            if test_unique_id in self.saved_manifest.nodes:
+                node = self.saved_manifest.nodes.pop(test_unique_id)
+                self.deleted_manifest.nodes[test_unique_id] = node
         schema_file.remove_tests(dict_key, name)
 
     def delete_schema_source(self, schema_file, source_dict):
@@ -634,19 +678,17 @@ class PartialParsing:
 
     def delete_schema_macro_patch(self, schema_file, macro):
         # This is just macro patches that need to be reapplied
-        for unique_id in schema_file.macro_patches:
-            parts = unique_id.split('.')
-            macro_name = parts[-1]
-            if macro_name == macro['name']:
-                macro_unique_id = unique_id
-                break
+        macro_unique_id = None
+        if macro['name'] in schema_file.macro_patches:
+            macro_unique_id = schema_file.macro_patches[macro['name']]
+            del schema_file.macro_patches[macro['name']]
         if macro_unique_id and macro_unique_id in self.saved_manifest.macros:
             macro = self.saved_manifest.macros.pop(macro_unique_id)
             self.deleted_manifest.macros[macro_unique_id] = macro
             macro_file_id = macro.file_id
-            self.add_to_pp_files(self.saved_files[macro_file_id])
-        if macro_unique_id in schema_file.macro_patches:
-            schema_file.macro_patches.remove(macro_unique_id)
+            if macro_file_id in self.new_files:
+                self.saved_files[macro_file_id] = self.new_files[macro_file_id]
+                self.add_to_pp_files(self.saved_files[macro_file_id])
 
     # exposures are created only from schema files, so just delete
     # the exposure.

core/dbt/parser/read_files.py

@@ -6,12 +6,13 @@ from dbt.contracts.files import (
 from dbt.parser.schemas import yaml_from_file, schema_file_keys, check_format_version
 from dbt.exceptions import CompilationException
 from dbt.parser.search import FilesystemSearcher
+from typing import Optional
 
 
 # This loads the files contents and creates the SourceFile object
 def load_source_file(
         path: FilePath, parse_file_type: ParseFileType,
-        project_name: str) -> AnySourceFile:
+        project_name: str) -> Optional[AnySourceFile]:
     file_contents = load_file_contents(path.absolute_path, strip=False)
     checksum = FileHash.from_contents(file_contents)
     sf_cls = SchemaSourceFile if parse_file_type == ParseFileType.Schema else SourceFile
@@ -20,8 +21,11 @@ def load_source_file(
     source_file.contents = file_contents.strip()
     if parse_file_type == ParseFileType.Schema and source_file.contents:
         dfy = yaml_from_file(source_file)
-        validate_yaml(source_file.path.original_file_path, dfy)
-        source_file.dfy = dfy
+        if dfy:
+            validate_yaml(source_file.path.original_file_path, dfy)
+            source_file.dfy = dfy
+        else:
+            source_file = None
     return source_file
 
 
@@ -76,8 +80,10 @@ def get_source_files(project, paths, extension, parse_file_type):
         if parse_file_type == ParseFileType.Seed:
             fb_list.append(load_seed_source_file(fp, project.project_name))
         else:
-            fb_list.append(load_source_file(
-                fp, parse_file_type, project.project_name))
+            file = load_source_file(fp, parse_file_type, project.project_name)
+            # only append the list if it has contents. added to fix #3568
+            if file:
+                fb_list.append(file)
     return fb_list
 
 

core/dbt/parser/schema_test_builders.py

@@ -190,9 +190,9 @@ class TestBuilder(Generic[Testable]):
         r'(?P<test_name>([a-zA-Z_][0-9a-zA-Z_]*))'
     )
     # kwargs representing test configs
-    MODIFIER_ARGS = (
+    CONFIG_ARGS = (
         'severity', 'tags', 'enabled', 'where', 'limit', 'warn_if', 'error_if',
-        'fail_calc', 'store_failures'
+        'fail_calc', 'store_failures', 'meta', 'database', 'schema', 'alias',
     )
 
     def __init__(
@@ -224,13 +224,24 @@ class TestBuilder(Generic[Testable]):
         groups = match.groupdict()
         self.name: str = groups['test_name']
         self.namespace: str = groups['test_namespace']
-        self.modifiers: Dict[str, Any] = {}
-        for key in self.MODIFIER_ARGS:
+        self.config: Dict[str, Any] = {}
+
+        for key in self.CONFIG_ARGS:
             value = self.args.pop(key, None)
+            # 'modifier' config could be either top level arg or in config
+            if value and 'config' in self.args and key in self.args['config']:
+                raise_compiler_error(
+                    'Test cannot have the same key at the top-level and in config'
+                )
+            if not value and 'config' in self.args:
+                value = self.args['config'].pop(key, None)
             if isinstance(value, str):
                 value = get_rendered(value, render_ctx, native=True)
             if value is not None:
-                self.modifiers[key] = value
+                self.config[key] = value
+
+        if 'config' in self.args:
+            del self.args['config']
 
         if self.namespace is not None:
             self.package_name = self.namespace
@@ -240,8 +251,8 @@ class TestBuilder(Generic[Testable]):
         self.fqn_name: str = fqn_name
 
         # use hashed name as alias if too long
-        if compiled_name != fqn_name:
-            self.modifiers['alias'] = compiled_name
+        if compiled_name != fqn_name and 'alias' not in self.config:
+            self.config['alias'] = compiled_name
 
     def _bad_type(self) -> TypeError:
         return TypeError('invalid target type "{}"'.format(type(self.target)))
@@ -282,15 +293,15 @@ class TestBuilder(Generic[Testable]):
 
     @property
     def enabled(self) -> Optional[bool]:
-        return self.modifiers.get('enabled')
+        return self.config.get('enabled')
 
     @property
     def alias(self) -> Optional[str]:
-        return self.modifiers.get('alias')
+        return self.config.get('alias')
 
     @property
     def severity(self) -> Optional[str]:
-        sev = self.modifiers.get('severity')
+        sev = self.config.get('severity')
         if sev:
             return sev.upper()
         else:
@@ -298,30 +309,72 @@ class TestBuilder(Generic[Testable]):
 
     @property
     def store_failures(self) -> Optional[bool]:
-        return self.modifiers.get('store_failures')
+        return self.config.get('store_failures')
 
     @property
     def where(self) -> Optional[str]:
-        return self.modifiers.get('where')
+        return self.config.get('where')
 
     @property
     def limit(self) -> Optional[int]:
-        return self.modifiers.get('limit')
+        return self.config.get('limit')
 
     @property
     def warn_if(self) -> Optional[str]:
-        return self.modifiers.get('warn_if')
+        return self.config.get('warn_if')
 
     @property
     def error_if(self) -> Optional[str]:
-        return self.modifiers.get('error_if')
+        return self.config.get('error_if')
 
     @property
     def fail_calc(self) -> Optional[str]:
-        return self.modifiers.get('fail_calc')
+        return self.config.get('fail_calc')
+
+    @property
+    def meta(self) -> Optional[dict]:
+        return self.config.get('meta')
+
+    @property
+    def database(self) -> Optional[str]:
+        return self.config.get('database')
+
+    @property
+    def schema(self) -> Optional[str]:
+        return self.config.get('schema')
+
+    def get_static_config(self):
+        config = {}
+        if self.alias is not None:
+            config['alias'] = self.alias
+        if self.severity is not None:
+            config['severity'] = self.severity
+        if self.enabled is not None:
+            config['enabled'] = self.enabled
+        if self.where is not None:
+            config['where'] = self.where
+        if self.limit is not None:
+            config['limit'] = self.limit
+        if self.warn_if is not None:
+            config['warn_if'] = self.warn_if
+        if self.error_if is not None:
+            config['error_id'] = self.error_if
+        if self.fail_calc is not None:
+            config['fail_calc'] = self.fail_calc
+        if self.store_failures is not None:
+            config['store_failures'] = self.store_failures
+        if self.meta is not None:
+            config['meta'] = self.meta
+        if self.database is not None:
+            config['database'] = self.database
+        if self.schema is not None:
+            config['schema'] = self.schema
+        if self.alias is not None:
+            config['alias'] = self.alias
+        return config
 
     def tags(self) -> List[str]:
-        tags = self.modifiers.get('tags', [])
+        tags = self.config.get('tags', [])
         if isinstance(tags, str):
             tags = [tags]
         if not isinstance(tags, list):
@@ -360,7 +413,7 @@ class TestBuilder(Generic[Testable]):
                 else str(value)
             )
             for key, value
-            in self.modifiers.items()
+            in self.config.items()
         ])
         if configs:
             return f"{{{{ config({configs}) }}}}"
@@ -380,12 +433,8 @@ class TestBuilder(Generic[Testable]):
 
     def build_model_str(self):
         targ = self.target
-        cfg_where = "config.get('where')"
         if isinstance(self.target, UnparsedNodeUpdate):
-            identifier = self.target.name
-            target_str = f"{{{{ ref('{targ.name}') }}}}"
+            target_str = f"ref('{targ.name}')"
         elif isinstance(self.target, UnpatchedSourceDefinition):
-            identifier = self.target.table.name
-            target_str = f"{{{{ source('{targ.source.name}', '{targ.table.name}') }}}}"
-        filtered = f"(select * from {target_str} where {{{{{cfg_where}}}}}) {identifier}"
-        return f"{{% if {cfg_where} %}}{filtered}{{% else %}}{target_str}{{% endif %}}"
+            target_str = f"source('{targ.source.name}', '{targ.table.name}')"
+        return f"{{{{ get_where_subquery({target_str}) }}}}"

core/dbt/parser/schemas.py

@@ -22,8 +22,7 @@ from dbt.context.providers import (
     generate_parse_exposure, generate_test_context
 )
 from dbt.context.macro_resolver import MacroResolver
-from dbt.contracts.files import FileHash
-from dbt.contracts.graph.manifest import SchemaSourceFile
+from dbt.contracts.files import FileHash, SchemaSourceFile
 from dbt.contracts.graph.parsed import (
     ParsedNodePatch,
     ColumnInfo,
@@ -47,7 +46,10 @@ from dbt.contracts.graph.unparsed import (
 from dbt.exceptions import (
     validator_error_message, JSONValidationException,
     raise_invalid_schema_yml_version, ValidationException,
-    CompilationException,
+    CompilationException, raise_duplicate_patch_name,
+    raise_duplicate_macro_patch_name, InternalException,
+    raise_duplicate_source_patch_name,
+    warn_or_error,
 )
 from dbt.node_types import NodeType
 from dbt.parser.base import SimpleParser
@@ -171,15 +173,15 @@ class SchemaParser(SimpleParser[SchemaTestBlock, ParsedSchemaTestNode]):
             self.project.config_version == 2
         )
         if all_v_2:
-            ctx = generate_schema_yml(
+            self.render_ctx = generate_schema_yml(
                 self.root_project, self.project.project_name
             )
         else:
-            ctx = generate_target_context(
+            self.render_ctx = generate_target_context(
                 self.root_project, self.root_project.cli_vars
             )
 
-        self.raw_renderer = SchemaYamlRenderer(ctx)
+        self.raw_renderer = SchemaYamlRenderer(self.render_ctx)
 
         internal_package_names = get_adapter_package_names(
             self.root_project.credentials.type
@@ -287,17 +289,13 @@ class SchemaParser(SimpleParser[SchemaTestBlock, ParsedSchemaTestNode]):
         tags: List[str],
         column_name: Optional[str],
     ) -> ParsedSchemaTestNode:
-
-        render_ctx = generate_target_context(
-            self.root_project, self.root_project.cli_vars
-        )
         try:
             builder = TestBuilder(
                 test=test,
                 target=target,
                 column_name=column_name,
                 package_name=target.package_name,
-                render_ctx=render_ctx,
+                render_ctx=self.render_ctx,
             )
         except CompilationException as exc:
             context = _trimmed(str(target))
@@ -318,8 +316,8 @@ class SchemaParser(SimpleParser[SchemaTestBlock, ParsedSchemaTestNode]):
         # is not necessarily this package's name
         fqn = self.get_fqn(fqn_path, builder.fqn_name)
 
-        # this is the config that is used in render_update
-        config = self.initial_config(fqn)
+        # this is the ContextConfig that is used in render_update
+        config: ContextConfig = self.initial_config(fqn)
 
         metadata = {
             'namespace': builder.namespace,
@@ -360,34 +358,10 @@ class SchemaParser(SimpleParser[SchemaTestBlock, ParsedSchemaTestNode]):
         node.depends_on.add_macro(macro_unique_id)
         if (macro_unique_id in
                 ['macro.dbt.test_not_null', 'macro.dbt.test_unique']):
-            self.update_parsed_node(node, config)
-            # manually set configs
-            # note: this does not respect generate_alias_name() macro
-            if builder.alias is not None:
-                node.unrendered_config['alias'] = builder.alias
-                node.config['alias'] = builder.alias
-                node.alias = builder.alias
-            if builder.severity is not None:
-                node.unrendered_config['severity'] = builder.severity
-                node.config['severity'] = builder.severity
-            if builder.enabled is not None:
-                node.unrendered_config['enabled'] = builder.enabled
-                node.config['enabled'] = builder.enabled
-            if builder.where is not None:
-                node.unrendered_config['where'] = builder.where
-                node.config['where'] = builder.where
-            if builder.limit is not None:
-                node.unrendered_config['limit'] = builder.limit
-                node.config['limit'] = builder.limit
-            if builder.warn_if is not None:
-                node.unrendered_config['warn_if'] = builder.warn_if
-                node.config['warn_if'] = builder.warn_if
-            if builder.error_if is not None:
-                node.unrendered_config['error_if'] = builder.error_if
-                node.config['error_if'] = builder.error_if
-            if builder.fail_calc is not None:
-                node.unrendered_config['fail_calc'] = builder.fail_calc
-                node.config['fail_calc'] = builder.fail_calc
+            config_call_dict = builder.get_static_config()
+            config._config_call_dict = config_call_dict
+            # This sets the config from dbt_project
+            self.update_parsed_node_config(node, config)
             # source node tests are processed at patch_source time
             if isinstance(builder.target, UnpatchedSourceDefinition):
                 sources = [builder.target.fqn[-2], builder.target.fqn[-1]]
@@ -407,7 +381,7 @@ class SchemaParser(SimpleParser[SchemaTestBlock, ParsedSchemaTestNode]):
                 get_rendered(
                     node.raw_sql, context, node, capture_macros=True
                 )
-                self.update_parsed_node(node, config)
+                self.update_parsed_node_config(node, config)
             except ValidationError as exc:
                 # we got a ValidationError - probably bad types in config()
                 msg = validator_error_message(exc)
@@ -675,7 +649,14 @@ class SourceParser(YamlDocsReader):
             if is_override:
                 data['path'] = self.yaml.path.original_file_path
                 patch = self._target_from_dict(SourcePatch, data)
-                self.manifest.add_source_patch(self.yaml.file, patch)
+                assert isinstance(self.yaml.file, SchemaSourceFile)
+                source_file = self.yaml.file
+                # source patches must be unique
+                key = (patch.overrides, patch.name)
+                if key in self.manifest.source_patches:
+                    raise_duplicate_source_patch_name(patch, self.manifest.source_patches[key])
+                self.manifest.source_patches[key] = patch
+                source_file.source_patches.append(key)
             else:
                 source = self._target_from_dict(UnparsedSourceDefinition, data)
                 self.add_source_definitions(source)
@@ -772,6 +753,9 @@ class NonSourceParser(YamlDocsReader, Generic[NonSourceTarget, Parsed]):
                 # target_type: UnparsedNodeUpdate, UnparsedAnalysisUpdate,
                 # or UnparsedMacroUpdate
                 self._target_type().validate(data)
+                if self.key != 'macros':
+                    # macros don't have the 'config' key support yet
+                    self.normalize_meta_attribute(data, path)
                 node = self._target_type().from_dict(data)
             except (ValidationError, JSONValidationException) as exc:
                 msg = error_context(path, self.key, data, exc)
@@ -779,6 +763,33 @@ class NonSourceParser(YamlDocsReader, Generic[NonSourceTarget, Parsed]):
             else:
                 yield node
 
+    # We want to raise an error if 'meta' is in two places, and move 'meta'
+    # from toplevel to config if necessary
+    def normalize_meta_attribute(self, data, path):
+        if 'meta' in data:
+            if 'config' in data and 'meta' in data['config']:
+                raise CompilationException(f"""
+                    In {path}: found meta dictionary in 'config' dictionary and as top-level key.
+                    Remove the top-level key and define it under 'config' dictionary only.
+                """.strip())
+            else:
+                if 'config' not in data:
+                    data['config'] = {}
+                data['config']['meta'] = data.pop('meta')
+
+    def patch_node_config(self, node, patch):
+        # Get the ContextConfig that's used in calculating the config
+        # This must match the model resource_type that's being patched
+        config = ContextConfig(
+            self.schema_parser.root_project,
+            node.fqn,
+            node.resource_type,
+            self.schema_parser.project.project_name,
+        )
+        # We need to re-apply the config_call_dict after the patch config
+        config._config_call_dict = node.config_call_dict
+        self.schema_parser.update_parsed_node_config(node, config, patch_config_dict=patch.config)
+
 
 class NodePatchParser(
     NonSourceParser[NodeTarget, ParsedNodePatch],
@@ -787,6 +798,9 @@ class NodePatchParser(
     def parse_patch(
         self, block: TargetBlock[NodeTarget], refs: ParserRef
     ) -> None:
+        # We're not passing the ParsedNodePatch around anymore, so we
+        # could possibly skip creating one. Leaving here for now for
+        # code consistency.
         patch = ParsedNodePatch(
             name=block.target.name,
             original_file_path=block.target.original_file_path,
@@ -796,8 +810,35 @@ class NodePatchParser(
             columns=refs.column_info,
             meta=block.target.meta,
             docs=block.target.docs,
+            config=block.target.config,
         )
-        self.manifest.add_patch(self.yaml.file, patch)
+        assert isinstance(self.yaml.file, SchemaSourceFile)
+        source_file: SchemaSourceFile = self.yaml.file
+        if patch.yaml_key in ['models', 'seeds', 'snapshots']:
+            unique_id = self.manifest.ref_lookup.get_unique_id(patch.name, None)
+        elif patch.yaml_key == 'analyses':
+            unique_id = self.manifest.analysis_lookup.get_unique_id(patch.name, None)
+        else:
+            raise InternalException(
+                f'Unexpected yaml_key {patch.yaml_key} for patch in '
+                f'file {source_file.path.original_file_path}'
+            )
+        if unique_id is None:
+            # This will usually happen when a node is disabled
+            return
+
+        # patches can't be overwritten
+        node = self.manifest.nodes.get(unique_id)
+        if node:
+            if node.patch_path:
+                package_name, existing_file_path = node.patch_path.split('://')
+                raise_duplicate_patch_name(patch, existing_file_path)
+            source_file.append_patch(patch.yaml_key, unique_id)
+            # If this patch has config changes, re-calculate the node config
+            # with the patch config
+            if patch.config:
+                self.patch_node_config(node, patch)
+            node.patch(patch)
 
 
 class TestablePatchParser(NodePatchParser[UnparsedNodeUpdate]):
@@ -835,8 +876,24 @@ class MacroPatchParser(NonSourceParser[UnparsedMacroUpdate, ParsedMacroPatch]):
             description=block.target.description,
             meta=block.target.meta,
             docs=block.target.docs,
+            config=block.target.config,
         )
-        self.manifest.add_macro_patch(self.yaml.file, patch)
+        assert isinstance(self.yaml.file, SchemaSourceFile)
+        source_file = self.yaml.file
+        # macros are fully namespaced
+        unique_id = f'macro.{patch.package_name}.{patch.name}'
+        macro = self.manifest.macros.get(unique_id)
+        if not macro:
+            warn_or_error(
+                f'WARNING: Found patch for macro "{patch.name}" '
+                f'which was not found'
+            )
+            return
+        if macro.patch_path:
+            package_name, existing_file_path = macro.patch_path.split('://')
+            raise_duplicate_macro_patch_name(patch, existing_file_path)
+        source_file.macro_patches[patch.name] = unique_id
+        macro.patch(patch)
 
 
 class ExposureParser(YamlReader):

core/dbt/parser/sources.py

@@ -286,7 +286,7 @@ class SourcePatcher:
             )
 
         return generator.calculate_node_config(
-            config_calls=[],
+            config_call_dict={},
             fqn=fqn,
             resource_type=NodeType.Source,
             project_name=project_name,

core/dbt/rpc/method.py

@@ -1,6 +1,5 @@
 import inspect
 from abc import abstractmethod
-from copy import deepcopy
 from typing import List, Optional, Type, TypeVar, Generic, Dict, Any
 
 from dbt.dataclass_schema import dbtClassMixin, ValidationError
@@ -21,7 +20,7 @@ class RemoteMethod(Generic[Parameters, Result]):
     METHOD_NAME: Optional[str] = None
 
     def __init__(self, args, config):
-        self.args = deepcopy(args)
+        self.args = args
         self.config = config
 
     @classmethod

core/dbt/rpc/task_manager.py

@@ -1,3 +1,4 @@
+from copy import deepcopy
 import threading
 import uuid
 from datetime import datetime
@@ -155,7 +156,7 @@ class TaskManager:
                     f'Manifest should not be None if the last parse state is '
                     f'{state}'
                 )
-            return task(self.args, self.config, self.manifest)
+            return task(deepcopy(self.args), self.config, self.manifest)
 
     def rpc_task(
         self, method_name: str
@@ -167,7 +168,7 @@ class TaskManager:
             elif issubclass(task, RemoteManifestMethod):
                 return self._get_manifest_callable(task)
             elif issubclass(task, RemoteMethod):
-                return task(self.args, self.config)
+                return task(deepcopy(self.args), self.config)
             else:
                 raise dbt.exceptions.InternalException(
                     f'Got a task with an invalid type! {task} with method '

core/dbt/semver.py

@@ -1,5 +1,8 @@
 from dataclasses import dataclass
 import re
+from typing import List
+
+from packaging import version as packaging_version
 
 from dbt.exceptions import VersionsNotCompatibleException
 import dbt.utils
@@ -125,12 +128,26 @@ class VersionSpecifier(VersionSpecification):
         if self.is_unbounded or other.is_unbounded:
             return 0
 
-        for key in ['major', 'minor', 'patch']:
-            comparison = int(getattr(self, key)) - int(getattr(other, key))
-
-            if comparison > 0:
+        for key in ['major', 'minor', 'patch', 'prerelease']:
+            (a, b) = (getattr(self, key), getattr(other, key))
+            if key == 'prerelease':
+                if a is None and b is None:
+                    continue
+                if a is None:
+                    if self.matcher == Matchers.LESS_THAN:
+                        # If 'a' is not a pre-release but 'b' is, and b must be
+                        # less than a, return -1 to prevent installations of
+                        # pre-releases with greater base version than a
+                        # maximum specified non-pre-release version.
+                        return -1
+                    # Otherwise, stable releases are considered greater than
+                    # pre-release
+                    return 1
+                if b is None:
+                    return -1
+            if packaging_version.parse(a) > packaging_version.parse(b):
                 return 1
-            elif comparison < 0:
+            elif packaging_version.parse(a) < packaging_version.parse(b):
                 return -1
 
         equal = ((self.matcher == Matchers.GREATER_THAN_OR_EQUAL and
@@ -408,10 +425,27 @@ def resolve_to_specific_version(requested_range, available_versions):
         version = VersionSpecifier.from_version_string(version_string)
 
         if(versions_compatible(version,
-                               requested_range.start,
-                               requested_range.end) and
+           requested_range.start, requested_range.end) and
            (max_version is None or max_version.compare(version) < 0)):
             max_version = version
             max_version_string = version_string
 
     return max_version_string
+
+
+def filter_installable(
+        versions: List[str],
+        install_prerelease: bool
+) -> List[str]:
+    installable = []
+    installable_dict = {}
+    for version_string in versions:
+        version = VersionSpecifier.from_version_string(version_string)
+        if install_prerelease or not version.prerelease:
+            installable.append(version)
+            installable_dict[str(version)] = version_string
+    sorted_installable = sorted(installable)
+    sorted_installable_original_versions = [
+        str(installable_dict.get(str(version))) for version in sorted_installable
+    ]
+    return sorted_installable_original_versions

core/dbt/task/base.py

@@ -158,7 +158,7 @@ class ConfiguredTask(BaseTask):
 
 
 INTERNAL_ERROR_STRING = """This is an error in dbt. Please try again. If \
-the error persists, open an issue at https://github.com/fishtown-analytics/dbt
+the error persists, open an issue at https://github.com/dbt-labs/dbt
 """.strip()
 
 

core/dbt/task/build.py

@@ -0,0 +1,41 @@
+from .run import RunTask, ModelRunner as run_model_runner
+from .snapshot import SnapshotRunner as snapshot_model_runner
+from .seed import SeedRunner as seed_runner
+from .test import TestRunner as test_runner
+
+from dbt.graph import ResourceTypeSelector
+from dbt.exceptions import InternalException
+from dbt.node_types import NodeType
+
+
+class BuildTask(RunTask):
+    """The Build task processes all assets of a given process and attempts to 'build'
+    them in an opinionated fashion.  Every resource type outlined in RUNNER_MAP
+    will be processed by the mapped runner class.
+
+    I.E. a resource of type Model is handled by the ModelRunner which is imported
+    as run_model_runner.
+    """
+
+    RUNNER_MAP = {
+        NodeType.Model: run_model_runner,
+        NodeType.Snapshot: snapshot_model_runner,
+        NodeType.Seed: seed_runner,
+        NodeType.Test: test_runner,
+    }
+
+    def get_node_selector(self) -> ResourceTypeSelector:
+        if self.manifest is None or self.graph is None:
+            raise InternalException(
+                'manifest and graph must be set to get node selection'
+            )
+
+        return ResourceTypeSelector(
+            graph=self.graph,
+            manifest=self.manifest,
+            previous_state=self.previous_state,
+            resource_types=[x for x in self.RUNNER_MAP.keys()],
+        )
+
+    def get_runner_type(self, node):
+        return self.RUNNER_MAP.get(node.resource_type)

core/dbt/task/compile.py

@@ -41,7 +41,7 @@ class CompileTask(GraphRunnableTask):
         if self.args.selector_name:
             spec = self.config.get_selector(self.args.selector_name)
         else:
-            spec = parse_difference(self.args.models, self.args.exclude)
+            spec = parse_difference(self.args.select, self.args.exclude)
         return spec
 
     def get_node_selector(self) -> ResourceTypeSelector:
@@ -56,7 +56,7 @@ class CompileTask(GraphRunnableTask):
             resource_types=NodeType.executable(),
         )
 
-    def get_runner_type(self):
+    def get_runner_type(self, _):
         return CompileRunner
 
     def task_end_messages(self, results):

core/dbt/task/debug.py

@@ -16,6 +16,7 @@ from dbt.context.target import generate_target_context
 from dbt.clients.yaml_helper import load_yaml_text
 from dbt.links import ProfileConfigDocs
 from dbt.ui import green, red
+from dbt.utils import pluralize
 from dbt.version import get_installed_version
 
 from dbt.task.base import BaseTask, get_nearest_project_dir
@@ -125,6 +126,11 @@ class DebugTask(BaseTask):
         self.test_dependencies()
         self.test_connection()
 
+        if self.any_failure:
+            print(red(f"{(pluralize(len(self.messages), 'check'))} failed:"))
+        else:
+            print(green('All checks passed!'))
+
         for message in self.messages:
             print(message)
             print('')
@@ -321,9 +327,12 @@ class DebugTask(BaseTask):
         self.any_failure = True
         if self.project_fail_details == FILE_NOT_FOUND:
             return
-        print('Project loading failed for the following reason:')
-        print(self.project_fail_details)
-        print('')
+        msg = (
+            f'Project loading failed for the following reason:'
+            f'\n{self.project_fail_details}'
+            f'\n'
+        )
+        self.messages.append(msg)
 
     def _log_profile_fail(self):
         if not self.profile_fail_details:
@@ -332,9 +341,12 @@ class DebugTask(BaseTask):
         self.any_failure = True
         if self.profile_fail_details == FILE_NOT_FOUND:
             return
-        print('Profile loading failed for the following reason:')
-        print(self.profile_fail_details)
-        print('')
+        msg = (
+            f'Profile loading failed for the following reason:'
+            f'\n{self.profile_fail_details}'
+            f'\n'
+        )
+        self.messages.append(msg)
 
     @staticmethod
     def attempt_connection(profile):
@@ -368,7 +380,7 @@ class DebugTask(BaseTask):
         print('Connection:')
         for k, v in self.profile.credentials.connection_info():
             print('  {}: {}'.format(k, v))
-        print('  Connection test: {}'.format(self._connection_result()))
+        print('  Connection test: [{}]'.format(self._connection_result()))
         print('')
 
     @classmethod

core/dbt/task/deps.py

@@ -56,19 +56,36 @@ class DepsTask(BaseTask):
                 self.config, self.config.cli_vars
             ))
 
+            packages_to_upgrade = []
             for package in final_deps:
+                package_name = package.name
+                source_type = package.source_type()
+                version = package.get_version()
+
                 logger.info('Installing {}', package)
                 package.install(self.config, renderer)
                 logger.info('  Installed from {}',
                             package.nice_version_name())
+                if source_type == 'hub':
+                    version_latest = package.get_version_latest()
+                    if version_latest != version:
+                        packages_to_upgrade.append(package_name)
+                        logger.info('  Updated version available: {}',
+                                    version_latest)
+                    else:
+                        logger.info('  Up to date!')
                 if package.get_subdirectory():
-                    logger.info('   and subdirectory {}\n',
+                    logger.info('   and subdirectory {}',
                                 package.get_subdirectory())
 
                 self.track_package_install(
-                    package_name=package.name,
-                    source_type=package.source_type(),
-                    version=package.get_version())
+                    package_name=package_name,
+                    source_type=source_type,
+                    version=version)
+            if packages_to_upgrade:
+                logger.info('\nUpdates available for packages: {} \
+                \nUpdate your versions in packages.yml, then run dbt deps',
+                            packages_to_upgrade)
 
     @classmethod
     def from_args(cls, args):

core/dbt/task/freshness.py

@@ -19,7 +19,7 @@ from dbt.exceptions import RuntimeException, InternalException
 from dbt.logger import print_timestamped_line
 from dbt.node_types import NodeType
 
-from dbt.graph import NodeSelector, SelectionSpec, parse_difference
+from dbt.graph import ResourceTypeSelector, SelectionSpec, parse_difference
 from dbt.contracts.graph.parsed import ParsedSourceDefinition
 
 
@@ -117,7 +117,7 @@ class FreshnessRunner(BaseRunner):
         return self.node
 
 
-class FreshnessSelector(NodeSelector):
+class FreshnessSelector(ResourceTypeSelector):
     def node_is_match(self, node):
         if not super().node_is_match(node):
             return False
@@ -137,11 +137,16 @@ class FreshnessTask(GraphRunnableTask):
         return False
 
     def get_selection_spec(self) -> SelectionSpec:
-        include = [
-            'source:{}'.format(s)
-            for s in (self.args.selected or ['*'])
-        ]
-        spec = parse_difference(include, None)
+        """Generates a selection spec from task arguments to use when
+        processing graph. A SelectionSpec describes what nodes to select
+        when creating queue from graph of nodes.
+        """
+        if self.args.selector_name:
+            # use pre-defined selector (--selector) to create selection spec
+            spec = self.config.get_selector(self.args.selector_name)
+        else:
+            # use --select and --exclude args to create selection spec
+            spec = parse_difference(self.args.select, self.args.exclude)
         return spec
 
     def get_node_selector(self):
@@ -153,9 +158,10 @@ class FreshnessTask(GraphRunnableTask):
             graph=self.graph,
             manifest=self.manifest,
             previous_state=self.previous_state,
+            resource_types=[NodeType.Source]
         )
 
-    def get_runner_type(self):
+    def get_runner_type(self, _):
         return FreshnessRunner
 
     def write_result(self, result):

core/dbt/task/list.py

@@ -44,7 +44,6 @@ class ListTask(GraphRunnableTask):
 
     def __init__(self, args, config):
         super().__init__(args, config)
-        self.args.single_threaded = True
         if self.args.models:
             if self.args.select:
                 raise RuntimeException(
@@ -113,7 +112,11 @@ class ListTask(GraphRunnableTask):
             yield json.dumps({
                 k: v
                 for k, v in node.to_dict(omit_none=False).items()
-                if k in self.ALLOWED_KEYS
+                if (
+                    k in self.args.output_keys
+                    if self.args.output_keys is not None
+                    else k in self.ALLOWED_KEYS
+                )
             })
 
     def generate_paths(self):

core/dbt/task/printer.py

@@ -87,9 +87,12 @@ def print_hook_end_line(
 
 
 def print_skip_line(
-    model, schema: str, relation: str, index: int, num_models: int
+    node, schema: str, relation: str, index: int, num_models: int
 ) -> None:
-    msg = 'SKIP relation {}.{}'.format(schema, relation)
+    if node.resource_type in NodeType.refable():
+        msg = f'SKIP relation {schema}.{relation}'
+    else:
+        msg = f'SKIP {node.resource_type} {node.name}'
     print_fancy_output_line(
         msg, ui.yellow('SKIP'), logger.info, index, num_models)
 

core/dbt/task/rpc/project_commands.py

@@ -21,6 +21,7 @@ from dbt.contracts.rpc import (
     RPCSnapshotParameters,
     RPCSourceFreshnessParameters,
     RPCListParameters,
+    RPCBuildParameters,
 )
 from dbt.exceptions import RuntimeException
 from dbt.rpc.method import (
@@ -37,6 +38,7 @@ from dbt.task.seed import SeedTask
 from dbt.task.snapshot import SnapshotTask
 from dbt.task.test import TestTask
 from dbt.task.list import ListTask
+from dbt.task.build import BuildTask
 
 from .base import RPCTask
 from .cli import HasCLI
@@ -77,7 +79,10 @@ class RemoteCompileProjectTask(
     METHOD_NAME = 'compile'
 
     def set_args(self, params: RPCCompileParameters) -> None:
-        self.args.models = self._listify(params.models)
+        if params.models:
+            self.args.select = self._listify(params.models)
+        else:
+            self.args.select = self._listify(params.select)
         self.args.exclude = self._listify(params.exclude)
         self.args.selector_name = params.selector
         if params.threads is not None:
@@ -92,7 +97,10 @@ class RemoteRunProjectTask(RPCCommandTask[RPCRunParameters], RunTask):
     METHOD_NAME = 'run'
 
     def set_args(self, params: RPCRunParameters) -> None:
-        self.args.models = self._listify(params.models)
+        if params.models:
+            self.args.select = self._listify(params.models)
+        else:
+            self.args.select = self._listify(params.select)
         self.args.exclude = self._listify(params.exclude)
         self.args.selector_name = params.selector
 
@@ -112,7 +120,7 @@ class RemoteSeedProjectTask(RPCCommandTask[RPCSeedParameters], SeedTask):
 
     def set_args(self, params: RPCSeedParameters) -> None:
         # select has an argparse `dest` value of `models`.
-        self.args.models = self._listify(params.select)
+        self.args.select = self._listify(params.select)
         self.args.exclude = self._listify(params.exclude)
         self.args.selector_name = params.selector
         if params.threads is not None:
@@ -127,7 +135,10 @@ class RemoteTestProjectTask(RPCCommandTask[RPCTestParameters], TestTask):
     METHOD_NAME = 'test'
 
     def set_args(self, params: RPCTestParameters) -> None:
-        self.args.models = self._listify(params.models)
+        if params.models:
+            self.args.select = self._listify(params.models)
+        else:
+            self.args.select = self._listify(params.select)
         self.args.exclude = self._listify(params.exclude)
         self.args.selector_name = params.selector
         self.args.data = params.data
@@ -150,7 +161,7 @@ class RemoteDocsGenerateProjectTask(
     METHOD_NAME = 'docs.generate'
 
     def set_args(self, params: RPCDocsGenerateParameters) -> None:
-        self.args.models = None
+        self.args.select = None
         self.args.exclude = None
         self.args.selector_name = None
         self.args.compile = params.compile
@@ -214,7 +225,7 @@ class RemoteSnapshotTask(RPCCommandTask[RPCSnapshotParameters], SnapshotTask):
 
     def set_args(self, params: RPCSnapshotParameters) -> None:
         # select has an argparse `dest` value of `models`.
-        self.args.models = self._listify(params.select)
+        self.args.select = self._listify(params.select)
         self.args.exclude = self._listify(params.exclude)
         self.args.selector_name = params.selector
         if params.threads is not None:
@@ -228,15 +239,24 @@ class RemoteSourceFreshnessTask(
     RPCCommandTask[RPCSourceFreshnessParameters],
     FreshnessTask
 ):
-    METHOD_NAME = 'snapshot-freshness'
+    METHOD_NAME = 'source-freshness'
 
     def set_args(self, params: RPCSourceFreshnessParameters) -> None:
-        self.args.selected = self._listify(params.select)
+        self.args.select = self._listify(params.select)
+        self.args.exclude = self._listify(params.exclude)
+        self.args.selector_name = params.selector
         if params.threads is not None:
             self.args.threads = params.threads
         self.args.output = None
 
 
+class RemoteSourceSnapshotFreshnessTask(
+    RemoteSourceFreshnessTask
+):
+    """ Deprecated task method name, aliases to `source-freshness` """
+    METHOD_NAME = 'snapshot-freshness'
+
+
 # this is a weird and special method.
 class GetManifest(
     RemoteManifestMethod[GetManifestParameters, GetManifestResult]
@@ -244,7 +264,7 @@ class GetManifest(
     METHOD_NAME = 'get-manifest'
 
     def set_args(self, params: GetManifestParameters) -> None:
-        self.args.models = None
+        self.args.select = None
         self.args.exclude = None
         self.args.selector_name = None
 
@@ -271,13 +291,14 @@ class RemoteListTask(
     METHOD_NAME = 'list'
 
     def set_args(self, params: RPCListParameters) -> None:
-
         self.args.output = params.output
+        self.args.output_keys = params.output_keys
         self.args.resource_types = self._listify(params.resource_types)
         self.args.models = self._listify(params.models)
         self.args.exclude = self._listify(params.exclude)
         self.args.selector_name = params.selector
         self.args.select = self._listify(params.select)
+        self.args.single_threaded = True
 
         if self.args.models:
             if self.args.select:
@@ -296,3 +317,25 @@ class RemoteListTask(
             output=[json.loads(x) for x in results],
             logs=None
         )
+
+
+class RemoteBuildProjectTask(RPCCommandTask[RPCBuildParameters], BuildTask):
+    METHOD_NAME = 'build'
+
+    def set_args(self, params: RPCBuildParameters) -> None:
+        if params.models:
+            self.args.select = self._listify(params.models)
+        else:
+            self.args.select = self._listify(params.select)
+        self.args.exclude = self._listify(params.exclude)
+        self.args.selector_name = params.selector
+
+        if params.threads is not None:
+            self.args.threads = params.threads
+        if params.defer is None:
+            self.args.defer = flags.DEFER_MODE
+        else:
+            self.args.defer = params.defer
+
+        self.args.state = state_path(params.state)
+        self.set_previous_state()

core/dbt/task/rpc/sql_commands.py

@@ -202,12 +202,12 @@ class RemoteRunSQLTask(RPCTask[RPCExecParameters]):
 class RemoteCompileTask(RemoteRunSQLTask, CompileTask):
     METHOD_NAME = 'compile_sql'
 
-    def get_runner_type(self):
+    def get_runner_type(self, _):
         return RPCCompileRunner
 
 
 class RemoteRunTask(RemoteRunSQLTask, RunTask):
     METHOD_NAME = 'run_sql'
 
-    def get_runner_type(self):
+    def get_runner_type(self, _):
         return RPCExecuteRunner

core/dbt/task/run.py

@@ -452,7 +452,7 @@ class RunTask(CompileTask):
             resource_types=[NodeType.Model],
         )
 
-    def get_runner_type(self):
+    def get_runner_type(self, _):
         return ModelRunner
 
     def task_end_messages(self, results):

core/dbt/task/runnable.py

@@ -12,6 +12,7 @@ from .printer import (
     print_run_end_messages,
     print_cancel_line,
 )
+
 from dbt import ui
 from dbt.task.base import ConfiguredTask
 from dbt.adapters.base import BaseRelation
@@ -37,8 +38,9 @@ from dbt.exceptions import (
     InternalException,
     NotImplementedException,
     RuntimeException,
-    FailFastException
+    FailFastException,
 )
+
 from dbt.graph import GraphQueue, NodeSelector, SelectionSpec, Graph
 from dbt.parser.manifest import ManifestLoader
 
@@ -127,7 +129,7 @@ class GraphRunnableTask(ManifestTask):
 
         self.job_queue = self.get_graph_queue()
 
-        # we use this a couple times. order does not matter.
+        # we use this a couple of times. order does not matter.
         self._flattened_nodes = []
         for uid in self.job_queue.get_selected_nodes():
             if uid in self.manifest.nodes:
@@ -148,7 +150,7 @@ class GraphRunnableTask(ManifestTask):
     def raise_on_first_error(self):
         return False
 
-    def get_runner_type(self):
+    def get_runner_type(self, node):
         raise NotImplementedException('Not Implemented')
 
     def result_path(self):
@@ -165,7 +167,7 @@ class GraphRunnableTask(ManifestTask):
             run_count = self.run_count
             num_nodes = self.num_nodes
 
-        cls = self.get_runner_type()
+        cls = self.get_runner_type(node)
         return cls(self.config, adapter, node, run_count, num_nodes)
 
     def call_runner(self, runner):

core/dbt/task/seed.py

@@ -57,7 +57,7 @@ class SeedTask(RunTask):
             resource_types=[NodeType.Seed],
         )
 
-    def get_runner_type(self):
+    def get_runner_type(self, _):
         return SeedRunner
 
     def task_end_messages(self, results):

core/dbt/task/snapshot.py

@@ -38,5 +38,5 @@ class SnapshotTask(RunTask):
             resource_types=[NodeType.Snapshot],
         )
 
-    def get_runner_type(self):
+    def get_runner_type(self, _):
         return SnapshotRunner

core/dbt/task/test.py

@@ -192,5 +192,5 @@ class TestTask(RunTask):
             previous_state=self.previous_state,
         )
 
-    def get_runner_type(self):
+    def get_runner_type(self, _):
         return TestRunner

core/dbt/tracking.py

@@ -21,7 +21,7 @@ sp_logger.setLevel(100)
 COLLECTOR_URL = "fishtownanalytics.sinter-collect.com"
 COLLECTOR_PROTOCOL = "https"
 
-INVOCATION_SPEC = 'iglu:com.dbt/invocation/jsonschema/1-0-1'
+INVOCATION_SPEC = 'iglu:com.dbt/invocation/jsonschema/1-0-2'
 PLATFORM_SPEC = 'iglu:com.dbt/platform/jsonschema/1-0-0'
 RUN_MODEL_SPEC = 'iglu:com.dbt/run_model/jsonschema/1-0-1'
 INVOCATION_ENV_SPEC = 'iglu:com.dbt/invocation_env/jsonschema/1-0-0'
@@ -31,6 +31,7 @@ DEPRECATION_WARN_SPEC = 'iglu:com.dbt/deprecation_warn/jsonschema/1-0-0'
 LOAD_ALL_TIMING_SPEC = 'iglu:com.dbt/load_all_timing/jsonschema/1-0-3'
 RESOURCE_COUNTS = 'iglu:com.dbt/resource_counts/jsonschema/1-0-0'
 EXPERIMENTAL_PARSER = 'iglu:com.dbt/experimental_parser/jsonschema/1-0-0'
+PARTIAL_PARSER = 'iglu:com.dbt/partial_parser/jsonschema/1-0-1'
 DBT_INVOCATION_ENV = 'DBT_INVOCATION_ENV'
 
 
@@ -131,7 +132,7 @@ class User:
         # will change in every dbt invocation until the user points to a
         # profile dir file which contains a valid profiles.yml file.
         #
-        # See: https://github.com/fishtown-analytics/dbt/issues/1645
+        # See: https://github.com/dbt-labs/dbt/issues/1645
 
         user = {"id": str(uuid.uuid4())}
 
@@ -165,10 +166,15 @@ def get_run_type(args):
 
 
 def get_invocation_context(user, config, args):
+    # this adapter might not have implemented the type or unique_field properties
     try:
         adapter_type = config.credentials.type
     except Exception:
         adapter_type = None
+    try:
+        adapter_unique_id = config.credentials.hashed_unique_field()
+    except Exception:
+        adapter_unique_id = None
 
     return {
         "project_id": None if config is None else config.hashed_name(),
@@ -181,6 +187,7 @@ def get_invocation_context(user, config, args):
 
         "run_type": get_run_type(args),
         "adapter_type": adapter_type,
+        "adapter_unique_id": adapter_unique_id,
     }
 
 
@@ -426,7 +433,7 @@ def track_invalid_invocation(
 def track_experimental_parser_sample(options):
     context = [SelfDescribingJson(EXPERIMENTAL_PARSER, options)]
     assert active_user is not None, \
-        'Cannot track project loading time when active user is None'
+        'Cannot track experimental parser info when active user is None'
 
     track(
         active_user,
@@ -437,9 +444,28 @@ def track_experimental_parser_sample(options):
     )
 
 
+def track_partial_parser(options):
+    context = [SelfDescribingJson(PARTIAL_PARSER, options)]
+    assert active_user is not None, \
+        'Cannot track partial parser info when active user is None'
+
+    track(
+        active_user,
+        category='dbt',
+        action='partial_parser',
+        label=active_user.invocation_id,
+        context=context
+    )
+
+
 def flush():
     logger.debug("Flushing usage events")
-    tracker.flush()
+    try:
+        tracker.flush()
+    except Exception:
+        logger.debug(
+            "An error was encountered while trying to flush usage events"
+        )
 
 
 def disable_tracking():