wait for successful connection before setting up db

* Specify macro_namespace of global_project dispatch macros

* Dispatch get_custom_alias, too

* Add integration tests

* Add changelog entry

.bumpversion.cfg

@@ -1,23 +1,27 @@
 [bumpversion]
-current_version = 0.20.0rc1
+current_version = 0.21.0b1
 parse = (?P<major>\d+)
 	\.(?P<minor>\d+)
 	\.(?P<patch>\d+)
-	((?P<prerelease>[a-z]+)(?P<num>\d+))?
+	((?P<prekind>a|b|rc)
+	(?P<pre>\d+)  # pre-release version num
+	)?
 serialize = 
-	{major}.{minor}.{patch}{prerelease}{num}
+	{major}.{minor}.{patch}{prekind}{pre}
 	{major}.{minor}.{patch}
 commit = False
 tag = False
 
-[bumpversion:part:prerelease]
+[bumpversion:part:prekind]
 first_value = a
+optional_value = final
 values = 
 	a
 	b
 	rc
+	final
 
-[bumpversion:part:num]
+[bumpversion:part:pre]
 first_value = 1
 
 [bumpversion:file:setup.py]
@@ -26,6 +30,8 @@ first_value = 1
 
 [bumpversion:file:core/dbt/version.py]
 
+[bumpversion:file:core/scripts/create_adapter_plugins.py]
+
 [bumpversion:file:plugins/postgres/setup.py]
 
 [bumpversion:file:plugins/redshift/setup.py]

.circleci/config.yml

@@ -1,123 +0,0 @@
-version: 2.1
-jobs:
-  unit:
-    docker: &test_only
-      - image: fishtownanalytics/test-container:12
-        environment:
-          DBT_INVOCATION_ENV: circle
-          DOCKER_TEST_DATABASE_HOST: "database"
-          TOX_PARALLEL_NO_SPINNER: 1
-    steps:
-      - checkout
-      - run: tox -p -e py36,py37,py38
-  lint:
-    docker: *test_only
-    steps:
-      - checkout
-      - run: tox -e mypy,flake8 -- -v
-  build-wheels:
-    docker: *test_only
-    steps:
-      - checkout
-      - run:
-          name: Build wheels
-          command: |
-            python3.8 -m venv "${PYTHON_ENV}"
-            export PYTHON_BIN="${PYTHON_ENV}/bin/python"
-            $PYTHON_BIN -m pip install -U pip setuptools
-            $PYTHON_BIN -m pip install -r requirements.txt
-            $PYTHON_BIN -m pip install -r dev-requirements.txt
-            /bin/bash ./scripts/build-wheels.sh
-            $PYTHON_BIN ./scripts/collect-dbt-contexts.py > ./dist/context_metadata.json
-            $PYTHON_BIN ./scripts/collect-artifact-schema.py > ./dist/artifact_schemas.json
-          environment:
-            PYTHON_ENV: /home/tox/build_venv/
-      - store_artifacts:
-          path: ./dist
-          destination: dist
-  integration-postgres:
-    docker:
-      - image: fishtownanalytics/test-container:12
-        environment:
-          DBT_INVOCATION_ENV: circle
-          DOCKER_TEST_DATABASE_HOST: "database"
-          TOX_PARALLEL_NO_SPINNER: 1
-      - image: postgres
-        name: database
-        environment:
-          POSTGRES_USER: "root"
-          POSTGRES_PASSWORD: "password"
-          POSTGRES_DB: "dbt"
-    steps:
-      - checkout
-      - run:
-          name: Setup postgres
-          command: bash test/setup_db.sh
-          environment:
-            PGHOST: database
-            PGUSER: root
-            PGPASSWORD: password
-            PGDATABASE: postgres
-      - run:
-          name: Postgres integration tests
-          command: tox -p -e py36-postgres,py38-postgres -- -v -n4
-          no_output_timeout: 30m
-      - store_artifacts:
-          path: ./logs
-  integration-snowflake:
-    docker: *test_only
-    steps:
-      - checkout
-      - run:
-          name: Snowflake integration tests
-          command: tox -p -e py36-snowflake,py38-snowflake -- -v -n4
-          no_output_timeout: 30m
-      - store_artifacts:
-          path: ./logs
-  integration-redshift:
-    docker: *test_only
-    steps:
-      - checkout
-      - run:
-          name: Redshift integration tests
-          command: tox -p -e py36-redshift,py38-redshift -- -v -n4
-          no_output_timeout: 30m
-      - store_artifacts:
-          path: ./logs
-  integration-bigquery:
-    docker: *test_only
-    steps:
-      - checkout
-      - run:
-          name: Bigquery integration test
-          command: tox -p -e py36-bigquery,py38-bigquery -- -v -n4
-          no_output_timeout: 30m
-      - store_artifacts:
-          path: ./logs
-
-workflows:
-  version: 2
-  test-everything:
-    jobs:
-      - lint
-      - unit
-      - integration-postgres:
-          requires:
-            - unit
-      - integration-redshift:
-          requires:
-            - unit
-      - integration-bigquery:
-          requires:
-            - unit
-      - integration-snowflake:
-          requires:
-            - unit
-      - build-wheels:
-          requires:
-            - lint
-            - unit
-            - integration-postgres
-            - integration-redshift
-            - integration-bigquery
-            - integration-snowflake

.github/ISSUE_TEMPLATE/beta-minor-version-release.md

@@ -0,0 +1,27 @@
+---
+name: Beta minor version release
+about: Creates a tracking checklist of items for a Beta minor version release
+title: "[Tracking] v#.##.#B# release "
+labels: 'release'
+assignees: ''
+
+---
+
+### Release Core
+- [ ] [Engineering] Follow [dbt-release workflow](https://www.notion.so/dbtlabs/Releasing-b97c5ea9a02949e79e81db3566bbc8ef#03ff37da697d4d8ba63d24fae1bfa817) 
+- [ ] [Engineering] Verify new release branch is created in the repo
+- [ ] [Product] Finalize migration guide (next.docs.getdbt.com)
+
+### Release Cloud
+- [ ] [Engineering] Create a platform issue to update dbt Cloud and verify it is completed. [Example issue](https://github.com/dbt-labs/dbt-cloud/issues/3481)
+- [ ] [Engineering] Determine if schemas have changed. If so, generate new schemas and push to schemas.getdbt.com
+
+### Announce
+- [ ] [Product] Announce in dbt Slack
+
+### Post-release
+- [ ] [Engineering] [Bump plugin versions](https://www.notion.so/dbtlabs/Releasing-b97c5ea9a02949e79e81db3566bbc8ef#f01854e8da3641179fbcbe505bdf515c) (dbt-spark + dbt-presto), add compatibility as needed
+   - [ ]  [Spark](https://github.com/dbt-labs/dbt-spark) 
+   - [ ]  [Presto](https://github.com/dbt-labs/dbt-presto)
+- [ ] [Engineering] Create a platform issue to update dbt-spark versions to dbt Cloud. [Example issue](https://github.com/dbt-labs/dbt-cloud/issues/3481)
+- [ ] [Engineering] Create an epic for the RC release

.github/ISSUE_TEMPLATE/final-minor-version-release.md

@@ -0,0 +1,28 @@
+---
+name: Final minor version release
+about: Creates a tracking checklist of items for a final minor version release
+title: "[Tracking] v#.##.# final release "
+labels: 'release'
+assignees: ''
+
+---
+
+### Release Core
+- [ ] [Engineering] Verify all necessary changes exist on the release branch
+- [ ] [Engineering] Follow [dbt-release workflow](https://www.notion.so/dbtlabs/Releasing-b97c5ea9a02949e79e81db3566bbc8ef#03ff37da697d4d8ba63d24fae1bfa817) 
+- [ ] [Product] Merge `next` into `current` for docs.getdbt.com
+
+### Release Cloud
+- [ ] [Engineering] Create a platform issue to update dbt Cloud and verify it is completed. [Example issue](https://github.com/dbt-labs/dbt-cloud/issues/3481)
+- [ ] [Engineering] Determine if schemas have changed. If so, generate new schemas and push to schemas.getdbt.com
+
+### Announce
+- [ ] [Product] Update discourse
+- [ ] [Product] Announce in dbt Slack
+
+### Post-release
+- [ ] [Engineering] [Bump plugin versions](https://www.notion.so/dbtlabs/Releasing-b97c5ea9a02949e79e81db3566bbc8ef#f01854e8da3641179fbcbe505bdf515c) (dbt-spark + dbt-presto), add compatibility as needed
+   - [ ]  [Spark](https://github.com/dbt-labs/dbt-spark) 
+   - [ ]  [Presto](https://github.com/dbt-labs/dbt-presto)
+- [ ] [Engineering] Create a platform issue to update dbt-spark versions to dbt Cloud. [Example issue](https://github.com/dbt-labs/dbt-cloud/issues/3481)
+- [ ] [Product] Release new version of dbt-utils with new dbt version compatibility. If there are breaking changes requiring a minor version, plan upgrades of other packages that depend on dbt-utils.

.github/ISSUE_TEMPLATE/rc-minor-version-release copy.md

@@ -0,0 +1,29 @@
+---
+name: RC minor version release
+about: Creates a tracking checklist of items for a RC minor version release
+title: "[Tracking] v#.##.#RC# release "
+labels: 'release'
+assignees: ''
+
+---
+
+### Release Core
+- [ ] [Engineering] Verify all necessary changes exist on the release branch
+- [ ] [Engineering] Follow [dbt-release workflow](https://www.notion.so/dbtlabs/Releasing-b97c5ea9a02949e79e81db3566bbc8ef#03ff37da697d4d8ba63d24fae1bfa817) 
+- [ ] [Product] Update migration guide (next.docs.getdbt.com)
+
+### Release Cloud
+- [ ] [Engineering] Create a platform issue to update dbt Cloud and verify it is completed. [Example issue](https://github.com/dbt-labs/dbt-cloud/issues/3481)
+- [ ] [Engineering] Determine if schemas have changed. If so, generate new schemas and push to schemas.getdbt.com
+
+### Announce
+- [ ] [Product] Publish discourse
+- [ ] [Product] Announce in dbt Slack
+
+### Post-release
+- [ ] [Engineering] [Bump plugin versions](https://www.notion.so/dbtlabs/Releasing-b97c5ea9a02949e79e81db3566bbc8ef#f01854e8da3641179fbcbe505bdf515c) (dbt-spark + dbt-presto), add compatibility as needed
+   - [ ]  [Spark](https://github.com/dbt-labs/dbt-spark) 
+   - [ ]  [Presto](https://github.com/dbt-labs/dbt-presto)
+- [ ] [Engineering] Create a platform issue to update dbt-spark versions to dbt Cloud. [Example issue](https://github.com/dbt-labs/dbt-cloud/issues/3481)
+- [ ] [Product] Release new version of dbt-utils with new dbt version compatibility. If there are breaking changes requiring a minor version, plan upgrades of other packages that depend on dbt-utils.
+- [ ] [Engineering] Create an epic for the final release

.github/actions/setup-postgres-linux/action.yml

@@ -0,0 +1,10 @@
+name: "Set up postgres (linux)"
+description: "Set up postgres service on linux vm for dbt integration tests"
+runs:
+  using: "composite"
+  steps:
+    - shell: bash
+      run: |
+        sudo systemctl start postgresql.service
+        pg_isready
+        sudo -u postgres bash ${{ github.action_path }}/setup_db.sh

.github/actions/setup-postgres-linux/setup_db.sh

@@ -0,0 +1 @@
+../../../test/setup_db.sh

.github/actions/setup-postgres-macos/action.yml

@@ -0,0 +1,24 @@
+name: "Set up postgres (macos)"
+description: "Set up postgres service on macos vm for dbt integration tests"
+runs:
+  using: "composite"
+  steps:
+    - shell: bash
+      run: |
+        brew services start postgresql
+        echo "Check PostgreSQL service is running"
+        i=10
+        COMMAND='pg_isready'
+        while [ $i -gt -1 ]; do
+            if [ $i == 0 ]; then
+                echo "PostgreSQL service not ready, all attempts exhausted"
+                exit 1
+            fi
+            echo "Check PostgreSQL service status"
+            eval $COMMAND && break
+            echo "PostgreSQL service not ready, wait 10 more sec, attempts left: $i"
+            sleep 10
+            ((i--))
+        done
+        createuser -s postgres
+        bash ${{ github.action_path }}/setup_db.sh

.github/actions/setup-postgres-macos/setup_db.sh

@@ -0,0 +1 @@
+../../../test/setup_db.sh

.github/actions/setup-postgres-windows/action.yml

@@ -0,0 +1,12 @@
+name: "Set up postgres (windows)"
+description: "Set up postgres service on windows vm for dbt integration tests"
+runs:
+  using: "composite"
+  steps:
+    - shell: pwsh
+      run: |
+        $pgService = Get-Service -Name postgresql*
+        Set-Service -InputObject $pgService -Status running -StartupType automatic
+        Start-Process -FilePath "$env:PGBIN\pg_isready" -Wait -PassThru
+        $env:Path += ";$env:PGBIN"
+        bash ${{ github.action_path }}/setup_db.sh

.github/actions/setup-postgres-windows/setup_db.sh

@@ -0,0 +1 @@
+../../../test/setup_db.sh

.github/pull_request_template.md

@@ -9,14 +9,13 @@ resolves #
     resolves #1234
 -->
 
-
 ### Description
 
 <!--- Describe the Pull Request here -->
 
-
 ### Checklist
- - [ ] I have signed the [CLA](https://docs.getdbt.com/docs/contributor-license-agreements)
- - [ ] I have run this code in development and it appears to resolve the stated issue
- - [ ] This PR includes tests, or tests are not required/relevant for this PR
- - [ ] I have updated the `CHANGELOG.md` and added information about my change to the "dbt next" section.
+
+- [ ] I have signed the [CLA](https://docs.getdbt.com/docs/contributor-license-agreements)
+- [ ] I have run this code in development and it appears to resolve the stated issue
+- [ ] This PR includes tests, or tests are not required/relevant for this PR
+- [ ] I have updated the `CHANGELOG.md` and added information about my change to the "dbt next" section.

.github/scripts/integration-test-matrix.js

@@ -0,0 +1,95 @@
+module.exports = ({ context }) => {
+  const defaultPythonVersion = "3.8";
+  const supportedPythonVersions = ["3.6", "3.7", "3.8", "3.9"];
+  const supportedAdapters = ["snowflake", "postgres", "bigquery", "redshift"];
+
+  // if PR, generate matrix based on files changed and PR labels
+  if (context.eventName.includes("pull_request")) {
+    // `changes` is a list of adapter names that have related
+    // file changes in the PR
+    // ex: ['postgres', 'snowflake']
+    const changes = JSON.parse(process.env.CHANGES);
+    const labels = context.payload.pull_request.labels.map(({ name }) => name);
+    console.log("labels", labels);
+    console.log("changes", changes);
+    const testAllLabel = labels.includes("test all");
+    const include = [];
+
+    for (const adapter of supportedAdapters) {
+      if (
+        changes.includes(adapter) ||
+        testAllLabel ||
+        labels.includes(`test ${adapter}`)
+      ) {
+        for (const pythonVersion of supportedPythonVersions) {
+          if (
+            pythonVersion === defaultPythonVersion ||
+            labels.includes(`test python${pythonVersion}`) ||
+            testAllLabel
+          ) {
+            // always run tests on ubuntu by default
+            include.push({
+              os: "ubuntu-latest",
+              adapter,
+              "python-version": pythonVersion,
+            });
+
+            if (labels.includes("test windows") || testAllLabel) {
+              include.push({
+                os: "windows-latest",
+                adapter,
+                "python-version": pythonVersion,
+              });
+            }
+
+            if (labels.includes("test macos") || testAllLabel) {
+              include.push({
+                os: "macos-latest",
+                adapter,
+                "python-version": pythonVersion,
+              });
+            }
+          }
+        }
+      }
+    }
+
+    console.log("matrix", { include });
+
+    return {
+      include,
+    };
+  }
+  // if not PR, generate matrix of python version, adapter, and operating
+  // system to run integration tests on
+
+  const include = [];
+  // run for all adapters and python versions on ubuntu
+  for (const adapter of supportedAdapters) {
+    for (const pythonVersion of supportedPythonVersions) {
+      include.push({
+        os: 'ubuntu-latest',
+        adapter: adapter,
+        "python-version": pythonVersion,
+      });
+    }
+  }
+
+  // additionally include runs for all adapters, on macos and windows,
+  // but only for the default python version
+  for (const adapter of supportedAdapters) {
+    for (const operatingSystem of ["windows-latest", "macos-latest"]) {
+      include.push({
+        os: operatingSystem,
+        adapter: adapter,
+        "python-version": defaultPythonVersion,
+      });
+    }
+  }
+
+  console.log("matrix", { include });
+
+  return {
+    include,
+  };
+};

.github/workflows/integration.yml

@@ -0,0 +1,266 @@
+# **what?**
+# This workflow runs all integration tests for supported OS
+# and python versions and core adapters. If triggered by PR,
+# the workflow will only run tests for adapters related
+# to code changes. Use the `test all` and `test ${adapter}`
+# label to run all or additional tests. Use `ok to test`
+# label to mark PRs from forked repositories that are safe
+# to run integration tests for. Requires secrets to run
+# against different warehouses.
+
+# **why?**
+# This checks the functionality of dbt from a user's perspective
+# and attempts to catch functional regressions.
+
+# **when?**
+# This workflow will run on every push to a protected branch
+# and when manually triggered. It will also run for all PRs, including
+# PRs from forks. The workflow will be skipped until there is a label
+# to mark the PR as safe to run.
+
+name: Adapter Integration Tests
+
+on:
+  # pushes to release branches
+  push:
+    branches:
+      - "main"
+      - "develop"
+      - "*.latest"
+      - "releases/*"
+  # all PRs, important to note that `pull_request_target` workflows
+  # will run in the context of the target branch of a PR
+  pull_request_target:
+  # manual tigger
+  workflow_dispatch:
+
+# explicitly turn off permissions for `GITHUB_TOKEN`
+permissions: read-all
+
+# will cancel previous workflows triggered by the same event and for the same ref for PRs or same SHA otherwise
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event_name }}-${{ contains(github.event_name, 'pull_request') && github.event.pull_request.head.ref || github.sha }}
+  cancel-in-progress: true
+
+# sets default shell to bash, for all operating systems
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  # generate test metadata about what files changed and the testing matrix to use
+  test-metadata:
+    # run if not a PR from a forked repository or has a label to mark as safe to test
+    if: >-
+      github.event_name != 'pull_request_target' ||
+      github.event.pull_request.head.repo.full_name == github.repository ||
+      contains(github.event.pull_request.labels.*.name, 'ok to test')
+
+    runs-on: ubuntu-latest
+
+    outputs:
+      matrix: ${{ steps.generate-matrix.outputs.result }}
+
+    steps:
+      - name: Check out the repository (non-PR)
+        if: github.event_name != 'pull_request_target'
+        uses: actions/checkout@v2
+        with:
+          persist-credentials: false
+
+      - name: Check out the repository (PR)
+        if: github.event_name == 'pull_request_target'
+        uses: actions/checkout@v2
+        with:
+          persist-credentials: false
+          ref: ${{ github.event.pull_request.head.sha }}
+
+      - name: Check if relevant files changed
+        # https://github.com/marketplace/actions/paths-changes-filter
+        # For each filter, it sets output variable named by the filter to the text:
+        #  'true' - if any of changed files matches any of filter rules
+        #  'false' - if none of changed files matches any of filter rules
+        # also, returns:
+        #  `changes` - JSON array with names of all filters matching any of the changed files
+        uses: dorny/paths-filter@v2
+        id: get-changes
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          filters: |
+            postgres:
+              - 'core/**'
+              - 'plugins/postgres/**'
+              - 'dev-requirements.txt'
+            snowflake:
+              - 'core/**'
+              - 'plugins/snowflake/**'
+            bigquery:
+              - 'core/**'
+              - 'plugins/bigquery/**'
+            redshift:
+              - 'core/**'
+              - 'plugins/redshift/**'
+              - 'plugins/postgres/**'
+
+      - name: Generate integration test matrix
+        id: generate-matrix
+        uses: actions/github-script@v4
+        env:
+          CHANGES: ${{ steps.get-changes.outputs.changes }}
+        with:
+          script: |
+            const script = require('./.github/scripts/integration-test-matrix.js')
+            const matrix = script({ context })
+            console.log(matrix)
+            return matrix
+
+  test:
+    name: ${{ matrix.adapter }} / python ${{ matrix.python-version }} / ${{ matrix.os }}
+
+    # run if not a PR from a forked repository or has a label to mark as safe to test
+    # also checks that the matrix generated is not empty
+    if: >-
+      needs.test-metadata.outputs.matrix &&
+      fromJSON( needs.test-metadata.outputs.matrix ).include[0] &&
+      (
+        github.event_name != 'pull_request_target' ||
+        github.event.pull_request.head.repo.full_name == github.repository ||
+        contains(github.event.pull_request.labels.*.name, 'ok to test')
+      )
+
+    runs-on: ${{ matrix.os }}
+
+    needs: test-metadata
+
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJSON(needs.test-metadata.outputs.matrix) }}
+
+    env:
+      TOXENV: integration-${{ matrix.adapter }}
+      PYTEST_ADDOPTS: "-v --color=yes -n4 --csv integration_results.csv"
+      DBT_INVOCATION_ENV: github-actions
+
+    steps:
+      - name: Check out the repository
+        if: github.event_name != 'pull_request_target'
+        uses: actions/checkout@v2
+        with:
+          persist-credentials: false
+
+      # explicity checkout the branch for the PR,
+      # this is necessary for the `pull_request_target` event
+      - name: Check out the repository (PR)
+        if: github.event_name == 'pull_request_target'
+        uses: actions/checkout@v2
+        with:
+          persist-credentials: false
+          ref: ${{ github.event.pull_request.head.sha }}
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Set up postgres (linux)
+        if: |
+          matrix.adapter == 'postgres' &&
+          runner.os == 'Linux'
+        uses: ./.github/actions/setup-postgres-linux
+
+      - name: Set up postgres (macos)
+        if: |
+          matrix.adapter == 'postgres' &&
+          runner.os == 'macOS'
+        uses: ./.github/actions/setup-postgres-macos
+
+      - name: Set up postgres (windows)
+        if: |
+          matrix.adapter == 'postgres' &&
+          runner.os == 'Windows'
+        uses: ./.github/actions/setup-postgres-windows
+
+      - name: Install python dependencies
+        run: |
+          pip install --upgrade pip
+          pip install tox
+          pip --version
+          tox --version
+
+      - name: Run tox (postgres)
+        if: matrix.adapter == 'postgres'
+        run: tox
+
+      - name: Run tox (redshift)
+        if: matrix.adapter == 'redshift'
+        env:
+          REDSHIFT_TEST_DBNAME: ${{ secrets.REDSHIFT_TEST_DBNAME }}
+          REDSHIFT_TEST_PASS: ${{ secrets.REDSHIFT_TEST_PASS }}
+          REDSHIFT_TEST_USER: ${{ secrets.REDSHIFT_TEST_USER }}
+          REDSHIFT_TEST_PORT: ${{ secrets.REDSHIFT_TEST_PORT }}
+          REDSHIFT_TEST_HOST: ${{ secrets.REDSHIFT_TEST_HOST }}
+        run: tox
+
+      - name: Run tox (snowflake)
+        if: matrix.adapter == 'snowflake'
+        env:
+          SNOWFLAKE_TEST_ACCOUNT: ${{ secrets.SNOWFLAKE_TEST_ACCOUNT }}
+          SNOWFLAKE_TEST_PASSWORD: ${{ secrets.SNOWFLAKE_TEST_PASSWORD }}
+          SNOWFLAKE_TEST_USER: ${{ secrets.SNOWFLAKE_TEST_USER }}
+          SNOWFLAKE_TEST_WAREHOUSE: ${{ secrets.SNOWFLAKE_TEST_WAREHOUSE }}
+          SNOWFLAKE_TEST_OAUTH_REFRESH_TOKEN: ${{ secrets.SNOWFLAKE_TEST_OAUTH_REFRESH_TOKEN }}
+          SNOWFLAKE_TEST_OAUTH_CLIENT_ID: ${{ secrets.SNOWFLAKE_TEST_OAUTH_CLIENT_ID }}
+          SNOWFLAKE_TEST_OAUTH_CLIENT_SECRET: ${{ secrets.SNOWFLAKE_TEST_OAUTH_CLIENT_SECRET }}
+          SNOWFLAKE_TEST_ALT_DATABASE: ${{ secrets.SNOWFLAKE_TEST_ALT_DATABASE }}
+          SNOWFLAKE_TEST_ALT_WAREHOUSE: ${{ secrets.SNOWFLAKE_TEST_ALT_WAREHOUSE }}
+          SNOWFLAKE_TEST_DATABASE: ${{ secrets.SNOWFLAKE_TEST_DATABASE }}
+          SNOWFLAKE_TEST_QUOTED_DATABASE: ${{ secrets.SNOWFLAKE_TEST_QUOTED_DATABASE }}
+          SNOWFLAKE_TEST_ROLE: ${{ secrets.SNOWFLAKE_TEST_ROLE }}
+        run: tox
+
+      - name: Run tox (bigquery)
+        if: matrix.adapter == 'bigquery'
+        env:
+          BIGQUERY_TEST_SERVICE_ACCOUNT_JSON: ${{ secrets.BIGQUERY_TEST_SERVICE_ACCOUNT_JSON }}
+          BIGQUERY_TEST_ALT_DATABASE: ${{ secrets.BIGQUERY_TEST_ALT_DATABASE }}
+        run: tox
+
+      - uses: actions/upload-artifact@v2
+        if: always()
+        with:
+          name: logs
+          path: ./logs
+
+      - name: Get current date
+        if: always()
+        id: date
+        run: echo "::set-output name=date::$(date +'%Y-%m-%dT%H_%M_%S')" #no colons allowed for artifacts
+
+      - uses: actions/upload-artifact@v2
+        if: always()
+        with:
+          name: integration_results_${{ matrix.python-version }}_${{ matrix.os }}_${{ matrix.adapter }}-${{ steps.date.outputs.date }}.csv
+          path: integration_results.csv
+
+  require-label-comment:
+    runs-on: ubuntu-latest
+
+    needs: test
+
+    permissions:
+      pull-requests: write
+
+    steps:
+      - name: Needs permission PR comment
+        if: >-
+          needs.test.result == 'skipped' &&
+          github.event_name == 'pull_request_target' &&
+          github.event.pull_request.head.repo.full_name != github.repository
+        uses: unsplash/comment-on-pr@master
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          msg: |
+            "You do not have permissions to run integration tests, @dbt-labs/core "\
+            "needs to label this PR with `ok to test` in order to run integration tests!"
+          check_for_duplicate_msg: true

.github/workflows/main.yml

@@ -0,0 +1,206 @@
+# **what?**
+# Runs code quality checks, unit tests, and verifies python build on
+# all code commited to the repository. This workflow should not
+# require any secrets since it runs for PRs from forked repos.
+# By default, secrets are not passed to workflows running from
+# a forked repo.
+
+# **why?**
+# Ensure code for dbt meets a certain quality standard.
+
+# **when?**
+# This will run for all PRs, when code is pushed to a release
+# branch, and when manually triggered.
+
+name: Tests and Code Checks
+
+on:
+  push:
+    branches:
+      - "main"
+      - "develop"
+      - "*.latest"
+      - "releases/*"
+  pull_request:
+  workflow_dispatch:
+
+permissions: read-all
+
+# will cancel previous workflows triggered by the same event and for the same ref for PRs or same SHA otherwise
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event_name }}-${{ contains(github.event_name, 'pull_request') && github.event.pull_request.head.ref || github.sha }}
+  cancel-in-progress: true
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  code-quality:
+    name: ${{ matrix.toxenv }}
+
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        toxenv: [flake8, mypy]
+
+    env:
+      TOXENV: ${{ matrix.toxenv }}
+      PYTEST_ADDOPTS: "-v --color=yes"
+
+    steps:
+      - name: Check out the repository
+        uses: actions/checkout@v2
+        with:
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@v2
+
+      - name: Install python dependencies
+        run: |
+          pip install --upgrade pip
+          pip install tox
+          pip --version
+          tox --version
+
+      - name: Run tox
+        run: tox
+
+  unit:
+    name: unit test / python ${{ matrix.python-version }}
+
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: [3.6, 3.7, 3.8] # TODO: support unit testing for python 3.9 (https://github.com/dbt-labs/dbt/issues/3689)
+
+    env:
+      TOXENV: "unit"
+      PYTEST_ADDOPTS: "-v --color=yes --csv unit_results.csv"
+
+    steps:
+      - name: Check out the repository
+        uses: actions/checkout@v2
+        with:
+          persist-credentials: false
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install python dependencies
+        run: |
+          pip install --upgrade pip
+          pip install tox
+          pip --version
+          tox --version
+
+      - name: Run tox
+        run: tox
+
+      - name: Get current date
+        if: always()
+        id: date
+        run: echo "::set-output name=date::$(date +'%Y-%m-%dT%H_%M_%S')" #no colons allowed for artifacts
+
+      - uses: actions/upload-artifact@v2
+        if: always()
+        with:
+          name: unit_results_${{ matrix.python-version }}-${{ steps.date.outputs.date }}.csv
+          path: unit_results.csv
+
+  build:
+    name: build packages
+
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Check out the repository
+        uses: actions/checkout@v2
+        with:
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+
+      - name: Install python dependencies
+        run: |
+          pip install --upgrade pip
+          pip install --upgrade setuptools wheel twine check-wheel-contents
+          pip --version
+
+      - name: Build distributions
+        run: ./scripts/build-dist.sh
+
+      - name: Show distributions
+        run: ls -lh dist/
+
+      - name: Check distribution descriptions
+        run: |
+          twine check dist/*
+
+      - name: Check wheel contents
+        run: |
+          check-wheel-contents dist/*.whl --ignore W007,W008
+
+      - uses: actions/upload-artifact@v2
+        with:
+          name: dist
+          path: dist/
+
+  test-build:
+    name: verify packages / python ${{ matrix.python-version }} / ${{ matrix.os }}
+
+    needs: build
+
+    runs-on: ${{ matrix.os }}
+
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+        python-version: [3.6, 3.7, 3.8, 3.9]
+
+    steps:
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install python dependencies
+        run: |
+          pip install --upgrade pip
+          pip install --upgrade wheel
+          pip --version
+
+      - uses: actions/download-artifact@v2
+        with:
+          name: dist
+          path: dist/
+
+      - name: Show distributions
+        run: ls -lh dist/
+
+      - name: Install wheel distributions
+        run: |
+          find ./dist/*.whl -maxdepth 1 -type f | xargs pip install --force-reinstall --find-links=dist/
+
+      - name: Check wheel distributions
+        run: |
+          dbt --version
+
+      - name: Install source distributions
+        run: |
+          find ./dist/*.gz -maxdepth 1 -type f | xargs pip install --force-reinstall --find-links=dist/
+
+      - name: Check source distributions
+        run: |
+          dbt --version

.github/workflows/performance.yml

@@ -0,0 +1,174 @@
+name: Performance Regression Tests
+# Schedule triggers
+on:
+  # runs twice a day at 10:05am and 10:05pm
+  schedule:
+    - cron: "5 10,22 * * *"
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+jobs:
+  # checks fmt of runner code
+  # purposefully not a dependency of any other job
+  # will block merging, but not prevent developing
+  fmt:
+    name: Cargo fmt
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: stable
+          override: true
+      - run: rustup component add rustfmt
+      - uses: actions-rs/cargo@v1
+        with:
+          command: fmt
+          args: --manifest-path performance/runner/Cargo.toml --all -- --check
+
+  # runs any tests associated with the runner
+  # these tests make sure the runner logic is correct
+  test-runner:
+    name: Test Runner
+    runs-on: ubuntu-latest
+    env:
+      # turns errors into warnings
+      RUSTFLAGS: "-D warnings"
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: stable
+          override: true
+      - uses: actions-rs/cargo@v1
+        with:
+          command: test
+          args: --manifest-path performance/runner/Cargo.toml
+
+  # build an optimized binary to be used as the runner in later steps
+  build-runner:
+    needs: [test-runner]
+    name: Build Runner
+    runs-on: ubuntu-latest
+    env:
+      RUSTFLAGS: "-D warnings"
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: stable
+          override: true
+      - uses: actions-rs/cargo@v1
+        with:
+          command: build
+          args: --release --manifest-path performance/runner/Cargo.toml
+      - uses: actions/upload-artifact@v2
+        with:
+          name: runner
+          path: performance/runner/target/release/runner
+
+  # run the performance measurements on the current or default branch
+  measure-dev:
+    needs: [build-runner]
+    name: Measure Dev Branch
+    runs-on: ubuntu-latest
+    steps:
+      - name: checkout dev
+        uses: actions/checkout@v2
+      - name: Setup Python
+        uses: actions/setup-python@v2.2.2
+        with:
+          python-version: "3.8"
+      - name: install dbt
+        run: pip install -r dev-requirements.txt -r editable-requirements.txt
+      - name: install hyperfine
+        run: wget https://github.com/sharkdp/hyperfine/releases/download/v1.11.0/hyperfine_1.11.0_amd64.deb && sudo dpkg -i hyperfine_1.11.0_amd64.deb
+      - uses: actions/download-artifact@v2
+        with:
+          name: runner
+      - name: change permissions
+        run: chmod +x ./runner
+      - name: run
+        run: ./runner measure -b dev -p ${{ github.workspace }}/performance/projects/
+      - uses: actions/upload-artifact@v2
+        with:
+          name: dev-results
+          path: performance/results/
+
+  # run the performance measurements on the release branch which we use
+  # as a performance baseline. This part takes by far the longest, so
+  # we do everything we can first so the job fails fast.
+  # -----
+  # we need to checkout dbt twice in this job: once for the baseline dbt
+  # version, and once to get the latest regression testing projects,
+  # metrics, and runner code from the develop or current branch so that
+  # the calculations match for both versions of dbt we are comparing.
+  measure-baseline:
+    needs: [build-runner]
+    name: Measure Baseline Branch
+    runs-on: ubuntu-latest
+    steps:
+      - name: checkout latest
+        uses: actions/checkout@v2
+        with:
+          ref: "0.20.latest"
+      - name: Setup Python
+        uses: actions/setup-python@v2.2.2
+        with:
+          python-version: "3.8"
+      - name: move repo up a level
+        run: mkdir ${{ github.workspace }}/../baseline/ && cp -r ${{ github.workspace }} ${{ github.workspace }}/../baseline
+      - name: "[debug] ls new dbt location"
+        run: ls ${{ github.workspace }}/../baseline/dbt/
+      # installation creates egg-links so we have to preserve source
+      - name: install dbt from new location
+        run: cd ${{ github.workspace }}/../baseline/dbt/ && pip install -r dev-requirements.txt -r editable-requirements.txt
+      # checkout the current branch to get all the target projects
+      # this deletes the old checked out code which is why we had to copy before
+      - name: checkout dev
+        uses: actions/checkout@v2
+      - name: install hyperfine
+        run: wget https://github.com/sharkdp/hyperfine/releases/download/v1.11.0/hyperfine_1.11.0_amd64.deb && sudo dpkg -i hyperfine_1.11.0_amd64.deb
+      - uses: actions/download-artifact@v2
+        with:
+          name: runner
+      - name: change permissions
+        run: chmod +x ./runner
+      - name: run runner
+        run: ./runner measure -b baseline -p ${{ github.workspace }}/performance/projects/
+      - uses: actions/upload-artifact@v2
+        with:
+          name: baseline-results
+          path: performance/results/
+
+  # detect regressions on the output generated from measuring
+  # the two branches. Exits with non-zero code if a regression is detected.
+  calculate-regressions:
+    needs: [measure-dev, measure-baseline]
+    name: Compare Results
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/download-artifact@v2
+        with:
+          name: dev-results
+      - uses: actions/download-artifact@v2
+        with:
+          name: baseline-results
+      - name: "[debug] ls result files"
+        run: ls
+      - uses: actions/download-artifact@v2
+        with:
+          name: runner
+      - name: change permissions
+        run: chmod +x ./runner
+      - name: run calculation
+        run: ./runner calculate -r ./
+        # always attempt to upload the results even if there were regressions found
+      - uses: actions/upload-artifact@v2
+        if: ${{ always() }}
+        with:
+          name: final-calculations
+          path: ./final_calculations.json

.github/workflows/tests.yml

@@ -1,178 +0,0 @@
-# This is a workflow to run our unit and integration tests for windows and mac
-
-name: dbt Tests
-
-# Triggers
-on:
-  # Triggers the workflow on push or pull request events and also adds a manual trigger
-  push:
-    branches: 
-      - 'develop' 
-      - '*.latest' 
-      - 'releases/*'
-  pull_request_target:
-    branches: 
-      - 'develop' 
-      - '*.latest' 
-      - 'pr/*'
-      - 'releases/*'
-  # Allows you to run this workflow manually from the Actions tab
-  workflow_dispatch:
-
-jobs:
-  Linting:
-    runs-on: ubuntu-latest #no need to run on every OS
-    steps:
-    - uses: actions/checkout@v2
-    - name: Setup Python
-      uses: actions/setup-python@v2.2.2
-      with:
-        python-version: '3.8'
-        architecture: 'x64'
-
-    - name: 'Install dependencies'
-      run: python -m pip install --upgrade pip && pip install tox
-      
-    - name: 'Linting'
-      run: tox -e mypy,flake8 -- -v
-      
-  UnitTest:
-    strategy:
-      matrix:
-        os: [windows-latest, ubuntu-latest, macos-latest]
-    runs-on: ${{ matrix.os }}
-    steps:
-    - uses: actions/checkout@v2
-    - name: Setup Python
-      uses: actions/setup-python@v2.2.2
-      with:
-        python-version: '3.8'
-        architecture: 'x64'
-
-    - name: 'Install dependencies'
-      run: python -m pip install --upgrade pip && pip install tox
-
-    - name: 'Run unit tests'
-      run: python -m tox -e py -- -v
-      
-  PostgresIntegrationTest:
-    runs-on: 'windows-latest' #TODO: Add Mac support
-    environment: 'Postgres'
-    needs: UnitTest
-    steps:
-    - uses: actions/checkout@v2
-    - name: 'Install postgresql and set up database'
-      shell: pwsh
-      run: |
-          $serviceName = Get-Service -Name postgresql*
-          Set-Service -InputObject $serviceName -StartupType Automatic
-          Start-Service -InputObject $serviceName
-          & $env:PGBIN\createdb.exe -U postgres dbt
-          & $env:PGBIN\psql.exe  -U postgres -c "CREATE ROLE root WITH PASSWORD '$env:ROOT_PASSWORD';"
-          & $env:PGBIN\psql.exe  -U postgres -c "ALTER ROLE root WITH LOGIN;"
-          & $env:PGBIN\psql.exe  -U postgres -c "GRANT CREATE, CONNECT ON DATABASE dbt TO root WITH GRANT OPTION;"
-          & $env:PGBIN\psql.exe  -U postgres -c "CREATE ROLE noaccess WITH PASSWORD '$env:NOACCESS_PASSWORD' NOSUPERUSER;"
-          & $env:PGBIN\psql.exe  -U postgres -c "ALTER ROLE noaccess WITH LOGIN;"
-          & $env:PGBIN\psql.exe  -U postgres -c "GRANT CONNECT ON DATABASE dbt TO noaccess;"
-      env:
-          ROOT_PASSWORD: ${{ secrets.ROOT_PASSWORD }}
-          NOACCESS_PASSWORD: ${{ secrets.NOACCESS_PASSWORD }}
-      
-    - name: Setup Python
-      uses: actions/setup-python@v2.2.2
-      with:
-        python-version: '3.7'
-        architecture: 'x64'
-        
-    - name: 'Install dependencies'
-      run: python -m pip install --upgrade pip && pip install tox
-      
-    - name: 'Run integration tests'
-      run: python -m tox -e py-postgres -- -v -n4
-      
-  # These three are all similar except secure environment variables, which MUST be passed along to their tasks,
-  # but there's probably a better way to do this!
-  SnowflakeIntegrationTest:
-    strategy:
-      matrix:
-        os: [windows-latest, macos-latest]
-    runs-on: ${{ matrix.os }}
-    environment: 'Snowflake'
-    needs: UnitTest
-    steps:
-      - uses: actions/checkout@v2
-      - name: Setup Python
-        uses: actions/setup-python@v2.2.2
-        with:
-          python-version: '3.7'
-          architecture: 'x64'
-
-      - name: 'Install dependencies'
-        run: python -m pip install --upgrade pip && pip install tox
-
-      - name: 'Run integration tests'
-        run: python -m tox -e py-snowflake -- -v -n4
-        env:
-          SNOWFLAKE_TEST_ACCOUNT: ${{ secrets.SNOWFLAKE_TEST_ACCOUNT }}
-          SNOWFLAKE_TEST_PASSWORD: ${{ secrets.SNOWFLAKE_TEST_PASSWORD }}
-          SNOWFLAKE_TEST_USER: ${{ secrets.SNOWFLAKE_TEST_USER }}
-          SNOWFLAKE_TEST_WAREHOUSE: ${{ secrets.SNOWFLAKE_TEST_WAREHOUSE }}
-          SNOWFLAKE_TEST_OAUTH_REFRESH_TOKEN: ${{ secrets.SNOWFLAKE_TEST_OAUTH_REFRESH_TOKEN }}
-          SNOWFLAKE_TEST_OAUTH_CLIENT_ID: ${{ secrets.SNOWFLAKE_TEST_OAUTH_CLIENT_ID }}
-          SNOWFLAKE_TEST_OAUTH_CLIENT_SECRET: ${{ secrets.SNOWFLAKE_TEST_OAUTH_CLIENT_SECRET }}
-          SNOWFLAKE_TEST_ALT_DATABASE: ${{ secrets.SNOWFLAKE_TEST_ALT_DATABASE }}
-          SNOWFLAKE_TEST_ALT_WAREHOUSE: ${{ secrets.SNOWFLAKE_TEST_ALT_WAREHOUSE }}
-          SNOWFLAKE_TEST_DATABASE: ${{ secrets.SNOWFLAKE_TEST_DATABASE }}
-          SNOWFLAKE_TEST_QUOTED_DATABASE: ${{ secrets.SNOWFLAKE_TEST_QUOTED_DATABASE }}
-          SNOWFLAKE_TEST_ROLE: ${{ secrets.SNOWFLAKE_TEST_ROLE }}
-
-  BigQueryIntegrationTest:
-    strategy:
-      matrix:
-        os: [windows-latest, macos-latest]
-    runs-on: ${{ matrix.os }}
-    environment: 'Bigquery'
-    needs: UnitTest
-    steps:
-      - uses: actions/checkout@v2
-      - name: Setup Python
-        uses: actions/setup-python@v2.2.2
-        with:
-          python-version: '3.7'
-          architecture: 'x64'
-        
-      - name: 'Install dependencies'
-        run: python -m pip install --upgrade pip && pip install tox
-      
-      - name: 'Run integration tests'
-        run: python -m tox -e py-bigquery -- -v -n4
-        env:
-          BIGQUERY_SERVICE_ACCOUNT_JSON: ${{ secrets.BIGQUERY_SERVICE_ACCOUNT_JSON }}
-          BIGQUERY_TEST_ALT_DATABASE: ${{ secrets.BIGQUERY_TEST_ALT_DATABASE }}
-
-  RedshiftIntegrationTest:
-    strategy:
-      matrix:
-        os: [windows-latest, macos-latest]
-    runs-on: ${{ matrix.os }}
-    environment: 'Redshift'
-    needs: UnitTest
-    steps:
-      - uses: actions/checkout@v2
-      - name: Setup Python
-        uses: actions/setup-python@v2.2.2
-        with:
-          python-version: '3.7'
-          architecture: 'x64'
-
-      - name: 'Install dependencies'
-        run: python -m pip install --upgrade pip && pip install tox
-
-      - name: 'Run integration tests' 
-        run: python -m tox -e py-redshift -- -v -n4
-        env:
-          REDSHIFT_TEST_DBNAME: ${{ secrets.REDSHIFT_TEST_DBNAME }}
-          REDSHIFT_TEST_PASS: ${{ secrets.REDSHIFT_TEST_PASS }}
-          REDSHIFT_TEST_USER: ${{ secrets.REDSHIFT_TEST_USER }}
-          REDSHIFT_TEST_PORT: ${{ secrets.REDSHIFT_TEST_PORT }}
-          REDSHIFT_TEST_HOST: ${{ secrets.REDSHIFT_TEST_HOST }}

ARCHITECTURE.md

@@ -26,7 +26,7 @@ This is the docs website code. It comes from the dbt-docs repository, and is gen
 
 ## Adapters
 
-dbt uses an adapter-plugin pattern to extend support to different databases, warehouses, query engines, etc. The four core adapters that are in the main repository, contained within the [`plugins`](plugins) subdirectory, are: Postgres Redshift, Snowflake and BigQuery. Other warehouses use adapter plugins defined in separate repositories (e.g. [dbt-spark](https://github.com/fishtown-analytics/dbt-spark), [dbt-presto](https://github.com/fishtown-analytics/dbt-presto)).
+dbt uses an adapter-plugin pattern to extend support to different databases, warehouses, query engines, etc. The four core adapters that are in the main repository, contained within the [`plugins`](plugins) subdirectory, are: Postgres Redshift, Snowflake and BigQuery. Other warehouses use adapter plugins defined in separate repositories (e.g. [dbt-spark](https://github.com/dbt-labs/dbt-spark), [dbt-presto](https://github.com/dbt-labs/dbt-presto)).
 
 Each adapter is a mix of python, Jinja2, and SQL. The adapter code also makes heavy use of Jinja2 to wrap modular chunks of SQL functionality, define default implementations, and allow plugins to override it. 
 

CONTRIBUTING.md

@@ -24,7 +24,7 @@ Please note that all contributors to `dbt` must sign the [Contributor License Ag
 
 ### Defining the problem
 
-If you have an idea for a new feature or if you've discovered a bug in `dbt`, the first step is to open an issue. Please check the list of [open issues](https://github.com/fishtown-analytics/dbt/issues) before creating a new one. If you find a relevant issue, please add a comment to the open issue instead of creating a new one. There are hundreds of open issues in this repository and it can be hard to know where to look for a relevant open issue. **The `dbt` maintainers are always happy to point contributors in the right direction**, so please err on the side of documenting your idea in a new issue if you are unsure where a problem statement belongs.
+If you have an idea for a new feature or if you've discovered a bug in `dbt`, the first step is to open an issue. Please check the list of [open issues](https://github.com/dbt-labs/dbt/issues) before creating a new one. If you find a relevant issue, please add a comment to the open issue instead of creating a new one. There are hundreds of open issues in this repository and it can be hard to know where to look for a relevant open issue. **The `dbt` maintainers are always happy to point contributors in the right direction**, so please err on the side of documenting your idea in a new issue if you are unsure where a problem statement belongs.
 
 > **Note:** All community-contributed Pull Requests _must_ be associated with an open issue. If you submit a Pull Request that does not pertain to an open issue, you will be asked to create an issue describing the problem before the Pull Request can be reviewed.
 
@@ -36,7 +36,7 @@ After you open an issue, a `dbt` maintainer will follow up by commenting on your
 
 If an issue is appropriately well scoped and describes a beneficial change to the `dbt` codebase, then anyone may submit a Pull Request to implement the functionality described in the issue. See the sections below on how to do this.
 
-The `dbt` maintainers will add a `good first issue` label if an issue is suitable for a first-time contributor. This label often means that the required code change is small, limited to one database adapter, or a net-new addition that does not impact existing functionality. You can see the list of currently open issues on the [Contribute](https://github.com/fishtown-analytics/dbt/contribute) page.
+The `dbt` maintainers will add a `good first issue` label if an issue is suitable for a first-time contributor. This label often means that the required code change is small, limited to one database adapter, or a net-new addition that does not impact existing functionality. You can see the list of currently open issues on the [Contribute](https://github.com/dbt-labs/dbt/contribute) page.
 
 Here's a good workflow:
 - Comment on the open issue, expressing your interest in contributing the required code change
@@ -52,15 +52,15 @@ The `dbt` maintainers use labels to categorize open issues. Some labels indicate
 
 | tag | description |
 | --- | ----------- |
-| [triage](https://github.com/fishtown-analytics/dbt/labels/triage) | This is a new issue which has not yet been reviewed by a `dbt` maintainer. This label is removed when a maintainer reviews and responds to the issue. |
-| [bug](https://github.com/fishtown-analytics/dbt/labels/bug) | This issue represents a defect or regression in `dbt` |
-| [enhancement](https://github.com/fishtown-analytics/dbt/labels/enhancement) | This issue represents net-new functionality in `dbt` |
-| [good first issue](https://github.com/fishtown-analytics/dbt/labels/good%20first%20issue) | This issue does not require deep knowledge of the `dbt` codebase to implement. This issue is appropriate for a first-time contributor. |
-| [help wanted](https://github.com/fishtown-analytics/`dbt`/labels/help%20wanted) / [discussion](https://github.com/fishtown-analytics/dbt/labels/discussion) | Conversation around this issue in ongoing, and there isn't yet a clear path forward. Input from community members is most welcome. |
-| [duplicate](https://github.com/fishtown-analytics/dbt/issues/duplicate) | This issue is functionally identical to another open issue. The `dbt` maintainers will close this issue and encourage community members to focus conversation on the other one. |
-| [snoozed](https://github.com/fishtown-analytics/dbt/labels/snoozed) | This issue describes a good idea, but one which will probably not be addressed in a six-month time horizon. The `dbt` maintainers will revist these issues periodically and re-prioritize them accordingly. |
-| [stale](https://github.com/fishtown-analytics/dbt/labels/stale) | This is an old issue which has not recently been updated. Stale issues will periodically be closed by `dbt` maintainers, but they can be re-opened if the discussion is restarted. |
-| [wontfix](https://github.com/fishtown-analytics/dbt/labels/wontfix) | This issue does not require a code change in the `dbt` repository, or the maintainers are unwilling/unable to merge a Pull Request which implements the behavior described in the issue. |
+| [triage](https://github.com/dbt-labs/dbt/labels/triage) | This is a new issue which has not yet been reviewed by a `dbt` maintainer. This label is removed when a maintainer reviews and responds to the issue. |
+| [bug](https://github.com/dbt-labs/dbt/labels/bug) | This issue represents a defect or regression in `dbt` |
+| [enhancement](https://github.com/dbt-labs/dbt/labels/enhancement) | This issue represents net-new functionality in `dbt` |
+| [good first issue](https://github.com/dbt-labs/dbt/labels/good%20first%20issue) | This issue does not require deep knowledge of the `dbt` codebase to implement. This issue is appropriate for a first-time contributor. |
+| [help wanted](https://github.com/dbt-labs/dbt/labels/help%20wanted) / [discussion](https://github.com/dbt-labs/dbt/labels/discussion) | Conversation around this issue in ongoing, and there isn't yet a clear path forward. Input from community members is most welcome. |
+| [duplicate](https://github.com/dbt-labs/dbt/issues/duplicate) | This issue is functionally identical to another open issue. The `dbt` maintainers will close this issue and encourage community members to focus conversation on the other one. |
+| [snoozed](https://github.com/dbt-labs/dbt/labels/snoozed) | This issue describes a good idea, but one which will probably not be addressed in a six-month time horizon. The `dbt` maintainers will revist these issues periodically and re-prioritize them accordingly. |
+| [stale](https://github.com/dbt-labs/dbt/labels/stale) | This is an old issue which has not recently been updated. Stale issues will periodically be closed by `dbt` maintainers, but they can be re-opened if the discussion is restarted. |
+| [wontfix](https://github.com/dbt-labs/dbt/labels/wontfix) | This issue does not require a code change in the `dbt` repository, or the maintainers are unwilling/unable to merge a Pull Request which implements the behavior described in the issue. |
 
 #### Branching Strategy
 
@@ -78,17 +78,17 @@ You will need `git` in order to download and modify the `dbt` source code. On ma
 
 ### External contributors
 
-If you are not a member of the `fishtown-analytics` GitHub organization, you can contribute to `dbt` by forking the `dbt` repository. For a detailed overview on forking, check out the [GitHub docs on forking](https://help.github.com/en/articles/fork-a-repo). In short, you will need to:
+If you are not a member of the `dbt-labs` GitHub organization, you can contribute to `dbt` by forking the `dbt` repository. For a detailed overview on forking, check out the [GitHub docs on forking](https://help.github.com/en/articles/fork-a-repo). In short, you will need to:
 
 1. fork the `dbt` repository
 2. clone your fork locally
 3. check out a new branch for your proposed changes
 4. push changes to your fork
-5. open a pull request against `fishtown-analytics/dbt` from your forked repository
+5. open a pull request against `dbt-labs/dbt` from your forked repository
 
 ### Core contributors
 
-If you are a member of the `fishtown-analytics` GitHub organization, you will have push access to the `dbt` repo. Rather than forking `dbt` to make your changes, just clone the repository, check out a new branch, and push directly to that branch.
+If you are a member of the `dbt-labs` GitHub organization, you will have push access to the `dbt` repo. Rather than forking `dbt` to make your changes, just clone the repository, check out a new branch, and push directly to that branch.
 
 ## Setting up an environment
 
@@ -155,7 +155,7 @@ Configure your [profile](https://docs.getdbt.com/docs/configure-your-profile) as
 
 Getting the `dbt` integration tests set up in your local environment will be very helpful as you start to make changes to your local version of `dbt`. The section that follows outlines some helpful tips for setting up the test environment.
 
-Since `dbt` works with a number of different databases, you will need to supply credentials for one or more of these databases in your test environment. Most organizations don't have access to each of a BigQuery, Redshift, Snowflake, and Postgres database, so it's likely that you will be unable to run every integration test locally. Fortunately, Fishtown Analytics provides a CI environment with access to sandboxed Redshift, Snowflake, BigQuery, and Postgres databases. See the section on [_Submitting a Pull Request_](#submitting-a-pull-request) below for more information on this CI setup.
+Since `dbt` works with a number of different databases, you will need to supply credentials for one or more of these databases in your test environment. Most organizations don't have access to each of a BigQuery, Redshift, Snowflake, and Postgres database, so it's likely that you will be unable to run every integration test locally. Fortunately, dbt Labs provides a CI environment with access to sandboxed Redshift, Snowflake, BigQuery, and Postgres databases. See the section on [_Submitting a Pull Request_](#submitting-a-pull-request) below for more information on this CI setup.
 
 ### Initial setup
 
@@ -224,7 +224,7 @@ python -m pytest test/unit/test_graph.py::GraphTest::test__dependency_list
 > is a list of useful command-line options for `pytest` to use while developing.
 ## Submitting a Pull Request
 
-Fishtown Analytics provides a sandboxed Redshift, Snowflake, and BigQuery database for use in a CI environment. When pull requests are submitted to the `fishtown-analytics/dbt` repo, GitHub will trigger automated tests in CircleCI and Azure Pipelines.
+dbt Labs provides a sandboxed Redshift, Snowflake, and BigQuery database for use in a CI environment. When pull requests are submitted to the `dbt-labs/dbt` repo, GitHub will trigger automated tests in CircleCI and Azure Pipelines.
 
 A `dbt` maintainer will review your PR. They may suggest code revision for style or clarity, or request that you add unit or integration test(s). These are good things! We believe that, with a little bit of help, anyone can contribute high-quality code.
 

Dockerfile.test

@@ -1,4 +1,4 @@
-FROM ubuntu:18.04
+FROM ubuntu:20.04
 
 ENV DEBIAN_FRONTEND noninteractive
 

License.md

@@ -186,7 +186,7 @@
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright {yyyy} {name of copyright owner}
+   Copyright 2021 dbt Labs, Inc.
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

README.md

@@ -1,28 +1,18 @@
 <p align="center">
-  <img src="https://raw.githubusercontent.com/fishtown-analytics/dbt/6c6649f9129d5d108aa3b0526f634cd8f3a9d1ed/etc/dbt-logo-full.svg" alt="dbt logo" width="500"/>
+  <img src="https://raw.githubusercontent.com/dbt-labs/dbt/ec7dee39f793aa4f7dd3dae37282cc87664813e4/etc/dbt-logo-full.svg" alt="dbt logo" width="500"/>
 </p>
 <p align="center">
-  <a href="https://codeclimate.com/github/fishtown-analytics/dbt">
-    <img src="https://codeclimate.com/github/fishtown-analytics/dbt/badges/gpa.svg" alt="Code Climate"/>
+  <a href="https://github.com/dbt-labs/dbt/actions/workflows/main.yml">
+    <img src="https://github.com/dbt-labs/dbt/actions/workflows/main.yml/badge.svg?event=push" alt="Unit Tests Badge"/>
   </a>
-  <a href="https://circleci.com/gh/fishtown-analytics/dbt/tree/master">
-    <img src="https://circleci.com/gh/fishtown-analytics/dbt/tree/master.svg?style=svg"  alt="CircleCI" />
-  </a>
-  <a href="https://ci.appveyor.com/project/DrewBanin/dbt/branch/development">
-    <img src="https://ci.appveyor.com/api/projects/status/v01rwd3q91jnwp9m/branch/development?svg=true" alt="AppVeyor" />
-  </a>
-  <a href="https://community.getdbt.com">
-    <img src="https://community.getdbt.com/badge.svg" alt="Slack" />
+  <a href="https://github.com/dbt-labs/dbt/actions/workflows/integration.yml">
+    <img src="https://github.com/dbt-labs/dbt/actions/workflows/integration.yml/badge.svg?event=push" alt="Integration Tests Badge"/>
   </a>
 </p>
 
-**[dbt](https://www.getdbt.com/)** (data build tool) enables data analysts and engineers to transform their data using the same practices that software engineers use to build applications.
+**[dbt](https://www.getdbt.com/)** enables data analysts and engineers to transform their data using the same practices that software engineers use to build applications.
 
-dbt is the T in ELT. Organize, cleanse, denormalize, filter, rename, and pre-aggregate the raw data in your warehouse so that it's ready for analysis.
-
-![dbt architecture](https://raw.githubusercontent.com/fishtown-analytics/dbt/6c6649f9129d5d108aa3b0526f634cd8f3a9d1ed/etc/dbt-arch.png)
-
-dbt can be used to [aggregate pageviews into sessions](https://github.com/fishtown-analytics/snowplow), calculate [ad spend ROI](https://github.com/fishtown-analytics/facebook-ads), or report on [email campaign performance](https://github.com/fishtown-analytics/mailchimp).
+![architecture](https://raw.githubusercontent.com/dbt-labs/dbt/6c6649f9129d5d108aa3b0526f634cd8f3a9d1ed/etc/dbt-arch.png)
 
 ## Understanding dbt
 
@@ -30,28 +20,22 @@ Analysts using dbt can transform their data by simply writing select statements,
 
 These select statements, or "models", form a dbt project. Models frequently build on top of one another – dbt makes it easy to [manage relationships](https://docs.getdbt.com/docs/ref) between models, and [visualize these relationships](https://docs.getdbt.com/docs/documentation), as well as assure the quality of your transformations through [testing](https://docs.getdbt.com/docs/testing).
 
-![dbt dag](https://raw.githubusercontent.com/fishtown-analytics/dbt/6c6649f9129d5d108aa3b0526f634cd8f3a9d1ed/etc/dbt-dag.png)
+![dbt dag](https://raw.githubusercontent.com/dbt-labs/dbt/6c6649f9129d5d108aa3b0526f634cd8f3a9d1ed/etc/dbt-dag.png)
 
 ## Getting started
 
--   [Install dbt](https://docs.getdbt.com/docs/installation)
--   Read the [documentation](https://docs.getdbt.com/).
--   Productionize your dbt project with [dbt Cloud](https://www.getdbt.com)
+- [Install dbt](https://docs.getdbt.com/docs/installation)
+- Read the [introduction](https://docs.getdbt.com/docs/introduction/) and [viewpoint](https://docs.getdbt.com/docs/about/viewpoint/)
 
-## Find out more
+## Join the dbt Community
 
--   Check out the [Introduction to dbt](https://docs.getdbt.com/docs/introduction/).
--   Read the [dbt Viewpoint](https://docs.getdbt.com/docs/about/viewpoint/).
-
-## Join thousands of analysts in the dbt community
-
--   Join the [chat](http://community.getdbt.com/) on Slack.
--   Find community posts on [dbt Discourse](https://discourse.getdbt.com).
+- Be part of the conversation in the [dbt Community Slack](http://community.getdbt.com/)
+- Read more on the [dbt Community Discourse](https://discourse.getdbt.com)
 
 ## Reporting bugs and contributing code
 
--   Want to report a bug or request a feature? Let us know on [Slack](http://community.getdbt.com/), or open [an issue](https://github.com/fishtown-analytics/dbt/issues/new).
--   Want to help us build dbt? Check out the [Contributing Getting Started Guide](https://github.com/fishtown-analytics/dbt/blob/HEAD/CONTRIBUTING.md)
+- Want to report a bug or request a feature? Let us know on [Slack](http://community.getdbt.com/), or open [an issue](https://github.com/dbt-labs/dbt/issues/new)
+- Want to help us build dbt? Check out the [Contributing Guide](https://github.com/dbt-labs/dbt/blob/HEAD/CONTRIBUTING.md)
 
 ## Code of Conduct
 

azure-pipelines.yml

@@ -1,154 +0,0 @@
-# Python package
-# Create and test a Python package on multiple Python versions.
-# Add steps that analyze code, save the dist with the build record, publish to a PyPI-compatible index, and more:
-# https://docs.microsoft.com/azure/devops/pipelines/languages/python
-
-trigger:
-  branches:
-    include:
-    - develop
-    - '*.latest'
-    - pr/*
-
-jobs:
-- job: UnitTest
-  pool:
-    vmImage: 'vs2017-win2016'
-  steps:
-  - task: UsePythonVersion@0
-    inputs:
-      versionSpec: '3.7'
-      architecture: 'x64'
-
-  - script: python -m pip install --upgrade pip && pip install tox
-    displayName: 'Install dependencies'
-
-  - script: python -m tox -e py -- -v
-    displayName: Run unit tests
-
-- job: PostgresIntegrationTest
-  pool:
-    vmImage: 'vs2017-win2016'
-  dependsOn: UnitTest
-
-  steps:
-  - pwsh: |
-      $serviceName = Get-Service -Name postgresql*
-      Set-Service -InputObject $serviceName -StartupType Automatic
-      Start-Service -InputObject $serviceName
-
-      & $env:PGBIN\createdb.exe -U postgres dbt
-      & $env:PGBIN\psql.exe  -U postgres -c "CREATE ROLE root WITH PASSWORD 'password';"
-      & $env:PGBIN\psql.exe  -U postgres -c "ALTER ROLE root WITH LOGIN;"
-      & $env:PGBIN\psql.exe  -U postgres -c "GRANT CREATE, CONNECT ON DATABASE dbt TO root WITH GRANT OPTION;"
-      & $env:PGBIN\psql.exe  -U postgres -c "CREATE ROLE noaccess WITH PASSWORD 'password' NOSUPERUSER;"
-      & $env:PGBIN\psql.exe  -U postgres -c "ALTER ROLE noaccess WITH LOGIN;"
-      & $env:PGBIN\psql.exe  -U postgres -c "GRANT CONNECT ON DATABASE dbt TO noaccess;"
-    displayName: Install postgresql and set up database
-
-  - task: UsePythonVersion@0
-    inputs:
-      versionSpec: '3.7'
-      architecture: 'x64'
-
-  - script: python -m pip install --upgrade pip && pip install tox
-    displayName: 'Install dependencies'
-
-  - script: python -m tox -e py-postgres -- -v -n4
-    displayName: Run integration tests
-
-# These three are all similar except secure environment variables, which MUST be passed along to their tasks,
-# but there's probably a better way to do this!
-- job: SnowflakeIntegrationTest
-  pool:
-    vmImage: 'vs2017-win2016'
-  dependsOn: UnitTest
-  condition: succeeded()
-  steps:
-  - task: UsePythonVersion@0
-    inputs:
-      versionSpec: '3.7'
-      architecture: 'x64'
-
-  - script: python -m pip install --upgrade pip && pip install tox
-    displayName: 'Install dependencies'
-
-  - script: python -m tox -e py-snowflake -- -v -n4
-    env:
-      SNOWFLAKE_TEST_ACCOUNT: $(SNOWFLAKE_TEST_ACCOUNT)
-      SNOWFLAKE_TEST_PASSWORD: $(SNOWFLAKE_TEST_PASSWORD)
-      SNOWFLAKE_TEST_USER: $(SNOWFLAKE_TEST_USER)
-      SNOWFLAKE_TEST_WAREHOUSE: $(SNOWFLAKE_TEST_WAREHOUSE)
-      SNOWFLAKE_TEST_OAUTH_REFRESH_TOKEN: $(SNOWFLAKE_TEST_OAUTH_REFRESH_TOKEN)
-      SNOWFLAKE_TEST_OAUTH_CLIENT_ID: $(SNOWFLAKE_TEST_OAUTH_CLIENT_ID)
-      SNOWFLAKE_TEST_OAUTH_CLIENT_SECRET: $(SNOWFLAKE_TEST_OAUTH_CLIENT_SECRET)
-    displayName: Run integration tests
-
-- job: BigQueryIntegrationTest
-  pool:
-    vmImage: 'vs2017-win2016'
-  dependsOn: UnitTest
-  condition: succeeded()
-  steps:
-  - task: UsePythonVersion@0
-    inputs:
-      versionSpec: '3.7'
-      architecture: 'x64'
-  - script: python -m pip install --upgrade pip && pip install tox
-    displayName: 'Install dependencies'
-  - script: python -m tox -e py-bigquery -- -v -n4
-    env:
-      BIGQUERY_SERVICE_ACCOUNT_JSON: $(BIGQUERY_SERVICE_ACCOUNT_JSON)
-    displayName: Run integration tests
-
-- job: RedshiftIntegrationTest
-  pool:
-    vmImage: 'vs2017-win2016'
-  dependsOn: UnitTest
-  condition: succeeded()
-  steps:
-  - task: UsePythonVersion@0
-    inputs:
-      versionSpec: '3.7'
-      architecture: 'x64'
-
-  - script: python -m pip install --upgrade pip && pip install tox
-    displayName: 'Install dependencies'
-
-  - script: python -m tox -e py-redshift -- -v -n4
-    env:
-      REDSHIFT_TEST_DBNAME: $(REDSHIFT_TEST_DBNAME)
-      REDSHIFT_TEST_PASS: $(REDSHIFT_TEST_PASS)
-      REDSHIFT_TEST_USER: $(REDSHIFT_TEST_USER)
-      REDSHIFT_TEST_PORT: $(REDSHIFT_TEST_PORT)
-      REDSHIFT_TEST_HOST: $(REDSHIFT_TEST_HOST)
-    displayName: Run integration tests
-
-- job: BuildWheel
-  pool:
-    vmImage: 'vs2017-win2016'
-  dependsOn:
-    - UnitTest
-    - PostgresIntegrationTest
-    - RedshiftIntegrationTest
-    - SnowflakeIntegrationTest
-    - BigQueryIntegrationTest
-  condition: succeeded()
-  steps:
-    - task: UsePythonVersion@0
-      inputs:
-        versionSpec: '3.7'
-        architecture: 'x64'
-    - script: python -m pip install --upgrade pip setuptools && python -m pip install -r requirements.txt && python -m pip install -r dev-requirements.txt
-      displayName: Install dependencies
-    - task: ShellScript@2
-      inputs:
-        scriptPath: scripts/build-wheels.sh
-    - task: CopyFiles@2
-      inputs:
-        contents: 'dist\?(*.whl|*.tar.gz)'
-        TargetFolder: '$(Build.ArtifactStagingDirectory)'
-    - task: PublishBuildArtifacts@1
-      inputs:
-        pathtoPublish: '$(Build.ArtifactStagingDirectory)'
-        artifactName: dists

converter.py

@@ -1,73 +0,0 @@
-#!/usr/bin/env python
-import json
-import yaml
-import sys
-import argparse
-from datetime import datetime, timezone
-import dbt.clients.registry as registry
-
-
-def yaml_type(fname):
-    with open(fname) as f:
-        return yaml.load(f)
-
-
-def parse_args():
-    parser = argparse.ArgumentParser()
-    parser.add_argument("--project", type=yaml_type, default="dbt_project.yml")
-    parser.add_argument("--namespace", required=True)
-    return parser.parse_args()
-
-
-def get_full_name(args):
-    return "{}/{}".format(args.namespace, args.project["name"])
-
-
-def init_project_in_packages(args, packages):
-    full_name = get_full_name(args)
-    if full_name not in packages:
-        packages[full_name] = {
-            "name": args.project["name"],
-            "namespace": args.namespace,
-            "latest": args.project["version"],
-            "assets": {},
-            "versions": {},
-        }
-    return packages[full_name]
-
-
-def add_version_to_package(args, project_json):
-    project_json["versions"][args.project["version"]] = {
-        "id": "{}/{}".format(get_full_name(args), args.project["version"]),
-        "name": args.project["name"],
-        "version": args.project["version"],
-        "description": "",
-        "published_at": datetime.now(timezone.utc).astimezone().isoformat(),
-        "packages": args.project.get("packages") or [],
-        "works_with": [],
-        "_source": {
-            "type": "github",
-            "url": "",
-            "readme": "",
-        },
-        "downloads": {
-            "tarball": "",
-            "format": "tgz",
-            "sha1": "",
-        },
-    }
-
-
-def main():
-    args = parse_args()
-    packages = registry.packages()
-    project_json = init_project_in_packages(args, packages)
-    if args.project["version"] in project_json["versions"]:
-        raise Exception("Version {} already in packages JSON"
-                        .format(args.project["version"]),
-                        file=sys.stderr)
-    add_version_to_package(args, project_json)
-    print(json.dumps(packages, indent=2))
-
-if __name__ == "__main__":
-    main()

core/MANIFEST.in

@@ -1 +1 @@
-recursive-include dbt/include *.py *.sql *.yml *.html *.md
+recursive-include dbt/include *.py *.sql *.yml *.html *.md .gitkeep .gitignore

core/dbt/adapters/base/impl.py

@@ -31,7 +31,6 @@ from dbt.contracts.graph.compiled import (
 from dbt.contracts.graph.manifest import Manifest, MacroManifest
 from dbt.contracts.graph.parsed import ParsedSeedNode
 from dbt.exceptions import warn_or_error
-from dbt.node_types import NodeType
 from dbt.logger import GLOBAL_LOGGER as logger
 from dbt.utils import filter_null_values, executor
 
@@ -310,8 +309,7 @@ class BaseAdapter(metaclass=AdapterMeta):
             self.Relation.create_from(self.config, node).without_identifier()
             for node in manifest.nodes.values()
             if (
-                node.resource_type in NodeType.executable() and
-                not node.is_ephemeral_model
+                node.is_relational and not node.is_ephemeral_model
             )
         }
 
@@ -513,7 +511,7 @@ class BaseAdapter(metaclass=AdapterMeta):
     def get_columns_in_relation(
         self, relation: BaseRelation
     ) -> List[BaseColumn]:
-        """Get a list of the columns in the given Relation."""
+        """Get a list of the columns in the given Relation. """
         raise NotImplementedException(
             '`get_columns_in_relation` is not implemented for this adapter!'
         )

core/dbt/adapters/base/relation.py

@@ -433,13 +433,14 @@ class SchemaSearchMap(Dict[InformationSchema, Set[Optional[str]]]):
             for schema in schemas:
                 yield information_schema_name, schema
 
-    def flatten(self):
+    def flatten(self, allow_multiple_databases: bool = False):
         new = self.__class__()
 
-        # make sure we don't have duplicates
-        seen = {r.database.lower() for r in self if r.database}
-        if len(seen) > 1:
-            dbt.exceptions.raise_compiler_error(str(seen))
+        # make sure we don't have multiple databases if allow_multiple_databases is set to False
+        if not allow_multiple_databases:
+            seen = {r.database.lower() for r in self if r.database}
+            if len(seen) > 1:
+                dbt.exceptions.raise_compiler_error(str(seen))
 
         for information_schema_name, schema in self.search():
             path = {

core/dbt/clients/agate_helper.py

@@ -35,7 +35,11 @@ class ISODateTime(agate.data_types.DateTime):
         )
 
 
-def build_type_tester(text_columns: Iterable[str]) -> agate.TypeTester:
+def build_type_tester(
+    text_columns: Iterable[str],
+    string_null_values: Optional[Iterable[str]] = ('null', '')
+) -> agate.TypeTester:
+
     types = [
         agate.data_types.Number(null_values=('null', '')),
         agate.data_types.Date(null_values=('null', ''),
@@ -46,10 +50,10 @@ def build_type_tester(text_columns: Iterable[str]) -> agate.TypeTester:
         agate.data_types.Boolean(true_values=('true',),
                                  false_values=('false',),
                                  null_values=('null', '')),
-        agate.data_types.Text(null_values=('null', ''))
+        agate.data_types.Text(null_values=string_null_values)
     ]
     force = {
-        k: agate.data_types.Text(null_values=('null', ''))
+        k: agate.data_types.Text(null_values=string_null_values)
         for k in text_columns
     }
     return agate.TypeTester(force=force, types=types)
@@ -66,7 +70,13 @@ def table_from_rows(
     if text_only_columns is None:
         column_types = DEFAULT_TYPE_TESTER
     else:
-        column_types = build_type_tester(text_only_columns)
+        # If text_only_columns are present, prevent coercing empty string or
+        # literal 'null' strings to a None representation.
+        column_types = build_type_tester(
+            text_only_columns,
+            string_null_values=()
+        )
+
     return agate.Table(rows, column_names, column_types=column_types)
 
 
@@ -86,19 +96,34 @@ def table_from_data(data, column_names: Iterable[str]) -> agate.Table:
 
 
 def table_from_data_flat(data, column_names: Iterable[str]) -> agate.Table:
-    "Convert list of dictionaries into an Agate table"
+    """
+    Convert a list of dictionaries into an Agate table. This method does not
+    coerce string values into more specific types (eg. '005' will not be
+    coerced to '5'). Additionally, this method does not coerce values to
+    None (eg. '' or 'null' will retain their string literal representations).
+    """
 
     rows = []
+    text_only_columns = set()
     for _row in data:
         row = []
-        for value in list(_row.values()):
+        for col_name in column_names:
+            value = _row[col_name]
             if isinstance(value, (dict, list, tuple)):
-                row.append(json.dumps(value, cls=dbt.utils.JSONEncoder))
-            else:
-                row.append(value)
+                # Represent container types as json strings
+                value = json.dumps(value, cls=dbt.utils.JSONEncoder)
+                text_only_columns.add(col_name)
+            elif isinstance(value, str):
+                text_only_columns.add(col_name)
+            row.append(value)
+
         rows.append(row)
 
-    return table_from_rows(rows=rows, column_names=column_names)
+    return table_from_rows(
+        rows=rows,
+        column_names=column_names,
+        text_only_columns=text_only_columns
+    )
 
 
 def empty_table():

core/dbt/clients/jinja_static.py

@@ -153,7 +153,7 @@ def statically_parse_adapter_dispatch(func_call, ctx, db_wrapper):
                 package_name = packages_arg.node.node.name
                 macro_name = packages_arg.node.attr
                 if (macro_name.startswith('_get') and 'namespaces' in macro_name):
-                    # noqa: https://github.com/fishtown-analytics/dbt-utils/blob/9e9407b/macros/cross_db_utils/_get_utils_namespaces.sql
+                    # noqa: https://github.com/dbt-labs/dbt-utils/blob/9e9407b/macros/cross_db_utils/_get_utils_namespaces.sql
                     var_name = f'{package_name}_dispatch_list'
                     # hard code compatibility for fivetran_utils, just a teensy bit different
                     # noqa: https://github.com/fivetran/dbt_fivetran_utils/blob/0978ba2/macros/_get_utils_namespaces.sql

core/dbt/clients/registry.py

@@ -1,10 +1,9 @@
-from functools import wraps
+import functools
 import requests
-from dbt.exceptions import RegistryException
-from dbt.utils import memoized
+from dbt.utils import memoized, _connection_exception_retry as connection_exception_retry
 from dbt.logger import GLOBAL_LOGGER as logger
+from dbt import deprecations
 import os
-import time
 
 if os.getenv('DBT_PACKAGE_HUB_URL'):
     DEFAULT_REGISTRY_BASE_URL = os.getenv('DBT_PACKAGE_HUB_URL')
@@ -19,26 +18,11 @@ def _get_url(url, registry_base_url=None):
     return '{}{}'.format(registry_base_url, url)
 
 
-def _wrap_exceptions(fn):
-    @wraps(fn)
-    def wrapper(*args, **kwargs):
-        max_attempts = 5
-        attempt = 0
-        while True:
-            attempt += 1
-            try:
-                return fn(*args, **kwargs)
-            except (requests.exceptions.ConnectionError, requests.exceptions.Timeout) as exc:
-                if attempt < max_attempts:
-                    time.sleep(1)
-                    continue
-                raise RegistryException(
-                    'Unable to connect to registry hub'
-                ) from exc
-    return wrapper
+def _get_with_retries(path, registry_base_url=None):
+    get_fn = functools.partial(_get, path, registry_base_url)
+    return connection_exception_retry(get_fn, 5)
 
 
-@_wrap_exceptions
 def _get(path, registry_base_url=None):
     url = _get_url(path, registry_base_url)
     logger.debug('Making package registry request: GET {}'.format(url))
@@ -50,22 +34,44 @@ def _get(path, registry_base_url=None):
 
 
 def index(registry_base_url=None):
-    return _get('api/v1/index.json', registry_base_url)
+    return _get_with_retries('api/v1/index.json', registry_base_url)
 
 
 index_cached = memoized(index)
 
 
 def packages(registry_base_url=None):
-    return _get('api/v1/packages.json', registry_base_url)
+    return _get_with_retries('api/v1/packages.json', registry_base_url)
 
 
 def package(name, registry_base_url=None):
-    return _get('api/v1/{}.json'.format(name), registry_base_url)
+    response = _get_with_retries('api/v1/{}.json'.format(name), registry_base_url)
+
+    # Either redirectnamespace or redirectname in the JSON response indicate a redirect
+    # redirectnamespace redirects based on package ownership
+    # redirectname redirects based on package name
+    # Both can be present at the same time, or neither. Fails gracefully to old name
+
+    if ('redirectnamespace' in response) or ('redirectname' in response):
+
+        if ('redirectnamespace' in response) and response['redirectnamespace'] is not None:
+            use_namespace = response['redirectnamespace']
+        else:
+            use_namespace = response['namespace']
+
+        if ('redirectname' in response) and response['redirectname'] is not None:
+            use_name = response['redirectname']
+        else:
+            use_name = response['name']
+
+        new_nwo = use_namespace + "/" + use_name
+        deprecations.warn('package-redirect', old_name=name, new_name=new_nwo)
+
+    return response
 
 
 def package_version(name, version, registry_base_url=None):
-    return _get('api/v1/{}/{}.json'.format(name, version), registry_base_url)
+    return _get_with_retries('api/v1/{}/{}.json'.format(name, version), registry_base_url)
 
 
 def get_available_versions(name):

core/dbt/clients/system.py

@@ -1,4 +1,5 @@
 import errno
+import functools
 import fnmatch
 import json
 import os
@@ -15,9 +16,8 @@ from typing import (
 )
 
 import dbt.exceptions
-import dbt.utils
-
 from dbt.logger import GLOBAL_LOGGER as logger
+from dbt.utils import _connection_exception_retry as connection_exception_retry
 
 if sys.platform == 'win32':
     from ctypes import WinDLL, c_bool
@@ -441,6 +441,13 @@ def run_cmd(
     return out, err
 
 
+def download_with_retries(
+    url: str, path: str, timeout: Optional[Union[float, tuple]] = None
+) -> None:
+    download_fn = functools.partial(download, url, path, timeout)
+    connection_exception_retry(download_fn, 5)
+
+
 def download(
     url: str, path: str, timeout: Optional[Union[float, tuple]] = None
 ) -> None:

core/dbt/clients/yaml_helper.py

@@ -1,5 +1,5 @@
 import dbt.exceptions
-
+from typing import Any, Dict, Optional
 import yaml
 import yaml.scanner
 
@@ -56,7 +56,7 @@ def contextualized_yaml_error(raw_contents, error):
                                      raw_error=error)
 
 
-def safe_load(contents):
+def safe_load(contents) -> Optional[Dict[str, Any]]:
     return yaml.load(contents, Loader=SafeLoader)
 
 

core/dbt/config/profile.py

@@ -84,7 +84,8 @@ def read_user_config(directory: str) -> UserConfig:
 
 # The Profile class is included in RuntimeConfig, so any attribute
 # additions must also be set where the RuntimeConfig class is created
-@dataclass
+# `init=False` is a workaround for https://bugs.python.org/issue45081
+@dataclass(init=False)
 class Profile(HasCredentials):
     profile_name: str
     target_name: str
@@ -92,6 +93,23 @@ class Profile(HasCredentials):
     threads: int
     credentials: Credentials
 
+    def __init__(
+        self,
+        profile_name: str,
+        target_name: str,
+        config: UserConfig,
+        threads: int,
+        credentials: Credentials
+    ):
+        """Explicitly defining `__init__` to work around bug in Python 3.9.7
+        https://bugs.python.org/issue45081
+        """
+        self.profile_name = profile_name
+        self.target_name = target_name
+        self.config = config
+        self.threads = threads
+        self.credentials = credentials
+
     def to_profile_info(
         self, serialize_credentials: bool = False
     ) -> Dict[str, Any]:

core/dbt/config/renderer.py

@@ -147,7 +147,7 @@ class DbtProjectYamlRenderer(BaseRenderer):
 
         if first in {'seeds', 'models', 'snapshots', 'tests'}:
             keypath_parts = {
-                (k.lstrip('+') if isinstance(k, str) else k)
+                (k.lstrip('+ ') if isinstance(k, str) else k)
                 for k in keypath
             }
             # model-level hooks

core/dbt/config/runtime.py

@@ -391,6 +391,10 @@ class UnsetCredentials(Credentials):
     def type(self):
         return None
 
+    @property
+    def unique_field(self):
+        return None
+
     def connection_info(self, *args, **kwargs):
         return {}
 

core/dbt/context/context_config.py

@@ -97,7 +97,7 @@ class BaseContextConfigGenerator(Generic[T]):
             result = {}
             for key, value in level_config.items():
                 if key.startswith('+'):
-                    result[key[1:]] = deepcopy(value)
+                    result[key[1:].strip()] = deepcopy(value)
                 elif not isinstance(value, dict):
                     result[key] = deepcopy(value)
 
@@ -120,11 +120,12 @@ class BaseContextConfigGenerator(Generic[T]):
 
     def calculate_node_config(
         self,
-        config_calls: List[Dict[str, Any]],
+        config_call_dict: Dict[str, Any],
         fqn: List[str],
         resource_type: NodeType,
         project_name: str,
         base: bool,
+        patch_config_dict: Dict[str, Any] = None
     ) -> BaseConfig:
         own_config = self.get_node_project(project_name)
 
@@ -134,8 +135,15 @@ class BaseContextConfigGenerator(Generic[T]):
         for fqn_config in project_configs:
             result = self._update_from_config(result, fqn_config)
 
-        for config_call in config_calls:
-            result = self._update_from_config(result, config_call)
+        # When schema files patch config, it has lower precedence than
+        # config in the models (config_call_dict), so we add the patch_config_dict
+        # before the config_call_dict
+        if patch_config_dict:
+            result = self._update_from_config(result, patch_config_dict)
+
+        # config_calls are created in the 'experimental' model parser and
+        # the ParseConfigObject (via add_config_call)
+        result = self._update_from_config(result, config_call_dict)
 
         if own_config.project_name != self._active_project.project_name:
             for fqn_config in self._active_project_configs(fqn, resource_type):
@@ -147,11 +155,12 @@ class BaseContextConfigGenerator(Generic[T]):
     @abstractmethod
     def calculate_node_config_dict(
         self,
-        config_calls: List[Dict[str, Any]],
+        config_call_dict: Dict[str, Any],
         fqn: List[str],
         resource_type: NodeType,
         project_name: str,
         base: bool,
+        patch_config_dict: Dict[str, Any],
     ) -> Dict[str, Any]:
         ...
 
@@ -186,18 +195,20 @@ class ContextConfigGenerator(BaseContextConfigGenerator[C]):
 
     def calculate_node_config_dict(
         self,
-        config_calls: List[Dict[str, Any]],
+        config_call_dict: Dict[str, Any],
         fqn: List[str],
         resource_type: NodeType,
         project_name: str,
         base: bool,
+        patch_config_dict: dict = None
     ) -> Dict[str, Any]:
         config = self.calculate_node_config(
-            config_calls=config_calls,
+            config_call_dict=config_call_dict,
             fqn=fqn,
             resource_type=resource_type,
             project_name=project_name,
             base=base,
+            patch_config_dict=patch_config_dict
         )
         finalized = config.finalize_and_validate()
         return finalized.to_dict(omit_none=True)
@@ -209,18 +220,20 @@ class UnrenderedConfigGenerator(BaseContextConfigGenerator[Dict[str, Any]]):
 
     def calculate_node_config_dict(
         self,
-        config_calls: List[Dict[str, Any]],
+        config_call_dict: Dict[str, Any],
         fqn: List[str],
         resource_type: NodeType,
         project_name: str,
         base: bool,
+        patch_config_dict: dict = None
     ) -> Dict[str, Any]:
         return self.calculate_node_config(
-            config_calls=config_calls,
+            config_call_dict=config_call_dict,
             fqn=fqn,
             resource_type=resource_type,
             project_name=project_name,
             base=base,
+            patch_config_dict=patch_config_dict
         )
 
     def initial_result(
@@ -251,20 +264,39 @@ class ContextConfig:
         resource_type: NodeType,
         project_name: str,
     ) -> None:
-        self._config_calls: List[Dict[str, Any]] = []
+        self._config_call_dict: Dict[str, Any] = {}
         self._active_project = active_project
         self._fqn = fqn
         self._resource_type = resource_type
         self._project_name = project_name
 
-    def update_in_model_config(self, opts: Dict[str, Any]) -> None:
-        self._config_calls.append(opts)
+    def add_config_call(self, opts: Dict[str, Any]) -> None:
+        dct = self._config_call_dict
+        self._add_config_call(dct, opts)
+
+    @classmethod
+    def _add_config_call(cls, config_call_dict, opts: Dict[str, Any]) -> None:
+        for k, v in opts.items():
+            # MergeBehavior for post-hook and pre-hook is to collect all
+            # values, instead of overwriting
+            if k in BaseConfig.mergebehavior['append']:
+                if not isinstance(v, list):
+                    v = [v]
+            if k in BaseConfig.mergebehavior['update'] and not isinstance(v, dict):
+                raise InternalException(f'expected dict, got {v}')
+            if k in config_call_dict and isinstance(config_call_dict[k], list):
+                config_call_dict[k].extend(v)
+            elif k in config_call_dict and isinstance(config_call_dict[k], dict):
+                config_call_dict[k].update(v)
+            else:
+                config_call_dict[k] = v
 
     def build_config_dict(
         self,
         base: bool = False,
         *,
         rendered: bool = True,
+        patch_config_dict: dict = None
     ) -> Dict[str, Any]:
         if rendered:
             src = ContextConfigGenerator(self._active_project)
@@ -272,9 +304,10 @@ class ContextConfig:
             src = UnrenderedConfigGenerator(self._active_project)
 
         return src.calculate_node_config_dict(
-            config_calls=self._config_calls,
+            config_call_dict=self._config_call_dict,
             fqn=self._fqn,
             resource_type=self._resource_type,
             project_name=self._project_name,
             base=base,
+            patch_config_dict=patch_config_dict
         )

core/dbt/context/macro_resolver.py

@@ -169,6 +169,8 @@ class TestMacroNamespace:
 
     def recursively_get_depends_on_macros(self, depends_on_macros, dep_macros):
         for macro_unique_id in depends_on_macros:
+            if macro_unique_id in dep_macros:
+                continue
             dep_macros.append(macro_unique_id)
             if macro_unique_id in self.macro_resolver.macros:
                 macro = self.macro_resolver.macros[macro_unique_id]

core/dbt/context/providers.py

@@ -279,7 +279,7 @@ class Config(Protocol):
         ...
 
 
-# `config` implementations
+# Implementation of "config(..)" calls in models
 class ParseConfigObject(Config):
     def __init__(self, model, context_config: Optional[ContextConfig]):
         self.model = model
@@ -316,7 +316,7 @@ class ParseConfigObject(Config):
             raise RuntimeException(
                 'At parse time, did not receive a context config'
             )
-        self.context_config.update_in_model_config(opts)
+        self.context_config.add_config_call(opts)
         return ''
 
     def set(self, name, value):
@@ -1243,7 +1243,7 @@ class ModelContext(ProviderContext):
 
     @contextproperty
     def pre_hooks(self) -> List[Dict[str, Any]]:
-        if isinstance(self.model, ParsedSourceDefinition):
+        if self.model.resource_type in [NodeType.Source, NodeType.Test]:
             return []
         return [
             h.to_dict(omit_none=True) for h in self.model.config.pre_hook
@@ -1251,7 +1251,7 @@ class ModelContext(ProviderContext):
 
     @contextproperty
     def post_hooks(self) -> List[Dict[str, Any]]:
-        if isinstance(self.model, ParsedSourceDefinition):
+        if self.model.resource_type in [NodeType.Source, NodeType.Test]:
             return []
         return [
             h.to_dict(omit_none=True) for h in self.model.config.post_hook

core/dbt/contracts/connection.py

@@ -1,5 +1,6 @@
 import abc
 import itertools
+import hashlib
 from dataclasses import dataclass, field
 from typing import (
     Any, ClassVar, Dict, Tuple, Iterable, Optional, List, Callable,
@@ -127,6 +128,15 @@ class Credentials(
             'type not implemented for base credentials class'
         )
 
+    @abc.abstractproperty
+    def unique_field(self) -> str:
+        raise NotImplementedError(
+            'type not implemented for base credentials class'
+        )
+
+    def hashed_unique_field(self) -> str:
+        return hashlib.md5(self.unique_field.encode('utf-8')).hexdigest()
+
     def connection_info(
         self, *, with_aliases: bool = False
     ) -> Iterable[Tuple[str, Any]]:

core/dbt/contracts/files.py

@@ -156,20 +156,11 @@ class BaseSourceFile(dbtClassMixin, SerializableType):
 
     def _serialize(self):
         dct = self.to_dict()
-        if 'pp_files' in dct:
-            del dct['pp_files']
-        if 'pp_test_index' in dct:
-            del dct['pp_test_index']
         return dct
 
     @classmethod
     def _deserialize(cls, dct: Dict[str, int]):
         if dct['parse_file_type'] == 'schema':
-            # TODO: why are these keys even here
-            if 'pp_files' in dct:
-                del dct['pp_files']
-            if 'pp_test_index' in dct:
-                del dct['pp_test_index']
             sf = SchemaSourceFile.from_dict(dct)
         else:
             sf = SourceFile.from_dict(dct)
@@ -223,13 +214,13 @@ class SourceFile(BaseSourceFile):
 class SchemaSourceFile(BaseSourceFile):
     dfy: Dict[str, Any] = field(default_factory=dict)
     # these are in the manifest.nodes dictionary
-    tests: List[str] = field(default_factory=list)
+    tests: Dict[str, Any] = field(default_factory=dict)
     sources: List[str] = field(default_factory=list)
     exposures: List[str] = field(default_factory=list)
     # node patches contain models, seeds, snapshots, analyses
     ndp: List[str] = field(default_factory=list)
     # any macro patches in this file by macro unique_id.
-    mcp: List[str] = field(default_factory=list)
+    mcp: Dict[str, str] = field(default_factory=dict)
     # any source patches in this file. The entries are package, name pairs
     # Patches are only against external sources. Sources can be
     # created too, but those are in 'sources'
@@ -255,14 +246,53 @@ class SchemaSourceFile(BaseSourceFile):
 
     def __post_serialize__(self, dct):
         dct = super().__post_serialize__(dct)
-        if 'pp_files' in dct:
-            del dct['pp_files']
-        if 'pp_test_index' in dct:
-            del dct['pp_test_index']
+        # Remove partial parsing specific data
+        for key in ('pp_files', 'pp_test_index', 'pp_dict'):
+            if key in dct:
+                del dct[key]
         return dct
 
     def append_patch(self, yaml_key, unique_id):
         self.node_patches.append(unique_id)
 
+    def add_test(self, node_unique_id, test_from):
+        name = test_from['name']
+        key = test_from['key']
+        if key not in self.tests:
+            self.tests[key] = {}
+        if name not in self.tests[key]:
+            self.tests[key][name] = []
+        self.tests[key][name].append(node_unique_id)
+
+    def remove_tests(self, yaml_key, name):
+        if yaml_key in self.tests:
+            if name in self.tests[yaml_key]:
+                del self.tests[yaml_key][name]
+
+    def get_tests(self, yaml_key, name):
+        if yaml_key in self.tests:
+            if name in self.tests[yaml_key]:
+                return self.tests[yaml_key][name]
+        return []
+
+    def get_key_and_name_for_test(self, test_unique_id):
+        yaml_key = None
+        block_name = None
+        for key in self.tests.keys():
+            for name in self.tests[key]:
+                for unique_id in self.tests[key][name]:
+                    if unique_id == test_unique_id:
+                        yaml_key = key
+                        block_name = name
+                        break
+        return (yaml_key, block_name)
+
+    def get_all_test_ids(self):
+        test_ids = []
+        for key in self.tests.keys():
+            for name in self.tests[key]:
+                test_ids.extend(self.tests[key][name])
+        return test_ids
+
 
 AnySourceFile = Union[SchemaSourceFile, SourceFile]

core/dbt/contracts/graph/compiled.py

@@ -109,7 +109,9 @@ class CompiledSnapshotNode(CompiledNode):
 @dataclass
 class CompiledDataTestNode(CompiledNode):
     resource_type: NodeType = field(metadata={'restrict': [NodeType.Test]})
-    config: TestConfig = field(default_factory=TestConfig)
+    # Was not able to make mypy happy and keep the code working. We need to
+    # refactor the various configs.
+    config: TestConfig = field(default_factory=TestConfig)  # type:ignore
 
 
 @dataclass
@@ -117,7 +119,9 @@ class CompiledSchemaTestNode(CompiledNode, HasTestMetadata):
     # keep this in sync with ParsedSchemaTestNode!
     resource_type: NodeType = field(metadata={'restrict': [NodeType.Test]})
     column_name: Optional[str] = None
-    config: TestConfig = field(default_factory=TestConfig)
+    # Was not able to make mypy happy and keep the code working. We need to
+    # refactor the various configs.
+    config: TestConfig = field(default_factory=TestConfig)  # type:ignore
 
     def same_contents(self, other) -> bool:
         if other is None:

core/dbt/contracts/graph/manifest.py

@@ -14,7 +14,7 @@ from dbt.contracts.graph.compiled import (
     CompileResultNode, ManifestNode, NonSourceCompiledNode, GraphMemberNode
 )
 from dbt.contracts.graph.parsed import (
-    ParsedMacro, ParsedDocumentation, ParsedNodePatch, ParsedMacroPatch,
+    ParsedMacro, ParsedDocumentation,
     ParsedSourceDefinition, ParsedExposure, HasUniqueID,
     UnpatchedSourceDefinition, ManifestNodes
 )
@@ -26,9 +26,7 @@ from dbt.contracts.util import (
 from dbt.dataclass_schema import dbtClassMixin
 from dbt.exceptions import (
     CompilationException,
-    raise_duplicate_resource_name, raise_compiler_error, warn_or_error,
-    raise_duplicate_patch_name,
-    raise_duplicate_macro_patch_name, raise_duplicate_source_patch_name,
+    raise_duplicate_resource_name, raise_compiler_error,
 )
 from dbt.helper_types import PathSet
 from dbt.logger import GLOBAL_LOGGER as logger
@@ -172,7 +170,7 @@ class RefableLookup(dbtClassMixin):
 
 
 class AnalysisLookup(RefableLookup):
-    _lookup_types: ClassVar[set] = set(NodeType.Analysis)
+    _lookup_types: ClassVar[set] = set([NodeType.Analysis])
 
 
 def _search_packages(
@@ -243,7 +241,7 @@ def _sort_values(dct):
     return {k: sorted(v) for k, v in dct.items()}
 
 
-def build_edges(nodes: List[ManifestNode]):
+def build_node_edges(nodes: List[ManifestNode]):
     """Build the forward and backward edges on the given list of ParsedNodes
     and return them as two separate dictionaries, each mapping unique IDs to
     lists of edges.
@@ -259,6 +257,18 @@ def build_edges(nodes: List[ManifestNode]):
     return _sort_values(forward_edges), _sort_values(backward_edges)
 
 
+# Build a map of children of macros
+def build_macro_edges(nodes: List[Any]):
+    forward_edges: Dict[str, List[str]] = {
+        n.unique_id: [] for n in nodes if n.unique_id.startswith('macro') or n.depends_on.macros
+    }
+    for node in nodes:
+        for unique_id in node.depends_on.macros:
+            if unique_id in forward_edges.keys():
+                forward_edges[unique_id].append(node.unique_id)
+    return _sort_values(forward_edges)
+
+
 def _deepcopy(value):
     return value.from_dict(value.to_dict(omit_none=True))
 
@@ -525,6 +535,12 @@ class MacroMethods:
         return candidates
 
 
+@dataclass
+class ParsingInfo:
+    static_analysis_parsed_path_count: int = 0
+    static_analysis_path_count: int = 0
+
+
 @dataclass
 class ManifestStateCheck(dbtClassMixin):
     vars_hash: FileHash = field(default_factory=FileHash.empty)
@@ -566,9 +582,13 @@ class Manifest(MacroMethods, DataClassMessagePackMixin, dbtClassMixin):
     _analysis_lookup: Optional[AnalysisLookup] = field(
         default=None, metadata={'serialize': lambda x: None, 'deserialize': lambda x: None}
     )
+    _parsing_info: ParsingInfo = field(
+        default_factory=ParsingInfo,
+        metadata={'serialize': lambda x: None, 'deserialize': lambda x: None}
+    )
     _lock: Lock = field(
         default_factory=flags.MP_CONTEXT.Lock,
-        metadata={'serialize': lambda x: None, 'deserialize': lambda x: flags.MP_CONTEXT.Lock}
+        metadata={'serialize': lambda x: None, 'deserialize': lambda x: None}
     )
 
     def __pre_serialize__(self):
@@ -577,6 +597,11 @@ class Manifest(MacroMethods, DataClassMessagePackMixin, dbtClassMixin):
         self.source_patches = {}
         return self
 
+    @classmethod
+    def __post_deserialize__(cls, obj):
+        obj._lock = flags.MP_CONTEXT.Lock()
+        return obj
+
     def sync_update_node(
         self, new_node: NonSourceCompiledNode
     ) -> NonSourceCompiledNode:
@@ -691,60 +716,6 @@ class Manifest(MacroMethods, DataClassMessagePackMixin, dbtClassMixin):
             resource_fqns[resource_type_plural].add(tuple(resource.fqn))
         return resource_fqns
 
-    # This is called by 'parse_patch' in the NodePatchParser
-    def add_patch(
-        self, source_file: SchemaSourceFile, patch: ParsedNodePatch,
-    ) -> None:
-        if patch.yaml_key in ['models', 'seeds', 'snapshots']:
-            unique_id = self.ref_lookup.get_unique_id(patch.name, None)
-        elif patch.yaml_key == 'analyses':
-            unique_id = self.analysis_lookup.get_unique_id(patch.name, None)
-        else:
-            raise dbt.exceptions.InternalException(
-                f'Unexpected yaml_key {patch.yaml_key} for patch in '
-                f'file {source_file.path.original_file_path}'
-            )
-        if unique_id is None:
-            # This will usually happen when a node is disabled
-            return
-
-        # patches can't be overwritten
-        node = self.nodes.get(unique_id)
-        if node:
-            if node.patch_path:
-                package_name, existing_file_path = node.patch_path.split('://')
-                raise_duplicate_patch_name(patch, existing_file_path)
-            source_file.append_patch(patch.yaml_key, unique_id)
-            node.patch(patch)
-
-    def add_macro_patch(
-        self, source_file: SchemaSourceFile, patch: ParsedMacroPatch,
-    ) -> None:
-        # macros are fully namespaced
-        unique_id = f'macro.{patch.package_name}.{patch.name}'
-        macro = self.macros.get(unique_id)
-        if not macro:
-            warn_or_error(
-                f'WARNING: Found documentation for macro "{patch.name}" '
-                f'which was not found'
-            )
-            return
-        if macro.patch_path:
-            package_name, existing_file_path = macro.patch_path.split('://')
-            raise_duplicate_macro_patch_name(patch, existing_file_path)
-        source_file.macro_patches.append(unique_id)
-        macro.patch(patch)
-
-    def add_source_patch(
-        self, source_file: SchemaSourceFile, patch: SourcePatch,
-    ) -> None:
-        # source patches must be unique
-        key = (patch.overrides, patch.name)
-        if key in self.source_patches:
-            raise_duplicate_source_patch_name(patch, self.source_patches[key])
-        self.source_patches[key] = patch
-        source_file.source_patches.append(key)
-
     def get_used_schemas(self, resource_types=None):
         return frozenset({
             (node.database, node.schema) for node in
@@ -779,10 +750,18 @@ class Manifest(MacroMethods, DataClassMessagePackMixin, dbtClassMixin):
             self.sources.values(),
             self.exposures.values(),
         ))
-        forward_edges, backward_edges = build_edges(edge_members)
+        forward_edges, backward_edges = build_node_edges(edge_members)
         self.child_map = forward_edges
         self.parent_map = backward_edges
 
+    def build_macro_child_map(self):
+        edge_members = list(chain(
+            self.nodes.values(),
+            self.macros.values(),
+        ))
+        forward_edges = build_macro_edges(edge_members)
+        return forward_edges
+
     def writable_manifest(self):
         self.build_parent_and_child_maps()
         return WritableManifest(
@@ -1016,10 +995,11 @@ class Manifest(MacroMethods, DataClassMessagePackMixin, dbtClassMixin):
         _check_duplicates(node, self.nodes)
         self.nodes[node.unique_id] = node
 
-    def add_node(self, source_file: AnySourceFile, node: ManifestNodes):
+    def add_node(self, source_file: AnySourceFile, node: ManifestNodes, test_from=None):
         self.add_node_nofile(node)
         if isinstance(source_file, SchemaSourceFile):
-            source_file.tests.append(node.unique_id)
+            assert test_from
+            source_file.add_test(node.unique_id, test_from)
         else:
             source_file.nodes.append(node.unique_id)
 
@@ -1034,10 +1014,11 @@ class Manifest(MacroMethods, DataClassMessagePackMixin, dbtClassMixin):
         else:
             self._disabled[node.unique_id] = [node]
 
-    def add_disabled(self, source_file: AnySourceFile, node: CompileResultNode):
+    def add_disabled(self, source_file: AnySourceFile, node: CompileResultNode, test_from=None):
         self.add_disabled_nofile(node)
         if isinstance(source_file, SchemaSourceFile):
-            source_file.tests.append(node.unique_id)
+            assert test_from
+            source_file.add_test(node.unique_id, test_from)
         else:
             source_file.nodes.append(node.unique_id)
 

core/dbt/contracts/graph/model_config.py

@@ -2,13 +2,13 @@ from dataclasses import field, Field, dataclass
 from enum import Enum
 from itertools import chain
 from typing import (
-    Any, List, Optional, Dict, Union, Type, TypeVar
+    Any, List, Optional, Dict, Union, Type, TypeVar, Callable
 )
 from dbt.dataclass_schema import (
     dbtClassMixin, ValidationError, register_pattern,
 )
 from dbt.contracts.graph.unparsed import AdditionalPropertiesAllowed
-from dbt.exceptions import InternalException
+from dbt.exceptions import InternalException, CompilationException
 from dbt.contracts.util import Replaceable, list_str
 from dbt import hooks
 from dbt.node_types import NodeType
@@ -204,6 +204,34 @@ class BaseConfig(
         else:
             self._extra[key] = value
 
+    def __delitem__(self, key):
+        if hasattr(self, key):
+            msg = (
+                'Error, tried to delete config key "{}": Cannot delete '
+                'built-in keys'
+            ).format(key)
+            raise CompilationException(msg)
+        else:
+            del self._extra[key]
+
+    def _content_iterator(self, include_condition: Callable[[Field], bool]):
+        seen = set()
+        for fld, _ in self._get_fields():
+            seen.add(fld.name)
+            if include_condition(fld):
+                yield fld.name
+
+        for key in self._extra:
+            if key not in seen:
+                seen.add(key)
+                yield key
+
+    def __iter__(self):
+        yield from self._content_iterator(include_condition=lambda f: True)
+
+    def __len__(self):
+        return len(self._get_fields()) + len(self._extra)
+
     @staticmethod
     def compare_key(
         unrendered: Dict[str, Any],
@@ -239,8 +267,15 @@ class BaseConfig(
                     return False
         return True
 
+    # This is used in 'add_config_call' to created the combined config_call_dict.
+    # 'meta' moved here from node
+    mergebehavior = {
+        "append": ['pre-hook', 'pre_hook', 'post-hook', 'post_hook', 'tags'],
+        "update": ['quoting', 'column_types', 'meta'],
+    }
+
     @classmethod
-    def _extract_dict(
+    def _merge_dicts(
         cls, src: Dict[str, Any], data: Dict[str, Any]
     ) -> Dict[str, Any]:
         """Find all the items in data that match a target_field on this class,
@@ -286,10 +321,10 @@ class BaseConfig(
 
         adapter_config_cls = get_config_class_by_name(adapter_type)
 
-        self_merged = self._extract_dict(dct, data)
+        self_merged = self._merge_dicts(dct, data)
         dct.update(self_merged)
 
-        adapter_merged = adapter_config_cls._extract_dict(dct, data)
+        adapter_merged = adapter_config_cls._merge_dicts(dct, data)
         dct.update(adapter_merged)
 
         # any remaining fields must be "clobber"
@@ -321,33 +356,8 @@ class SourceConfig(BaseConfig):
 
 
 @dataclass
-class NodeConfig(BaseConfig):
+class NodeAndTestConfig(BaseConfig):
     enabled: bool = True
-    materialized: str = 'view'
-    persist_docs: Dict[str, Any] = field(default_factory=dict)
-    post_hook: List[Hook] = field(
-        default_factory=list,
-        metadata=MergeBehavior.Append.meta(),
-    )
-    pre_hook: List[Hook] = field(
-        default_factory=list,
-        metadata=MergeBehavior.Append.meta(),
-    )
-    # this only applies for config v1, so it doesn't participate in comparison
-    vars: Dict[str, Any] = field(
-        default_factory=dict,
-        metadata=metas(CompareBehavior.Exclude, MergeBehavior.Update),
-    )
-    quoting: Dict[str, Any] = field(
-        default_factory=dict,
-        metadata=MergeBehavior.Update.meta(),
-    )
-    # This is actually only used by seeds. Should it be available to others?
-    # That would be a breaking change!
-    column_types: Dict[str, Any] = field(
-        default_factory=dict,
-        metadata=MergeBehavior.Update.meta(),
-    )
     # these fields are included in serialized output, but are not part of
     # config comparison (they are part of database_representation)
     alias: Optional[str] = field(
@@ -368,7 +378,38 @@ class NodeConfig(BaseConfig):
                        MergeBehavior.Append,
                        CompareBehavior.Exclude),
     )
+    meta: Dict[str, Any] = field(
+        default_factory=dict,
+        metadata=MergeBehavior.Update.meta(),
+    )
+
+
+@dataclass
+class NodeConfig(NodeAndTestConfig):
+    # Note: if any new fields are added with MergeBehavior, also update the
+    # 'mergebehavior' dictionary
+    materialized: str = 'view'
+    persist_docs: Dict[str, Any] = field(default_factory=dict)
+    post_hook: List[Hook] = field(
+        default_factory=list,
+        metadata=MergeBehavior.Append.meta(),
+    )
+    pre_hook: List[Hook] = field(
+        default_factory=list,
+        metadata=MergeBehavior.Append.meta(),
+    )
+    quoting: Dict[str, Any] = field(
+        default_factory=dict,
+        metadata=MergeBehavior.Update.meta(),
+    )
+    # This is actually only used by seeds. Should it be available to others?
+    # That would be a breaking change!
+    column_types: Dict[str, Any] = field(
+        default_factory=dict,
+        metadata=MergeBehavior.Update.meta(),
+    )
     full_refresh: Optional[bool] = None
+    on_schema_change: Optional[str] = 'ignore'
 
     @classmethod
     def __pre_deserialize__(cls, data):
@@ -410,7 +451,8 @@ class SeedConfig(NodeConfig):
 
 
 @dataclass
-class TestConfig(NodeConfig):
+class TestConfig(NodeAndTestConfig):
+    # this is repeated because of a different default
     schema: Optional[str] = field(
         default='dbt_test__audit',
         metadata=CompareBehavior.Exclude.meta(),

core/dbt/contracts/graph/parsed.py

@@ -148,6 +148,7 @@ class ParsedNodeMixins(dbtClassMixin):
         """Given a ParsedNodePatch, add the new information to the node."""
         # explicitly pick out the parts to update so we don't inadvertently
         # step on the model name or anything
+        # Note: config should already be updated
         self.patch_path: Optional[str] = patch.file_id
         # update created_at so process_docs will run in partial parsing
         self.created_at = int(time.time())
@@ -166,9 +167,6 @@ class ParsedNodeMixins(dbtClassMixin):
     def get_materialization(self):
         return self.config.materialized
 
-    def local_vars(self):
-        return self.config.vars
-
 
 @dataclass
 class ParsedNodeMandatory(
@@ -203,6 +201,7 @@ class ParsedNodeDefaults(ParsedNodeMandatory):
     deferred: bool = False
     unrendered_config: Dict[str, Any] = field(default_factory=dict)
     created_at: int = field(default_factory=lambda: int(time.time()))
+    config_call_dict: Dict[str, Any] = field(default_factory=dict)
 
     def write_node(self, target_path: str, subdirectory: str, payload: str):
         if (os.path.basename(self.path) ==
@@ -229,6 +228,11 @@ class ParsedNode(ParsedNodeDefaults, ParsedNodeMixins, SerializableType):
     def _serialize(self):
         return self.to_dict()
 
+    def __post_serialize__(self, dct):
+        if 'config_call_dict' in dct:
+            del dct['config_call_dict']
+        return dct
+
     @classmethod
     def _deserialize(cls, dct: Dict[str, int]):
         # The serialized ParsedNodes do not differ from each other
@@ -258,10 +262,16 @@ class ParsedNode(ParsedNodeDefaults, ParsedNodeMixins, SerializableType):
             return cls.from_dict(dct)
 
     def _persist_column_docs(self) -> bool:
-        return bool(self.config.persist_docs.get('columns'))
+        if hasattr(self.config, 'persist_docs'):
+            assert isinstance(self.config, NodeConfig)
+            return bool(self.config.persist_docs.get('columns'))
+        return False
 
     def _persist_relation_docs(self) -> bool:
-        return bool(self.config.persist_docs.get('relation'))
+        if hasattr(self.config, 'persist_docs'):
+            assert isinstance(self.config, NodeConfig)
+            return bool(self.config.persist_docs.get('relation'))
+        return False
 
     def same_body(self: T, other: T) -> bool:
         return self.raw_sql == other.raw_sql
@@ -411,7 +421,9 @@ class HasTestMetadata(dbtClassMixin):
 @dataclass
 class ParsedDataTestNode(ParsedNode):
     resource_type: NodeType = field(metadata={'restrict': [NodeType.Test]})
-    config: TestConfig = field(default_factory=TestConfig)
+    # Was not able to make mypy happy and keep the code working. We need to
+    # refactor the various configs.
+    config: TestConfig = field(default_factory=TestConfig)  # type: ignore
 
 
 @dataclass
@@ -419,7 +431,9 @@ class ParsedSchemaTestNode(ParsedNode, HasTestMetadata):
     # keep this in sync with CompiledSchemaTestNode!
     resource_type: NodeType = field(metadata={'restrict': [NodeType.Test]})
     column_name: Optional[str] = None
-    config: TestConfig = field(default_factory=TestConfig)
+    # Was not able to make mypy happy and keep the code working. We need to
+    # refactor the various configs.
+    config: TestConfig = field(default_factory=TestConfig)  # type: ignore
 
     def same_contents(self, other) -> bool:
         if other is None:
@@ -456,6 +470,7 @@ class ParsedPatch(HasYamlMetadata, Replaceable):
     description: str
     meta: Dict[str, Any]
     docs: Docs
+    config: Dict[str, Any]
 
 
 # The parsed node update is only the 'patch', not the test. The test became a
@@ -487,9 +502,6 @@ class ParsedMacro(UnparsedBaseNode, HasUniqueID):
     arguments: List[MacroArgument] = field(default_factory=list)
     created_at: int = field(default_factory=lambda: int(time.time()))
 
-    def local_vars(self):
-        return {}
-
     def patch(self, patch: ParsedMacroPatch):
         self.patch_path: Optional[str] = patch.file_id
         self.description = patch.description
@@ -592,7 +604,8 @@ class ParsedSourceDefinition(
     UnparsedBaseNode,
     HasUniqueID,
     HasRelationMetadata,
-    HasFqn
+    HasFqn,
+
 ):
     name: str
     source_name: str
@@ -689,6 +702,10 @@ class ParsedSourceDefinition(
     def depends_on_nodes(self):
         return []
 
+    @property
+    def depends_on(self):
+        return DependsOn(macros=[], nodes=[])
+
     @property
     def refs(self):
         return []

core/dbt/contracts/graph/unparsed.py

@@ -126,12 +126,17 @@ class HasYamlMetadata(dbtClassMixin):
 
 
 @dataclass
-class UnparsedAnalysisUpdate(HasColumnDocs, HasDocs, HasYamlMetadata):
+class HasConfig():
+    config: Dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass
+class UnparsedAnalysisUpdate(HasConfig, HasColumnDocs, HasDocs, HasYamlMetadata):
     pass
 
 
 @dataclass
-class UnparsedNodeUpdate(HasColumnTests, HasTests, HasYamlMetadata):
+class UnparsedNodeUpdate(HasConfig, HasColumnTests, HasTests, HasYamlMetadata):
     quote_columns: Optional[bool] = None
 
 
@@ -143,7 +148,7 @@ class MacroArgument(dbtClassMixin):
 
 
 @dataclass
-class UnparsedMacroUpdate(HasDocs, HasYamlMetadata):
+class UnparsedMacroUpdate(HasConfig, HasDocs, HasYamlMetadata):
     arguments: List[MacroArgument] = field(default_factory=list)
 
 
@@ -261,6 +266,7 @@ class UnparsedSourceDefinition(dbtClassMixin, Replaceable):
     loaded_at_field: Optional[str] = None
     tables: List[UnparsedSourceTableDefinition] = field(default_factory=list)
     tags: List[str] = field(default_factory=list)
+    config: Dict[str, Any] = field(default_factory=dict)
 
     @property
     def yaml_key(self) -> 'str':

core/dbt/contracts/project.py

@@ -83,6 +83,7 @@ class GitPackage(Package):
 class RegistryPackage(Package):
     package: str
     version: Union[RawVersion, List[RawVersion]]
+    install_prerelease: Optional[bool] = False
 
     def get_versions(self) -> List[str]:
         if isinstance(self.version, list):

core/dbt/contracts/results.py

@@ -78,6 +78,7 @@ class TestStatus(StrEnum):
     Error = NodeStatus.Error
     Fail = NodeStatus.Fail
     Warn = NodeStatus.Warn
+    Skipped = NodeStatus.Skipped
 
 
 class FreshnessStatus(StrEnum):

core/dbt/contracts/rpc.py

@@ -58,6 +58,7 @@ class RPCExecParameters(RPCParameters):
 class RPCCompileParameters(RPCParameters):
     threads: Optional[int] = None
     models: Union[None, str, List[str]] = None
+    select: Union[None, str, List[str]] = None
     exclude: Union[None, str, List[str]] = None
     selector: Optional[str] = None
     state: Optional[str] = None
@@ -71,12 +72,14 @@ class RPCListParameters(RPCParameters):
     select: Union[None, str, List[str]] = None
     selector: Optional[str] = None
     output: Optional[str] = 'json'
+    output_keys: Optional[List[str]] = None
 
 
 @dataclass
 class RPCRunParameters(RPCParameters):
     threads: Optional[int] = None
     models: Union[None, str, List[str]] = None
+    select: Union[None, str, List[str]] = None
     exclude: Union[None, str, List[str]] = None
     selector: Optional[str] = None
     state: Optional[str] = None
@@ -116,6 +119,17 @@ class RPCDocsGenerateParameters(RPCParameters):
     state: Optional[str] = None
 
 
+@dataclass
+class RPCBuildParameters(RPCParameters):
+    threads: Optional[int] = None
+    models: Union[None, str, List[str]] = None
+    select: Union[None, str, List[str]] = None
+    exclude: Union[None, str, List[str]] = None
+    selector: Optional[str] = None
+    state: Optional[str] = None
+    defer: Optional[bool] = None
+
+
 @dataclass
 class RPCCliParameters(RPCParameters):
     cli: str
@@ -186,6 +200,8 @@ class RPCRunOperationParameters(RPCParameters):
 class RPCSourceFreshnessParameters(RPCParameters):
     threads: Optional[int] = None
     select: Union[None, str, List[str]] = None
+    exclude: Union[None, str, List[str]] = None
+    selector: Optional[str] = None
 
 
 @dataclass

core/dbt/deprecations.py

@@ -131,6 +131,14 @@ class AdapterMacroDeprecation(DBTDeprecation):
     '''
 
 
+class PackageRedirectDeprecation(DBTDeprecation):
+    _name = 'package-redirect'
+    _description = '''\
+    The `{old_name}` package is deprecated in favor of `{new_name}`. Please update
+    your `packages.yml` configuration to use `{new_name}` instead.
+    '''
+
+
 _adapter_renamed_description = """\
 The adapter function `adapter.{old_name}` is deprecated and will be removed in
 a future release of dbt. Please use `adapter.{new_name}` instead.
@@ -176,6 +184,7 @@ deprecations_list: List[DBTDeprecation] = [
     ModelsKeyNonModelDeprecation(),
     ExecuteMacrosReleaseDeprecation(),
     AdapterMacroDeprecation(),
+    PackageRedirectDeprecation()
 ]
 
 deprecations: Dict[str, DBTDeprecation] = {

core/dbt/deps/registry.py

@@ -30,9 +30,13 @@ class RegistryPackageMixin:
 
 
 class RegistryPinnedPackage(RegistryPackageMixin, PinnedPackage):
-    def __init__(self, package: str, version: str) -> None:
+    def __init__(self,
+                 package: str,
+                 version: str,
+                 version_latest: str) -> None:
         super().__init__(package)
         self.version = version
+        self.version_latest = version_latest
 
     @property
     def name(self):
@@ -44,6 +48,9 @@ class RegistryPinnedPackage(RegistryPackageMixin, PinnedPackage):
     def get_version(self):
         return self.version
 
+    def get_version_latest(self):
+        return self.version_latest
+
     def nice_version_name(self):
         return 'version {}'.format(self.version)
 
@@ -61,7 +68,7 @@ class RegistryPinnedPackage(RegistryPackageMixin, PinnedPackage):
         system.make_directory(os.path.dirname(tar_path))
 
         download_url = metadata.downloads.tarball
-        system.download(download_url, tar_path)
+        system.download_with_retries(download_url, tar_path)
         deps_path = project.modules_path
         package_name = self.get_project_name(project, renderer)
         system.untar_package(tar_path, deps_path, package_name)
@@ -71,10 +78,14 @@ class RegistryUnpinnedPackage(
     RegistryPackageMixin, UnpinnedPackage[RegistryPinnedPackage]
 ):
     def __init__(
-        self, package: str, versions: List[semver.VersionSpecifier]
+        self,
+        package: str,
+        versions: List[semver.VersionSpecifier],
+        install_prerelease: bool
     ) -> None:
         super().__init__(package)
         self.versions = versions
+        self.install_prerelease = install_prerelease
 
     def _check_in_index(self):
         index = registry.index_cached()
@@ -91,13 +102,18 @@ class RegistryUnpinnedPackage(
             semver.VersionSpecifier.from_version_string(v)
             for v in raw_version
         ]
-        return cls(package=contract.package, versions=versions)
+        return cls(
+            package=contract.package,
+            versions=versions,
+            install_prerelease=contract.install_prerelease
+        )
 
     def incorporate(
         self, other: 'RegistryUnpinnedPackage'
     ) -> 'RegistryUnpinnedPackage':
         return RegistryUnpinnedPackage(
             package=self.package,
+            install_prerelease=self.install_prerelease,
             versions=self.versions + other.versions,
         )
 
@@ -111,12 +127,18 @@ class RegistryUnpinnedPackage(
             raise DependencyException(new_msg) from e
 
         available = registry.get_available_versions(self.package)
+        installable = semver.filter_installable(
+            available,
+            self.install_prerelease
+        )
+        available_latest = installable[-1]
 
         # for now, pick a version and then recurse. later on,
         # we'll probably want to traverse multiple options
         # so we can match packages. not going to make a difference
         # right now.
-        target = semver.resolve_to_specific_version(range_, available)
+        target = semver.resolve_to_specific_version(range_, installable)
         if not target:
-            package_version_not_found(self.package, range_, available)
-        return RegistryPinnedPackage(package=self.package, version=target)
+            package_version_not_found(self.package, range_, installable)
+        return RegistryPinnedPackage(package=self.package, version=target,
+                                     version_latest=available_latest)

core/dbt/exceptions.py

@@ -710,11 +710,11 @@ def system_error(operation_name):
     raise_compiler_error(
         "dbt encountered an error when attempting to {}. "
         "If this error persists, please create an issue at: \n\n"
-        "https://github.com/fishtown-analytics/dbt"
+        "https://github.com/dbt-labs/dbt"
         .format(operation_name))
 
 
-class RegistryException(Exception):
+class ConnectionException(Exception):
     pass
 
 

core/dbt/graph/queue.py

@@ -1,10 +1,8 @@
-import threading
-from queue import PriorityQueue
-from typing import (
-    Dict, Set, Optional
-)
-
 import networkx as nx  # type: ignore
+import threading
+
+from queue import PriorityQueue
+from typing import Dict, Set, List, Generator, Optional
 
 from .graph import UniqueId
 from dbt.contracts.graph.parsed import ParsedSourceDefinition, ParsedExposure
@@ -21,9 +19,8 @@ class GraphQueue:
     that separate threads do not call `.empty()` or `__len__()` and `.get()` at
     the same time, as there is an unlocked race!
     """
-    def __init__(
-        self, graph: nx.DiGraph, manifest: Manifest, selected: Set[UniqueId]
-    ):
+
+    def __init__(self, graph: nx.DiGraph, manifest: Manifest, selected: Set[UniqueId]):
         self.graph = graph
         self.manifest = manifest
         self._selected = selected
@@ -37,7 +34,7 @@ class GraphQueue:
         # this lock controls most things
         self.lock = threading.Lock()
         # store the 'score' of each node as a number. Lower is higher priority.
-        self._scores = self._calculate_scores()
+        self._scores = self._get_scores(self.graph)
         # populate the initial queue
         self._find_new_additions()
         # awaits after task end
@@ -56,30 +53,59 @@ class GraphQueue:
             return False
         return True
 
-    def _calculate_scores(self) -> Dict[UniqueId, int]:
-        """Calculate the 'value' of each node in the graph based on how many
-        blocking descendants it has. We use this score for the internal
-        priority queue's ordering, so the quality of this metric is important.
+    @staticmethod
+    def _grouped_topological_sort(
+        graph: nx.DiGraph,
+    ) -> Generator[List[str], None, None]:
+        """Topological sort of given graph that groups ties.
 
-        The score is stored as a negative number because the internal
-        PriorityQueue picks lowest values first.
+        Adapted from `nx.topological_sort`, this function returns a topo sort of a graph however
+        instead of arbitrarily ordering ties in the sort order, ties are grouped together in
+        lists.
 
-        We could do this in one pass over the graph instead of len(self.graph)
-        passes but this is easy. For large graphs this may hurt performance.
+        Args:
+            graph: The graph to be sorted.
 
-        This operates on the graph, so it would require a lock if called from
-        outside __init__.
-
-        :return Dict[str, int]: The score dict, mapping unique IDs to integer
-            scores. Lower scores are higher priority.
+        Returns:
+            A generator that yields lists of nodes, one list per graph depth level.
         """
+        indegree_map = {v: d for v, d in graph.in_degree() if d > 0}
+        zero_indegree = [v for v, d in graph.in_degree() if d == 0]
+
+        while zero_indegree:
+            yield zero_indegree
+            new_zero_indegree = []
+            for v in zero_indegree:
+                for _, child in graph.edges(v):
+                    indegree_map[child] -= 1
+                    if not indegree_map[child]:
+                        new_zero_indegree.append(child)
+            zero_indegree = new_zero_indegree
+
+    def _get_scores(self, graph: nx.DiGraph) -> Dict[str, int]:
+        """Scoring nodes for processing order.
+
+        Scores are calculated by the graph depth level. Lowest score (0) should be processed first.
+
+        Args:
+            graph: The graph to be scored.
+
+        Returns:
+            A dictionary consisting of `node name`:`score` pairs.
+        """
+        # split graph by connected subgraphs
+        subgraphs = (
+            graph.subgraph(x) for x in nx.connected_components(nx.Graph(graph))
+        )
+
+        # score all nodes in all subgraphs
         scores = {}
-        for node in self.graph.nodes():
-            score = -1 * len([
-                d for d in nx.descendants(self.graph, node)
-                if self._include_in_cost(d)
-            ])
-            scores[node] = score
+        for subgraph in subgraphs:
+            grouped_nodes = self._grouped_topological_sort(subgraph)
+            for level, group in enumerate(grouped_nodes):
+                for node in group:
+                    scores[node] = level
+
         return scores
 
     def get(
@@ -133,8 +159,6 @@ class GraphQueue:
     def _find_new_additions(self) -> None:
         """Find any nodes in the graph that need to be added to the internal
         queue and add them.
-
-        Callers must hold the lock.
         """
         for node, in_degree in self.graph.in_degree():
             if not self._already_known(node) and in_degree == 0:

core/dbt/graph/selector_methods.py

@@ -22,13 +22,11 @@ from dbt.contracts.graph.parsed import (
     ParsedSourceDefinition,
 )
 from dbt.contracts.state import PreviousState
-from dbt.logger import GLOBAL_LOGGER as logger
 from dbt.exceptions import (
     InternalException,
     RuntimeException,
 )
 from dbt.node_types import NodeType
-from dbt.ui import warning_tag
 
 
 SELECTOR_GLOB = '*'
@@ -381,7 +379,7 @@ class TestTypeSelectorMethod(SelectorMethod):
 class StateSelectorMethod(SelectorMethod):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
-        self.macros_were_modified: Optional[List[str]] = None
+        self.modified_macros: Optional[List[str]] = None
 
     def _macros_modified(self) -> List[str]:
         # we checked in the caller!
@@ -394,44 +392,74 @@ class StateSelectorMethod(SelectorMethod):
 
         modified = []
         for uid, macro in new_macros.items():
-            name = f'{macro.package_name}.{macro.name}'
             if uid in old_macros:
                 old_macro = old_macros[uid]
                 if macro.macro_sql != old_macro.macro_sql:
-                    modified.append(f'{name} changed')
+                    modified.append(uid)
             else:
-                modified.append(f'{name} added')
+                modified.append(uid)
 
         for uid, macro in old_macros.items():
             if uid not in new_macros:
-                modified.append(f'{macro.package_name}.{macro.name} removed')
+                modified.append(uid)
 
-        return modified[:3]
+        return modified
 
-    def check_modified(
-        self,
-        old: Optional[SelectorTarget],
-        new: SelectorTarget,
+    def recursively_check_macros_modified(self, node):
+        # check if there are any changes in macros the first time
+        if self.modified_macros is None:
+            self.modified_macros = self._macros_modified()
+
+        # loop through all macros that this node depends on
+        for macro_uid in node.depends_on.macros:
+            # is this macro one of the modified macros?
+            if macro_uid in self.modified_macros:
+                return True
+            # if not, and this macro depends on other macros, keep looping
+            macro = self.manifest.macros[macro_uid]
+            if len(macro.depends_on.macros) > 0:
+                return self.recursively_check_macros_modified(macro)
+            else:
+                return False
+        return False
+
+    def check_modified(self, old: Optional[SelectorTarget], new: SelectorTarget) -> bool:
+        different_contents = not new.same_contents(old)  # type: ignore
+        upstream_macro_change = self.recursively_check_macros_modified(new)
+        return different_contents or upstream_macro_change
+
+    def check_modified_body(self, old: Optional[SelectorTarget], new: SelectorTarget) -> bool:
+        if hasattr(new, "same_body"):
+            return not new.same_body(old)  # type: ignore
+        else:
+            return False
+
+    def check_modified_configs(self, old: Optional[SelectorTarget], new: SelectorTarget) -> bool:
+        if hasattr(new, "same_config"):
+            return not new.same_config(old)  # type: ignore
+        else:
+            return False
+
+    def check_modified_persisted_descriptions(
+        self, old: Optional[SelectorTarget], new: SelectorTarget
     ) -> bool:
-        # check if there are any changes in macros, if so, log a warning the
-        # first time
-        if self.macros_were_modified is None:
-            self.macros_were_modified = self._macros_modified()
-            if self.macros_were_modified:
-                log_str = ', '.join(self.macros_were_modified)
-                logger.warning(warning_tag(
-                    f'During a state comparison, dbt detected a change in '
-                    f'macros. This will not be marked as a modification. Some '
-                    f'macros: {log_str}'
-                ))
+        if hasattr(new, "same_persisted_description"):
+            return not new.same_persisted_description(old)  # type: ignore
+        else:
+            return False
 
-        return not new.same_contents(old)  # type: ignore
-
-    def check_new(
-        self,
-        old: Optional[SelectorTarget],
-        new: SelectorTarget,
+    def check_modified_relation(
+        self, old: Optional[SelectorTarget], new: SelectorTarget
     ) -> bool:
+        if hasattr(new, "same_database_representation"):
+            return not new.same_database_representation(old)  # type: ignore
+        else:
+            return False
+
+    def check_modified_macros(self, _, new: SelectorTarget) -> bool:
+        return self.recursively_check_macros_modified(new)
+
+    def check_new(self, old: Optional[SelectorTarget], new: SelectorTarget) -> bool:
         return old is None
 
     def search(
@@ -443,8 +471,15 @@ class StateSelectorMethod(SelectorMethod):
             )
 
         state_checks = {
+            # it's new if there is no old version
+            'new': lambda old, _: old is None,
+            # use methods defined above to compare properties of old + new
             'modified': self.check_modified,
-            'new': self.check_new,
+            'modified.body': self.check_modified_body,
+            'modified.configs': self.check_modified_configs,
+            'modified.persisted_descriptions': self.check_modified_persisted_descriptions,
+            'modified.relation': self.check_modified_relation,
+            'modified.macros': self.check_modified_macros,
         }
         if selector in state_checks:
             checker = state_checks[selector]

core/dbt/include/global_project/macros/adapters/common.sql

@@ -1,5 +1,5 @@
 {% macro get_columns_in_query(select_sql) -%}
-  {{ return(adapter.dispatch('get_columns_in_query')(select_sql)) }}
+  {{ return(adapter.dispatch('get_columns_in_query', 'dbt')(select_sql)) }}
 {% endmacro %}
 
 {% macro default__get_columns_in_query(select_sql) %}
@@ -15,7 +15,7 @@
 {% endmacro %}
 
 {% macro create_schema(relation) -%}
-  {{ adapter.dispatch('create_schema')(relation) }}
+  {{ adapter.dispatch('create_schema', 'dbt')(relation) }}
 {% endmacro %}
 
 {% macro default__create_schema(relation) -%}
@@ -25,7 +25,7 @@
 {% endmacro %}
 
 {% macro drop_schema(relation) -%}
-  {{ adapter.dispatch('drop_schema')(relation) }}
+  {{ adapter.dispatch('drop_schema', 'dbt')(relation) }}
 {% endmacro %}
 
 {% macro default__drop_schema(relation) -%}
@@ -35,7 +35,7 @@
 {% endmacro %}
 
 {% macro create_table_as(temporary, relation, sql) -%}
-  {{ adapter.dispatch('create_table_as')(temporary, relation, sql) }}
+  {{ adapter.dispatch('create_table_as', 'dbt')(temporary, relation, sql) }}
 {%- endmacro %}
 
 {% macro default__create_table_as(temporary, relation, sql) -%}
@@ -52,7 +52,7 @@
 {% endmacro %}
 
 {% macro get_create_index_sql(relation, index_dict) -%}
-  {{ return(adapter.dispatch('get_create_index_sql')(relation, index_dict)) }}
+  {{ return(adapter.dispatch('get_create_index_sql', 'dbt')(relation, index_dict)) }}
 {% endmacro %}
 
 {% macro default__get_create_index_sql(relation, index_dict) -%}
@@ -60,7 +60,7 @@
 {% endmacro %}
 
 {% macro create_indexes(relation) -%}
-  {{ adapter.dispatch('create_indexes')(relation) }}
+  {{ adapter.dispatch('create_indexes', 'dbt')(relation) }}
 {%- endmacro %}
 
 {% macro default__create_indexes(relation) -%}
@@ -75,7 +75,7 @@
 {% endmacro %}
 
 {% macro create_view_as(relation, sql) -%}
-  {{ adapter.dispatch('create_view_as')(relation, sql) }}
+  {{ adapter.dispatch('create_view_as', 'dbt')(relation, sql) }}
 {%- endmacro %}
 
 {% macro default__create_view_as(relation, sql) -%}
@@ -89,7 +89,7 @@
 
 
 {% macro get_catalog(information_schema, schemas) -%}
-  {{ return(adapter.dispatch('get_catalog')(information_schema, schemas)) }}
+  {{ return(adapter.dispatch('get_catalog', 'dbt')(information_schema, schemas)) }}
 {%- endmacro %}
 
 {% macro default__get_catalog(information_schema, schemas) -%}
@@ -104,7 +104,7 @@
 
 
 {% macro get_columns_in_relation(relation) -%}
-  {{ return(adapter.dispatch('get_columns_in_relation')(relation)) }}
+  {{ return(adapter.dispatch('get_columns_in_relation', 'dbt')(relation)) }}
 {% endmacro %}
 
 {% macro sql_convert_columns_in_relation(table) -%}
@@ -121,13 +121,13 @@
 {% endmacro %}
 
 {% macro alter_column_type(relation, column_name, new_column_type) -%}
-  {{ return(adapter.dispatch('alter_column_type')(relation, column_name, new_column_type)) }}
+  {{ return(adapter.dispatch('alter_column_type', 'dbt')(relation, column_name, new_column_type)) }}
 {% endmacro %}
 
 
 
 {% macro alter_column_comment(relation, column_dict) -%}
-  {{ return(adapter.dispatch('alter_column_comment')(relation, column_dict)) }}
+  {{ return(adapter.dispatch('alter_column_comment', 'dbt')(relation, column_dict)) }}
 {% endmacro %}
 
 {% macro default__alter_column_comment(relation, column_dict) -%}
@@ -136,7 +136,7 @@
 {% endmacro %}
 
 {% macro alter_relation_comment(relation, relation_comment) -%}
-  {{ return(adapter.dispatch('alter_relation_comment')(relation, relation_comment)) }}
+  {{ return(adapter.dispatch('alter_relation_comment', 'dbt')(relation, relation_comment)) }}
 {% endmacro %}
 
 {% macro default__alter_relation_comment(relation, relation_comment) -%}
@@ -145,7 +145,7 @@
 {% endmacro %}
 
 {% macro persist_docs(relation, model, for_relation=true, for_columns=true) -%}
-  {{ return(adapter.dispatch('persist_docs')(relation, model, for_relation, for_columns)) }}
+  {{ return(adapter.dispatch('persist_docs', 'dbt')(relation, model, for_relation, for_columns)) }}
 {% endmacro %}
 
 {% macro default__persist_docs(relation, model, for_relation, for_columns) -%}
@@ -180,7 +180,7 @@
 
 
 {% macro drop_relation(relation) -%}
-  {{ return(adapter.dispatch('drop_relation')(relation)) }}
+  {{ return(adapter.dispatch('drop_relation', 'dbt')(relation)) }}
 {% endmacro %}
 
 
@@ -191,7 +191,7 @@
 {% endmacro %}
 
 {% macro truncate_relation(relation) -%}
-  {{ return(adapter.dispatch('truncate_relation')(relation)) }}
+  {{ return(adapter.dispatch('truncate_relation', 'dbt')(relation)) }}
 {% endmacro %}
 
 
@@ -202,7 +202,7 @@
 {% endmacro %}
 
 {% macro rename_relation(from_relation, to_relation) -%}
-  {{ return(adapter.dispatch('rename_relation')(from_relation, to_relation)) }}
+  {{ return(adapter.dispatch('rename_relation', 'dbt')(from_relation, to_relation)) }}
 {% endmacro %}
 
 {% macro default__rename_relation(from_relation, to_relation) -%}
@@ -214,7 +214,7 @@
 
 
 {% macro information_schema_name(database) %}
-  {{ return(adapter.dispatch('information_schema_name')(database)) }}
+  {{ return(adapter.dispatch('information_schema_name', 'dbt')(database)) }}
 {% endmacro %}
 
 {% macro default__information_schema_name(database) -%}
@@ -227,7 +227,7 @@
 
 
 {% macro list_schemas(database) -%}
-  {{ return(adapter.dispatch('list_schemas')(database)) }}
+  {{ return(adapter.dispatch('list_schemas', 'dbt')(database)) }}
 {% endmacro %}
 
 {% macro default__list_schemas(database) -%}
@@ -241,7 +241,7 @@
 
 
 {% macro check_schema_exists(information_schema, schema) -%}
-  {{ return(adapter.dispatch('check_schema_exists')(information_schema, schema)) }}
+  {{ return(adapter.dispatch('check_schema_exists', 'dbt')(information_schema, schema)) }}
 {% endmacro %}
 
 {% macro default__check_schema_exists(information_schema, schema) -%}
@@ -256,7 +256,7 @@
 
 
 {% macro list_relations_without_caching(schema_relation) %}
-  {{ return(adapter.dispatch('list_relations_without_caching')(schema_relation)) }}
+  {{ return(adapter.dispatch('list_relations_without_caching', 'dbt')(schema_relation)) }}
 {% endmacro %}
 
 
@@ -267,7 +267,7 @@
 
 
 {% macro current_timestamp() -%}
-  {{ adapter.dispatch('current_timestamp')() }}
+  {{ adapter.dispatch('current_timestamp', 'dbt')() }}
 {%- endmacro %}
 
 
@@ -278,7 +278,7 @@
 
 
 {% macro collect_freshness(source, loaded_at_field, filter) %}
-  {{ return(adapter.dispatch('collect_freshness')(source, loaded_at_field, filter))}}
+  {{ return(adapter.dispatch('collect_freshness', 'dbt')(source, loaded_at_field, filter))}}
 {% endmacro %}
 
 
@@ -296,7 +296,7 @@
 {% endmacro %}
 
 {% macro make_temp_relation(base_relation, suffix='__dbt_tmp') %}
-  {{ return(adapter.dispatch('make_temp_relation')(base_relation, suffix))}}
+  {{ return(adapter.dispatch('make_temp_relation', 'dbt')(base_relation, suffix))}}
 {% endmacro %}
 
 {% macro default__make_temp_relation(base_relation, suffix) %}
@@ -311,3 +311,34 @@
   {{ config.set('sql_header', caller()) }}
 {%- endmacro %}
 
+
+{% macro alter_relation_add_remove_columns(relation, add_columns = none, remove_columns = none) -%}
+  {{ return(adapter.dispatch('alter_relation_add_remove_columns', 'dbt')(relation, add_columns, remove_columns)) }}
+{% endmacro %}
+
+{% macro default__alter_relation_add_remove_columns(relation, add_columns, remove_columns) %}
+  
+  {% if add_columns is none %}
+    {% set add_columns = [] %}
+  {% endif %}
+  {% if remove_columns is none %}
+    {% set remove_columns = [] %}
+  {% endif %}
+  
+  {% set sql -%}
+     
+     alter {{ relation.type }} {{ relation }}
+       
+            {% for column in add_columns %}
+               add column {{ column.name }} {{ column.data_type }}{{ ',' if not loop.last }}
+            {% endfor %}{{ ',' if remove_columns | length > 0 }}
+            
+            {% for column in remove_columns %}
+                drop column {{ column.name }}{{ ',' if not loop.last }}
+            {% endfor %}
+  
+  {%- endset -%}
+
+  {% do run_query(sql) %}
+
+{% endmacro %}

core/dbt/include/global_project/macros/etc/get_custom_alias.sql

@@ -13,6 +13,10 @@
 
 #}
 {% macro generate_alias_name(custom_alias_name=none, node=none) -%}
+    {% do return(adapter.dispatch('generate_alias_name', 'dbt')(custom_alias_name, node)) %}
+{%- endmacro %}
+
+{% macro default__generate_alias_name(custom_alias_name=none, node=none) -%}
 
     {%- if custom_alias_name is none -%}
 

core/dbt/include/global_project/macros/etc/get_custom_database.sql

@@ -14,7 +14,7 @@
 
 #}
 {% macro generate_database_name(custom_database_name=none, node=none) -%}
-    {% do return(adapter.dispatch('generate_database_name')(custom_database_name, node)) %}
+    {% do return(adapter.dispatch('generate_database_name', 'dbt')(custom_database_name, node)) %}
 {%- endmacro %}
 
 {% macro default__generate_database_name(custom_database_name=none, node=none) -%}

core/dbt/include/global_project/macros/etc/get_custom_schema.sql

@@ -15,6 +15,10 @@
 
 #}
 {% macro generate_schema_name(custom_schema_name, node) -%}
+    {{ return(adapter.dispatch('generate_schema_name', 'dbt')(custom_schema_name, node)) }}
+{% endmacro %}
+
+{% macro default__generate_schema_name(custom_schema_name, node) -%}
 
     {%- set default_schema = target.schema -%}
     {%- if custom_schema_name is none -%}

core/dbt/include/global_project/macros/etc/where_subquery.sql

@@ -0,0 +1,15 @@
+{% macro get_where_subquery(relation) -%}
+    {% do return(adapter.dispatch('get_where_subquery')(relation)) %}
+{%- endmacro %}
+
+{% macro default__get_where_subquery(relation) -%}
+    {% set where = config.get('where', '') %}
+    {% if where %}
+        {%- set filtered -%}
+            (select * from {{ relation }} where {{ where }}) dbt_subquery
+        {%- endset -%}
+        {% do return(filtered) %}
+    {%- else -%}
+        {% do return(relation) %}
+    {%- endif -%}
+{%- endmacro %}

core/dbt/include/global_project/macros/materializations/common/merge.sql

@@ -1,17 +1,17 @@
 
 
 {% macro get_merge_sql(target, source, unique_key, dest_columns, predicates=none) -%}
-  {{ adapter.dispatch('get_merge_sql')(target, source, unique_key, dest_columns, predicates) }}
+  {{ adapter.dispatch('get_merge_sql', 'dbt')(target, source, unique_key, dest_columns, predicates) }}
 {%- endmacro %}
 
 
 {% macro get_delete_insert_merge_sql(target, source, unique_key, dest_columns) -%}
-  {{ adapter.dispatch('get_delete_insert_merge_sql')(target, source, unique_key, dest_columns) }}
+  {{ adapter.dispatch('get_delete_insert_merge_sql', 'dbt')(target, source, unique_key, dest_columns) }}
 {%- endmacro %}
 
 
 {% macro get_insert_overwrite_merge_sql(target, source, dest_columns, predicates, include_sql_header=false) -%}
-  {{ adapter.dispatch('get_insert_overwrite_merge_sql')(target, source, dest_columns, predicates, include_sql_header) }}
+  {{ adapter.dispatch('get_insert_overwrite_merge_sql', 'dbt')(target, source, dest_columns, predicates, include_sql_header) }}
 {%- endmacro %}
 
 
@@ -79,7 +79,7 @@
     (
         select {{ dest_cols_csv }}
         from {{ source }}
-    );
+    )
 
 {%- endmacro %}
 

core/dbt/include/global_project/macros/materializations/incremental/helpers.sql

@@ -1,5 +1,6 @@
 
 {% macro incremental_upsert(tmp_relation, target_relation, unique_key=none, statement_name="main") %}
+    
     {%- set dest_columns = adapter.get_columns_in_relation(target_relation) -%}
     {%- set dest_cols_csv = dest_columns | map(attribute='quoted') | join(', ') -%}
 

core/dbt/include/global_project/macros/materializations/incremental/incremental.sql

@@ -5,6 +5,26 @@
 
   {% set target_relation = this.incorporate(type='table') %}
   {% set existing_relation = load_relation(this) %}
+  {% set tmp_relation = make_temp_relation(target_relation) %}
+  {%- set full_refresh_mode = (should_full_refresh()) -%}
+
+  {% set on_schema_change = incremental_validate_on_schema_change(config.get('on_schema_change'), default='ignore') %}
+
+  {% set tmp_identifier = model['name'] + '__dbt_tmp' %}
+  {% set backup_identifier = model['name'] + "__dbt_backup" %}
+
+  -- the intermediate_ and backup_ relations should not already exist in the database; get_relation
+  -- will return None in that case. Otherwise, we get a relation that we can drop
+  -- later, before we try to use this name for the current operation. This has to happen before
+  -- BEGIN, in a separate transaction
+  {% set preexisting_intermediate_relation = adapter.get_relation(identifier=tmp_identifier, 
+                                                                  schema=schema,
+                                                                  database=database) %}                                               
+  {% set preexisting_backup_relation = adapter.get_relation(identifier=backup_identifier,
+                                                            schema=schema,
+                                                            database=database) %}
+  {{ drop_relation_if_exists(preexisting_intermediate_relation) }}
+  {{ drop_relation_if_exists(preexisting_backup_relation) }}
 
   {{ run_hooks(pre_hooks, inside_transaction=False) }}
 
@@ -12,29 +32,30 @@
   {{ run_hooks(pre_hooks, inside_transaction=True) }}
 
   {% set to_drop = [] %}
+
+  {# -- first check whether we want to full refresh for source view or config reasons #}
+  {% set trigger_full_refresh = (full_refresh_mode or existing_relation.is_view) %}
+
   {% if existing_relation is none %}
       {% set build_sql = create_table_as(False, target_relation, sql) %}
-  {% elif existing_relation.is_view or should_full_refresh() %}
+{% elif trigger_full_refresh %}
       {#-- Make sure the backup doesn't exist so we don't encounter issues with the rename below #}
       {% set tmp_identifier = model['name'] + '__dbt_tmp' %}
-      {% set backup_identifier = model['name'] + "__dbt_backup" %}
-
+      {% set backup_identifier = model['name'] + '__dbt_backup' %}
       {% set intermediate_relation = existing_relation.incorporate(path={"identifier": tmp_identifier}) %}
       {% set backup_relation = existing_relation.incorporate(path={"identifier": backup_identifier}) %}
 
-      {% do adapter.drop_relation(intermediate_relation) %}
-      {% do adapter.drop_relation(backup_relation) %}
-
       {% set build_sql = create_table_as(False, intermediate_relation, sql) %}
       {% set need_swap = true %}
       {% do to_drop.append(backup_relation) %}
   {% else %}
-      {% set tmp_relation = make_temp_relation(target_relation) %}
-      {% do run_query(create_table_as(True, tmp_relation, sql)) %}
-      {% do adapter.expand_target_column_types(
+    {% do run_query(create_table_as(True, tmp_relation, sql)) %}
+    {% do adapter.expand_target_column_types(
              from_relation=tmp_relation,
              to_relation=target_relation) %}
-      {% set build_sql = incremental_upsert(tmp_relation, target_relation, unique_key=unique_key) %}
+    {% do process_schema_changes(on_schema_change, tmp_relation, existing_relation) %}
+    {% set build_sql = incremental_upsert(tmp_relation, target_relation, unique_key=unique_key) %}
+  
   {% endif %}
 
   {% call statement("main") %}

core/dbt/include/global_project/macros/materializations/incremental/on_schema_change.sql

@@ -0,0 +1,164 @@
+{% macro incremental_validate_on_schema_change(on_schema_change, default='ignore') %}
+   
+   {% if on_schema_change not in ['sync_all_columns', 'append_new_columns', 'fail', 'ignore'] %}
+     
+     {% set log_message = 'Invalid value for on_schema_change (%s) specified. Setting default value of %s.' % (on_schema_change, default) %}
+     {% do log(log_message) %}
+     
+     {{ return(default) }}
+
+   {% else %}
+
+     {{ return(on_schema_change) }}
+   
+   {% endif %}
+
+{% endmacro %}
+
+{% macro diff_columns(source_columns, target_columns) %}
+
+  {% set result = [] %}
+  {% set source_names = source_columns | map(attribute = 'column') | list %}
+  {% set target_names = target_columns | map(attribute = 'column') | list %}
+   
+   {# --check whether the name attribute exists in the target - this does not perform a data type check #}
+   {% for sc in source_columns %}
+     {% if sc.name not in target_names %}
+        {{ result.append(sc) }}
+     {% endif %}
+   {% endfor %}
+  
+  {{ return(result) }}
+
+{% endmacro %}
+
+{% macro diff_column_data_types(source_columns, target_columns) %}
+  
+  {% set result = [] %}
+  {% for sc in source_columns %}
+    {% set tc = target_columns | selectattr("name", "equalto", sc.name) | list | first %}
+    {% if tc %}
+      {% if sc.data_type != tc.data_type %}
+        {{ result.append( { 'column_name': tc.name, 'new_type': sc.data_type } ) }} 
+      {% endif %}
+    {% endif %}
+  {% endfor %}
+
+  {{ return(result) }}
+
+{% endmacro %}
+
+
+{% macro check_for_schema_changes(source_relation, target_relation) %}
+  
+  {% set schema_changed = False %}
+  
+  {%- set source_columns = adapter.get_columns_in_relation(source_relation) -%}
+  {%- set target_columns = adapter.get_columns_in_relation(target_relation) -%}
+  {%- set source_not_in_target = diff_columns(source_columns, target_columns) -%}
+  {%- set target_not_in_source = diff_columns(target_columns, source_columns) -%}
+  
+  {% set new_target_types = diff_column_data_types(source_columns, target_columns) %}
+
+  {% if source_not_in_target != [] %}
+    {% set schema_changed = True %}
+  {% elif target_not_in_source != [] or new_target_types != [] %}
+    {% set schema_changed = True %}
+  {% elif new_target_types != [] %}
+    {% set schema_changed = True %}
+  {% endif %}
+  
+  {% set changes_dict = {
+    'schema_changed': schema_changed,
+    'source_not_in_target': source_not_in_target,
+    'target_not_in_source': target_not_in_source,
+    'new_target_types': new_target_types
+  } %}
+
+  {% set msg %}
+    In {{ target_relation }}:
+        Schema changed: {{ schema_changed }}
+        Source columns not in target: {{ source_not_in_target }}
+        Target columns not in source: {{ target_not_in_source }}
+        New column types: {{ new_target_types }}
+  {% endset %}
+  
+  {% do log(msg) %}
+
+  {{ return(changes_dict) }}
+
+{% endmacro %}
+
+
+{% macro sync_column_schemas(on_schema_change, target_relation, schema_changes_dict) %}
+  
+  {%- set add_to_target_arr = schema_changes_dict['source_not_in_target'] -%}
+
+  {%- if on_schema_change == 'append_new_columns'-%}
+     {%- if add_to_target_arr | length > 0 -%}
+       {%- do alter_relation_add_remove_columns(target_relation, add_to_target_arr, none) -%}
+     {%- endif -%}
+  
+  {% elif on_schema_change == 'sync_all_columns' %}
+     {%- set remove_from_target_arr = schema_changes_dict['target_not_in_source'] -%}
+     {%- set new_target_types = schema_changes_dict['new_target_types'] -%}
+  
+     {% if add_to_target_arr | length > 0 or remove_from_target_arr | length > 0 %} 
+       {%- do alter_relation_add_remove_columns(target_relation, add_to_target_arr, remove_from_target_arr) -%}
+     {% endif %}
+
+     {% if new_target_types != [] %}
+       {% for ntt in new_target_types %}
+         {% set column_name = ntt['column_name'] %}
+         {% set new_type = ntt['new_type'] %}
+         {% do alter_column_type(target_relation, column_name, new_type) %}
+       {% endfor %}
+     {% endif %}
+  
+  {% endif %}
+
+  {% set schema_change_message %}
+    In {{ target_relation }}:
+        Schema change approach: {{ on_schema_change }}
+        Columns added: {{ add_to_target_arr }}
+        Columns removed: {{ remove_from_target_arr }}
+        Data types changed: {{ new_target_types }}
+  {% endset %}
+  
+  {% do log(schema_change_message) %}
+  
+{% endmacro %}
+
+
+{% macro process_schema_changes(on_schema_change, source_relation, target_relation) %}
+    
+    {% if on_schema_change != 'ignore' %}
+    
+      {% set schema_changes_dict = check_for_schema_changes(source_relation, target_relation) %}
+      
+      {% if schema_changes_dict['schema_changed'] %}
+    
+        {% if on_schema_change == 'fail' %}
+        
+          {% set fail_msg %}
+              The source and target schemas on this incremental model are out of sync!
+              They can be reconciled in several ways: 
+                - set the `on_schema_change` config to either append_new_columns or sync_all_columns, depending on your situation.
+                - Re-run the incremental model with `full_refresh: True` to update the target schema.
+                - update the schema manually and re-run the process.
+          {% endset %}
+          
+          {% do exceptions.raise_compiler_error(fail_msg) %}
+        
+        {# -- unless we ignore, run the sync operation per the config #}
+        {% else %}
+          
+          {% do sync_column_schemas(on_schema_change, target_relation, schema_changes_dict) %}
+        
+        {% endif %}
+      
+      {% endif %}
+    
+    {% endif %}
+
+{% endmacro %}

core/dbt/include/global_project/macros/materializations/seed/seed.sql

@@ -1,14 +1,6 @@
 
 {% macro create_csv_table(model, agate_table) -%}
-  {{ adapter.dispatch('create_csv_table')(model, agate_table) }}
-{%- endmacro %}
-
-{% macro reset_csv_table(model, full_refresh, old_relation, agate_table) -%}
-  {{ adapter.dispatch('reset_csv_table')(model, full_refresh, old_relation, agate_table) }}
-{%- endmacro %}
-
-{% macro load_csv_rows(model, agate_table) -%}
-  {{ adapter.dispatch('load_csv_rows')(model, agate_table) }}
+  {{ adapter.dispatch('create_csv_table', 'dbt')(model, agate_table) }}
 {%- endmacro %}
 
 {% macro default__create_csv_table(model, agate_table) %}
@@ -33,6 +25,9 @@
   {{ return(sql) }}
 {% endmacro %}
 
+{% macro reset_csv_table(model, full_refresh, old_relation, agate_table) -%}
+  {{ adapter.dispatch('reset_csv_table', 'dbt')(model, full_refresh, old_relation, agate_table) }}
+{%- endmacro %}
 
 {% macro default__reset_csv_table(model, full_refresh, old_relation, agate_table) %}
     {% set sql = "" %}
@@ -47,6 +42,21 @@
     {{ return(sql) }}
 {% endmacro %}
 
+{% macro get_binding_char() -%}
+  {{ adapter.dispatch('get_binding_char', 'dbt')() }}
+{%- endmacro %}
+
+{% macro default__get_binding_char() %}
+  {{ return('%s') }}
+{% endmacro %}
+
+{% macro get_batch_size() -%}
+  {{ adapter.dispatch('get_batch_size', 'dbt')() }}
+{%- endmacro %}
+
+{% macro default__get_batch_size() %}
+  {{ return(10000) }}
+{% endmacro %}
 
 {% macro get_seed_column_quoted_csv(model, column_names) %}
   {%- set quote_seed_column = model['config'].get('quote_columns', None) -%}
@@ -59,47 +69,47 @@
     {{ return(dest_cols_csv) }}
 {% endmacro %}
 
-
-{% macro basic_load_csv_rows(model, batch_size, agate_table) %}
-    {% set cols_sql = get_seed_column_quoted_csv(model, agate_table.column_names) %}
-    {% set bindings = [] %}
-
-    {% set statements = [] %}
-
-    {% for chunk in agate_table.rows | batch(batch_size) %}
-        {% set bindings = [] %}
-
-        {% for row in chunk %}
-            {% do bindings.extend(row) %}
-        {% endfor %}
-
-        {% set sql %}
-            insert into {{ this.render() }} ({{ cols_sql }}) values
-            {% for row in chunk -%}
-                ({%- for column in agate_table.column_names -%}
-                    %s
-                    {%- if not loop.last%},{%- endif %}
-                {%- endfor -%})
-                {%- if not loop.last%},{%- endif %}
-            {%- endfor %}
-        {% endset %}
-
-        {% do adapter.add_query(sql, bindings=bindings, abridge_sql_log=True) %}
-
-        {% if loop.index0 == 0 %}
-            {% do statements.append(sql) %}
-        {% endif %}
-    {% endfor %}
-
-    {# Return SQL so we can render it out into the compiled files #}
-    {{ return(statements[0]) }}
-{% endmacro %}
-
+{% macro load_csv_rows(model, agate_table) -%}
+  {{ adapter.dispatch('load_csv_rows', 'dbt')(model, agate_table) }}
+{%- endmacro %}
 
 {% macro default__load_csv_rows(model, agate_table) %}
-  {{ return(basic_load_csv_rows(model, 10000, agate_table) )}}
-{% endmacro %}
 
+  {% set batch_size = get_batch_size() %}
+
+  {% set cols_sql = get_seed_column_quoted_csv(model, agate_table.column_names) %}
+  {% set bindings = [] %}
+
+  {% set statements = [] %}
+
+  {% for chunk in agate_table.rows | batch(batch_size) %}
+      {% set bindings = [] %}
+
+      {% for row in chunk %}
+          {% do bindings.extend(row) %}
+      {% endfor %}
+
+      {% set sql %}
+          insert into {{ this.render() }} ({{ cols_sql }}) values
+          {% for row in chunk -%}
+              ({%- for column in agate_table.column_names -%}
+                  {{ get_binding_char() }}
+                  {%- if not loop.last%},{%- endif %}
+              {%- endfor -%})
+              {%- if not loop.last%},{%- endif %}
+          {%- endfor %}
+      {% endset %}
+
+      {% do adapter.add_query(sql, bindings=bindings, abridge_sql_log=True) %}
+
+      {% if loop.index0 == 0 %}
+          {% do statements.append(sql) %}
+      {% endif %}
+  {% endfor %}
+
+  {# Return SQL so we can render it out into the compiled files #}
+  {{ return(statements[0]) }}
+{% endmacro %}
 
 {% materialization seed, default %}
 

core/dbt/include/global_project/macros/materializations/snapshot/snapshot.sql

@@ -2,7 +2,7 @@
     Add new columns to the table if applicable
 #}
 {% macro create_columns(relation, columns) %}
-  {{ adapter.dispatch('create_columns')(relation, columns) }}
+  {{ adapter.dispatch('create_columns', 'dbt')(relation, columns) }}
 {% endmacro %}
 
 {% macro default__create_columns(relation, columns) %}
@@ -15,7 +15,7 @@
 
 
 {% macro post_snapshot(staging_relation) %}
-  {{ adapter.dispatch('post_snapshot')(staging_relation) }}
+  {{ adapter.dispatch('post_snapshot', 'dbt')(staging_relation) }}
 {% endmacro %}
 
 {% macro default__post_snapshot(staging_relation) %}

core/dbt/include/global_project/macros/materializations/snapshot/snapshot_merge.sql

@@ -1,6 +1,6 @@
 
 {% macro snapshot_merge_sql(target, source, insert_cols) -%}
-  {{ adapter.dispatch('snapshot_merge_sql')(target, source, insert_cols) }}
+  {{ adapter.dispatch('snapshot_merge_sql', 'dbt')(target, source, insert_cols) }}
 {%- endmacro %}
 
 
@@ -21,7 +21,6 @@
      and DBT_INTERNAL_SOURCE.dbt_change_type = 'insert'
         then insert ({{ insert_cols_csv }})
         values ({{ insert_cols_csv }})
-    ;
+
 {% endmacro %}
 
-

core/dbt/include/global_project/macros/materializations/snapshot/strategies.sql

@@ -36,7 +36,7 @@
     Create SCD Hash SQL fields cross-db
 #}
 {% macro snapshot_hash_arguments(args) -%}
-  {{ adapter.dispatch('snapshot_hash_arguments')(args) }}
+  {{ adapter.dispatch('snapshot_hash_arguments', 'dbt')(args) }}
 {%- endmacro %}
 
 
@@ -52,7 +52,7 @@
     Get the current time cross-db
 #}
 {% macro snapshot_get_time() -%}
-  {{ adapter.dispatch('snapshot_get_time')() }}
+  {{ adapter.dispatch('snapshot_get_time', 'dbt')() }}
 {%- endmacro %}
 
 {% macro default__snapshot_get_time() -%}
@@ -75,7 +75,7 @@
         table instead of assuming that the user-supplied {{ updated_at }}
         will be present in the historical data.
 
-        See https://github.com/fishtown-analytics/dbt/issues/2350
+        See https://github.com/dbt-labs/dbt/issues/2350
     */ #}
     {% set row_changed_expr -%}
         ({{ snapshotted_rel }}.dbt_valid_from < {{ current_rel }}.{{ updated_at }})
@@ -94,7 +94,7 @@
 
 
 {% macro snapshot_string_as_time(timestamp) -%}
-    {{ adapter.dispatch('snapshot_string_as_time')(timestamp) }}
+    {{ adapter.dispatch('snapshot_string_as_time', 'dbt')(timestamp) }}
 {%- endmacro %}
 
 

core/dbt/include/global_project/macros/materializations/table/table.sql

@@ -12,7 +12,12 @@
                                                       schema=schema,
                                                       database=database,
                                                       type='table') -%}
-
+  -- the intermediate_relation should not already exist in the database; get_relation
+  -- will return None in that case. Otherwise, we get a relation that we can drop
+  -- later, before we try to use this name for the current operation
+  {%- set preexisting_intermediate_relation = adapter.get_relation(identifier=tmp_identifier, 
+                                                                   schema=schema,
+                                                                   database=database) -%}
   /*
       See ../view/view.sql for more information about this relation.
   */
@@ -21,14 +26,15 @@
                                                 schema=schema,
                                                 database=database,
                                                 type=backup_relation_type) -%}
-
-  {%- set exists_as_table = (old_relation is not none and old_relation.is_table) -%}
-  {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%}
+  -- as above, the backup_relation should not already exist
+  {%- set preexisting_backup_relation = adapter.get_relation(identifier=backup_identifier,
+                                                             schema=schema,
+                                                             database=database) -%}
 
 
-  -- drop the temp relations if they exists for some reason
-  {{ adapter.drop_relation(intermediate_relation) }}
-  {{ adapter.drop_relation(backup_relation) }}
+  -- drop the temp relations if they exist already in the database
+  {{ drop_relation_if_exists(preexisting_intermediate_relation) }}
+  {{ drop_relation_if_exists(preexisting_backup_relation) }}
 
   {{ run_hooks(pre_hooks, inside_transaction=False) }}
 
@@ -42,7 +48,7 @@
 
   -- cleanup
   {% if old_relation is not none %}
-      {{ adapter.rename_relation(target_relation, backup_relation) }}
+      {{ adapter.rename_relation(old_relation, backup_relation) }}
   {% endif %}
 
   {{ adapter.rename_relation(intermediate_relation, target_relation) }}

core/dbt/include/global_project/macros/materializations/test.sql

@@ -1,3 +1,19 @@
+{% macro get_test_sql(main_sql, fail_calc, warn_if, error_if, limit) -%}
+  {{ adapter.dispatch('get_test_sql', 'dbt')(main_sql, fail_calc, warn_if, error_if, limit) }}
+{%- endmacro %}
+
+
+{% macro default__get_test_sql(main_sql, fail_calc, warn_if, error_if, limit) -%}
+    select
+      {{ fail_calc }} as failures,
+      {{ fail_calc }} {{ warn_if }} as should_warn,
+      {{ fail_calc }} {{ error_if }} as should_error
+    from (
+      {{ main_sql }}
+      {{ "limit " ~ limit if limit != none }}
+    ) dbt_internal_test
+{%- endmacro %}
+
 {%- materialization test, default -%}
 
   {% set relations = [] %}
@@ -39,14 +55,7 @@
 
   {% call statement('main', fetch_result=True) -%}
 
-    select
-      {{ fail_calc }} as failures,
-      {{ fail_calc }} {{ warn_if }} as should_warn,
-      {{ fail_calc }} {{ error_if }} as should_error
-    from (
-      {{ main_sql }}
-      {{ "limit " ~ limit if limit != none }}
-    ) dbt_internal_test
+    {{ get_test_sql(main_sql, fail_calc, warn_if, error_if, limit)}}
 
   {%- endcall %}
   

core/dbt/include/global_project/macros/materializations/view/create_or_replace_view.sql

@@ -1,9 +1,10 @@
 
 {% macro handle_existing_table(full_refresh, old_relation) %}
-    {{ adapter.dispatch('handle_existing_table', macro_namespace = 'dbt')(full_refresh, old_relation) }}
+    {{ adapter.dispatch('handle_existing_table', 'dbt')(full_refresh, old_relation) }}
 {% endmacro %}
 
 {% macro default__handle_existing_table(full_refresh, old_relation) %}
+    {{ log("Dropping relation " ~ old_relation ~ " because it is of type " ~ old_relation.type) }}
     {{ adapter.drop_relation(old_relation) }}
 {% endmacro %}
 
@@ -19,7 +20,7 @@
     */
 #}
 
-{% macro create_or_replace_view(run_outside_transaction_hooks=True) %}
+{% macro create_or_replace_view() %}
   {%- set identifier = model['alias'] -%}
 
   {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}
@@ -30,13 +31,7 @@
       identifier=identifier, schema=schema, database=database,
       type='view') -%}
 
-  {% if run_outside_transaction_hooks %}
-      -- no transactions on BigQuery
-      {{ run_hooks(pre_hooks, inside_transaction=False) }}
-  {% endif %}
-
-  -- `BEGIN` happens here on Snowflake
-  {{ run_hooks(pre_hooks, inside_transaction=True) }}
+  {{ run_hooks(pre_hooks) }}
 
   -- If there's a table with the same name and we weren't told to full refresh,
   -- that's an error. If we were told to full refresh, drop it. This behavior differs
@@ -50,14 +45,7 @@
     {{ create_view_as(target_relation, sql) }}
   {%- endcall %}
 
-  {{ run_hooks(post_hooks, inside_transaction=True) }}
-
-  {{ adapter.commit() }}
-
-  {% if run_outside_transaction_hooks %}
-      -- No transactions on BigQuery
-      {{ run_hooks(post_hooks, inside_transaction=False) }}
-  {% endif %}
+  {{ run_hooks(post_hooks) }}
 
   {{ return({'relations': [target_relation]}) }}
 

core/dbt/include/global_project/macros/materializations/view/view.sql

@@ -9,7 +9,12 @@
                                                 type='view') -%}
   {%- set intermediate_relation = api.Relation.create(identifier=tmp_identifier,
                                                       schema=schema, database=database, type='view') -%}
-
+  -- the intermediate_relation should not already exist in the database; get_relation
+  -- will return None in that case. Otherwise, we get a relation that we can drop
+  -- later, before we try to use this name for the current operation
+  {%- set preexisting_intermediate_relation = adapter.get_relation(identifier=tmp_identifier, 
+                                                                   schema=schema,
+                                                                   database=database) -%}
   /*
      This relation (probably) doesn't exist yet. If it does exist, it's a leftover from
      a previous run, and we're going to try to drop it immediately. At the end of this
@@ -27,14 +32,16 @@
   {%- set backup_relation = api.Relation.create(identifier=backup_identifier,
                                                 schema=schema, database=database,
                                                 type=backup_relation_type) -%}
-
-  {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%}
+  -- as above, the backup_relation should not already exist
+  {%- set preexisting_backup_relation = adapter.get_relation(identifier=backup_identifier,
+                                                             schema=schema,
+                                                             database=database) -%}
 
   {{ run_hooks(pre_hooks, inside_transaction=False) }}
 
-  -- drop the temp relations if they exists for some reason
-  {{ adapter.drop_relation(intermediate_relation) }}
-  {{ adapter.drop_relation(backup_relation) }}
+  -- drop the temp relations if they exist already in the database
+  {{ drop_relation_if_exists(preexisting_intermediate_relation) }}
+  {{ drop_relation_if_exists(preexisting_backup_relation) }}
 
   -- `BEGIN` happens here:
   {{ run_hooks(pre_hooks, inside_transaction=True) }}
@@ -47,7 +54,7 @@
   -- cleanup
   -- move the existing view out of the way
   {% if old_relation is not none %}
-    {{ adapter.rename_relation(target_relation, backup_relation) }}
+    {{ adapter.rename_relation(old_relation, backup_relation) }}
   {% endif %}
   {{ adapter.rename_relation(intermediate_relation, target_relation) }}
 

core/dbt/include/global_project/macros/schema_tests/accepted_values.sql

@@ -28,6 +28,6 @@ where value_field not in (
 
 
 {% test accepted_values(model, column_name, values, quote=True) %}
-    {% set macro = adapter.dispatch('test_accepted_values') %}
+    {% set macro = adapter.dispatch('test_accepted_values', 'dbt') %}
     {{ macro(model, column_name, values, quote) }}
 {% endtest %}

core/dbt/include/global_project/macros/schema_tests/not_null.sql

@@ -8,6 +8,6 @@ where {{ column_name }} is null
 
 
 {% test not_null(model, column_name) %}
-    {% set macro = adapter.dispatch('test_not_null') %}
+    {% set macro = adapter.dispatch('test_not_null', 'dbt') %}
     {{ macro(model, column_name) }}
 {% endtest %}

core/dbt/include/global_project/macros/schema_tests/relationships.sql

@@ -1,21 +1,30 @@
 
 {% macro default__test_relationships(model, column_name, to, field) %}
 
+with child as (
+    select {{ column_name }} as from_field
+    from {{ model }}
+    where {{ column_name }} is not null
+),
+
+parent as (
+    select {{ field }} as to_field
+    from {{ to }}
+)
+
 select
-    child.{{ column_name }}
+    from_field
 
-from {{ model }} as child
+from child
+left join parent
+    on child.from_field = parent.to_field
 
-left join {{ to }} as parent
-    on child.{{ column_name }} = parent.{{ field }}
-
-where child.{{ column_name }} is not null
-  and parent.{{ field }} is null
+where parent.to_field is null
 
 {% endmacro %}
 
 
 {% test relationships(model, column_name, to, field) %}
-    {% set macro = adapter.dispatch('test_relationships') %}
+    {% set macro = adapter.dispatch('test_relationships', 'dbt') %}
     {{ macro(model, column_name, to, field) }}
 {% endtest %}

core/dbt/include/global_project/macros/schema_tests/unique.sql

@@ -1,7 +1,7 @@
 {% macro default__test_unique(model, column_name) %}
 
 select
-    {{ column_name }},
+    {{ column_name }} as unique_field,
     count(*) as n_records
 
 from {{ model }}
@@ -13,6 +13,6 @@ having count(*) > 1
 
 
 {% test unique(model, column_name) %}
-    {% set macro = adapter.dispatch('test_unique') %}
+    {% set macro = adapter.dispatch('test_unique', 'dbt') %}
     {{ macro(model, column_name) }}
 {% endtest %}

core/dbt/include/starter_project/.gitignore

@@ -0,0 +1,4 @@
+
+target/
+dbt_modules/
+logs/

core/dbt/include/starter_project/README.md

@@ -0,0 +1,15 @@
+Welcome to your new dbt project!
+
+### Using the starter project
+
+Try running the following commands:
+- dbt run
+- dbt test
+
+
+### Resources:
+- Learn more about dbt [in the docs](https://docs.getdbt.com/docs/introduction)
+- Check out [Discourse](https://discourse.getdbt.com/) for commonly asked questions and answers
+- Join the [chat](https://community.getdbt.com/) on Slack for live discussions and support
+- Find [dbt events](https://events.getdbt.com) near you
+- Check out [the blog](https://blog.getdbt.com/) for the latest news on dbt's development and best practices

core/dbt/include/starter_project/__init__.py

@@ -0,0 +1,3 @@
+import os
+
+PACKAGE_PATH = os.path.dirname(__file__)

core/dbt/include/starter_project/dbt_project.yml

@@ -0,0 +1,38 @@
+
+# Name your project! Project names should contain only lowercase characters
+# and underscores. A good package name should reflect your organization's
+# name or the intended use of these models
+name: 'my_new_project'
+version: '1.0.0'
+config-version: 2
+
+# This setting configures which "profile" dbt uses for this project.
+profile: 'default'
+
+# These configurations specify where dbt should look for different types of files.
+# The `source-paths` config, for example, states that models in this project can be
+# found in the "models/" directory. You probably won't need to change these!
+source-paths: ["models"]
+analysis-paths: ["analysis"]
+test-paths: ["tests"]
+data-paths: ["data"]
+macro-paths: ["macros"]
+snapshot-paths: ["snapshots"]
+
+target-path: "target"  # directory which will store compiled SQL files
+clean-targets:         # directories to be removed by `dbt clean`
+  - "target"
+  - "dbt_modules"
+
+
+# Configuring models
+# Full documentation: https://docs.getdbt.com/docs/configuring-models
+
+# In this example config, we tell dbt to build all models in the example/ directory
+# as tables. These settings can be overridden in the individual model files
+# using the `{{ config(...) }}` macro.
+models:
+  my_new_project:
+    # Config indicated by + and applies to all files under models/example/
+    example:
+      +materialized: view

core/dbt/include/starter_project/models/example/my_first_dbt_model.sql

@@ -0,0 +1,27 @@
+
+/*
+    Welcome to your first dbt model!
+    Did you know that you can also configure models directly within SQL files?
+    This will override configurations stated in dbt_project.yml
+
+    Try changing "table" to "view" below
+*/
+
+{{ config(materialized='table') }}
+
+with source_data as (
+
+    select 1 as id
+    union all
+    select null as id
+
+)
+
+select *
+from source_data
+
+/*
+    Uncomment the line below to remove records with null `id` values
+*/
+
+-- where id is not null

core/dbt/include/starter_project/models/example/my_second_dbt_model.sql

@@ -0,0 +1,6 @@
+
+-- Use the `ref` function to select from other models
+
+select *
+from {{ ref('my_first_dbt_model') }}
+where id = 1

core/dbt/include/starter_project/models/example/schema.yml

@@ -0,0 +1,21 @@
+
+version: 2
+
+models:
+  - name: my_first_dbt_model
+    description: "A starter dbt model"
+    columns:
+      - name: id
+        description: "The primary key for this table"
+        tests:
+          - unique
+          - not_null
+
+  - name: my_second_dbt_model
+    description: "A starter dbt model"
+    columns:
+      - name: id
+        description: "The primary key for this table"
+        tests:
+          - unique
+          - not_null

core/dbt/logger.py

@@ -43,6 +43,15 @@ DEBUG_LOG_FORMAT = (
     '{record.message}'
 )
 
+SECRET_ENV_PREFIX = 'DBT_ENV_SECRET_'
+
+
+def get_secret_env() -> List[str]:
+    return [
+        v for k, v in os.environ.items()
+        if k.startswith(SECRET_ENV_PREFIX)
+    ]
+
 
 ExceptionInformation = str
 
@@ -333,6 +342,12 @@ class TimestampNamed(logbook.Processor):
         record.extra[self.name] = datetime.utcnow().isoformat()
 
 
+class ScrubSecrets(logbook.Processor):
+    def process(self, record):
+        for secret in get_secret_env():
+            record.message = record.message.replace(secret, "*****")
+
+
 logger = logbook.Logger('dbt')
 # provide this for the cache, disabled by default
 CACHE_LOGGER = logbook.Logger('dbt.cache')
@@ -473,7 +488,8 @@ class LogManager(logbook.NestedSetup):
         self._file_handler = DelayedFileHandler()
         self._relevel_processor = Relevel(allowed=['dbt', 'werkzeug'])
         self._state_processor = DbtProcessState('internal')
-        # keep track of wheter we've already entered to decide if we should
+        self._scrub_processor = ScrubSecrets()
+        # keep track of whether we've already entered to decide if we should
         # be actually pushing. This allows us to log in main() and also
         # support entering dbt execution via handle_and_check.
         self._stack_depth = 0
@@ -483,6 +499,7 @@ class LogManager(logbook.NestedSetup):
             self._file_handler,
             self._relevel_processor,
             self._state_processor,
+            self._scrub_processor
         ])
 
     def push_application(self):

core/dbt/main.py

@@ -11,6 +11,7 @@ from pathlib import Path
 import dbt.version
 import dbt.flags as flags
 import dbt.task.run as run_task
+import dbt.task.build as build_task
 import dbt.task.compile as compile_task
 import dbt.task.debug as debug_task
 import dbt.task.clean as clean_task
@@ -40,6 +41,7 @@ class DBTVersion(argparse.Action):
     """This is very very similar to the builtin argparse._Version action,
     except it just calls dbt.version.get_version_information().
     """
+
     def __init__(self,
                  option_strings,
                  version=None,
@@ -367,7 +369,6 @@ def _build_init_subparser(subparsers, base_subparser):
     )
     sub.add_argument(
         '--adapter',
-        default='redshift',
         type=str,
         help='''
         Write sample profiles.yml for which adapter
@@ -377,6 +378,30 @@ def _build_init_subparser(subparsers, base_subparser):
     return sub
 
 
+def _build_build_subparser(subparsers, base_subparser):
+    sub = subparsers.add_parser(
+        'build',
+        parents=[base_subparser],
+        help='''
+        Run all Seeds, Models, Snapshots, and tests in DAG order
+        '''
+    )
+    sub.set_defaults(
+        cls=build_task.BuildTask,
+        which='build',
+        rpc_method='build'
+    )
+    sub.add_argument(
+        '-x',
+        '--fail-fast',
+        action='store_true',
+        help='''
+        Stop execution upon a first failure.
+        '''
+    )
+    return sub
+
+
 def _build_clean_subparser(subparsers, base_subparser):
     sub = subparsers.add_parser(
         'clean',
@@ -529,39 +554,6 @@ def _build_docs_generate_subparser(subparsers, base_subparser):
     return generate_sub
 
 
-def _add_models_argument(sub, help_override=None, **kwargs):
-    help_str = '''
-        Specify the models to include.
-    '''
-    if help_override is not None:
-        help_str = help_override
-    sub.add_argument(
-        '-m',
-        '--models',
-        dest='models',
-        nargs='+',
-        help=help_str,
-        **kwargs
-    )
-
-
-def _add_select_argument(sub, dest='models', help_override=None, **kwargs):
-    help_str = '''
-        Specify the nodes to include.
-    '''
-    if help_override is not None:
-        help_str = help_override
-
-    sub.add_argument(
-        '-s',
-        '--select',
-        dest=dest,
-        nargs='+',
-        help=help_str,
-        **kwargs
-    )
-
-
 def _add_common_selector_arguments(sub):
     sub.add_argument(
         '--exclude',
@@ -590,17 +582,26 @@ def _add_common_selector_arguments(sub):
     )
 
 
-def _add_selection_arguments(*subparsers, **kwargs):
-    models_name = kwargs.get('models_name', 'models')
+def _add_selection_arguments(*subparsers):
     for sub in subparsers:
-        if models_name == 'models':
-            _add_models_argument(sub)
-        elif models_name == 'select':
-            # these still get stored in 'models', so they present the same
-            # interface to the task
-            _add_select_argument(sub)
-        else:
-            raise InternalException(f'Unknown models style {models_name}')
+        sub.add_argument(
+            '-m',
+            '--models',
+            dest='select',
+            nargs='+',
+            help='''
+                Specify the nodes to include.
+            ''',
+        )
+        sub.add_argument(
+            '-s',
+            '--select',
+            dest='select',
+            nargs='+',
+            help='''
+                Specify the nodes to include.
+            ''',
+        )
         _add_common_selector_arguments(sub)
 
 
@@ -731,23 +732,14 @@ def _build_test_subparser(subparsers, base_subparser):
     return sub
 
 
-def _build_source_snapshot_freshness_subparser(subparsers, base_subparser):
+def _build_source_freshness_subparser(subparsers, base_subparser):
     sub = subparsers.add_parser(
-        'snapshot-freshness',
+        'freshness',
         parents=[base_subparser],
         help='''
         Snapshots the current freshness of the project's sources
         ''',
-    )
-    sub.add_argument(
-        '-s',
-        '--select',
-        required=False,
-        nargs='+',
-        help='''
-        Specify the sources to snapshot freshness
-        ''',
-        dest='selected'
+        aliases=['snapshot-freshness'],
     )
     sub.add_argument(
         '-o',
@@ -768,9 +760,19 @@ def _build_source_snapshot_freshness_subparser(subparsers, base_subparser):
     )
     sub.set_defaults(
         cls=freshness_task.FreshnessTask,
-        which='snapshot-freshness',
-        rpc_method='snapshot-freshness',
+        which='source-freshness',
+        rpc_method='source-freshness',
     )
+    sub.add_argument(
+        '-s',
+        '--select',
+        dest='select',
+        nargs='+',
+        help='''
+            Specify the nodes to include.
+        ''',
+    )
+    _add_common_selector_arguments(sub)
     return sub
 
 
@@ -825,19 +827,28 @@ def _build_list_subparser(subparsers, base_subparser):
     sub.add_argument('--output',
                      choices=['json', 'name', 'path', 'selector'],
                      default='selector')
+    sub.add_argument('--output-keys')
 
-    _add_models_argument(
-        sub,
-        help_override='''
+    sub.add_argument(
+        '-m',
+        '--models',
+        dest='models',
+        nargs='+',
+        help='''
         Specify the models to select and set the resource-type to 'model'.
         Mutually exclusive with '--select' (or '-s') and '--resource-type'
         ''',
         metavar='SELECTOR',
-        required=False
+        required=False,
     )
-    _add_select_argument(
-        sub,
+    sub.add_argument(
+        '-s',
+        '--select',
         dest='select',
+        nargs='+',
+        help='''
+            Specify the nodes to include.
+        ''',
         metavar='SELECTOR',
         required=False,
     )
@@ -983,8 +994,6 @@ def parse_args(args, cls=DBTArgumentParser):
         enable_help='''
         Allow for partial parsing by looking for and writing to a pickle file
         in the target directory. This overrides the user configuration file.
-
-        WARNING: This can result in unexpected behavior if you use env_var()!
         ''',
         disable_help='''
         Disallow partial parsing. This overrides the user configuration file.
@@ -1038,6 +1047,7 @@ def parse_args(args, cls=DBTArgumentParser):
     _build_deps_subparser(subs, base_subparser)
     _build_list_subparser(subs, base_subparser)
 
+    build_sub = _build_build_subparser(subs, base_subparser)
     snapshot_sub = _build_snapshot_subparser(subs, base_subparser)
     rpc_sub = _build_rpc_subparser(subs, base_subparser)
     run_sub = _build_run_subparser(subs, base_subparser)
@@ -1048,18 +1058,18 @@ def parse_args(args, cls=DBTArgumentParser):
     seed_sub = _build_seed_subparser(subs, base_subparser)
     # --threads, --no-version-check
     _add_common_arguments(run_sub, compile_sub, generate_sub, test_sub,
-                          rpc_sub, seed_sub, parse_sub)
-    # --models, --exclude
+                          rpc_sub, seed_sub, parse_sub, build_sub)
+    # --select, --exclude
     # list_sub sets up its own arguments.
-    _add_selection_arguments(run_sub, compile_sub, generate_sub, test_sub)
-    _add_selection_arguments(snapshot_sub, seed_sub, models_name='select')
+    _add_selection_arguments(
+        build_sub, run_sub, compile_sub, generate_sub, test_sub, snapshot_sub, seed_sub)
     # --defer
-    _add_defer_argument(run_sub, test_sub)
+    _add_defer_argument(run_sub, test_sub, build_sub)
     # --full-refresh
-    _add_table_mutability_arguments(run_sub, compile_sub)
+    _add_table_mutability_arguments(run_sub, compile_sub, build_sub)
 
     _build_docs_serve_subparser(docs_subs, base_subparser)
-    _build_source_snapshot_freshness_subparser(source_subs, base_subparser)
+    _build_source_freshness_subparser(source_subs, base_subparser)
     _build_run_operation_subparser(subs, base_subparser)
 
     if len(args) == 0:

core/dbt/parser/base.py

@@ -256,9 +256,7 @@ class ConfiguredParser(
             parsed_node, self.root_project, self.manifest, config
         )
 
-    def render_with_context(
-        self, parsed_node: IntermediateNode, config: ContextConfig
-    ) -> None:
+    def render_with_context(self, parsed_node: IntermediateNode, config: ContextConfig):
         # Given the parsed node and a ContextConfig to use during parsing,
         # render the node's sql wtih macro capture enabled.
         # Note: this mutates the config object when config calls are rendered.
@@ -273,16 +271,17 @@ class ConfiguredParser(
             get_rendered(
                 parsed_node.raw_sql, context, parsed_node, capture_macros=True
             )
+        return context
 
     # This is taking the original config for the node, converting it to a dict,
     # updating the config with new config passed in, then re-creating the
     # config from the dict in the node.
-    def update_parsed_node_config(
+    def update_parsed_node_config_dict(
         self, parsed_node: IntermediateNode, config_dict: Dict[str, Any]
     ) -> None:
         # Overwrite node config
         final_config_dict = parsed_node.config.to_dict(omit_none=True)
-        final_config_dict.update(config_dict)
+        final_config_dict.update({k.strip(): v for (k, v) in config_dict.items()})
         # re-mangle hooks, in case we got new ones
         self._mangle_hooks(final_config_dict)
         parsed_node.config = parsed_node.config.from_dict(final_config_dict)
@@ -294,28 +293,50 @@ class ConfiguredParser(
         self._update_node_schema(parsed_node, config_dict)
         self._update_node_alias(parsed_node, config_dict)
 
-    def update_parsed_node(
-        self, parsed_node: IntermediateNode, config: ContextConfig
+    def update_parsed_node_config(
+        self, parsed_node: IntermediateNode, config: ContextConfig,
+        context=None, patch_config_dict=None
     ) -> None:
         """Given the ContextConfig used for parsing and the parsed node,
         generate and set the true values to use, overriding the temporary parse
         values set in _build_intermediate_parsed_node.
         """
-        config_dict = config.build_config_dict()
 
-        # Set tags on node provided in config blocks
+        # build_config_dict takes the config_call_dict in the ContextConfig object
+        # and calls calculate_node_config to combine dbt_project configs and
+        # config calls from SQL files
+        config_dict = config.build_config_dict(patch_config_dict=patch_config_dict)
+
+        # Set tags on node provided in config blocks. Tags are additive, so even if
+        # config has been built before, we don't have to reset tags in the parsed_node.
         model_tags = config_dict.get('tags', [])
-        parsed_node.tags.extend(model_tags)
+        for tag in model_tags:
+            if tag not in parsed_node.tags:
+                parsed_node.tags.append(tag)
 
+        # If we have meta in the config, copy to node level, for backwards
+        # compatibility with earlier node-only config.
+        if 'meta' in config_dict and config_dict['meta']:
+            parsed_node.meta = config_dict['meta']
+
+        # unrendered_config is used to compare the original database/schema/alias
+        # values and to handle 'same_config' and 'same_contents' calls
         parsed_node.unrendered_config = config.build_config_dict(
             rendered=False
         )
 
+        parsed_node.config_call_dict = config._config_call_dict
+
         # do this once before we parse the node database/schema/alias, so
         # parsed_node.config is what it would be if they did nothing
-        self.update_parsed_node_config(parsed_node, config_dict)
+        self.update_parsed_node_config_dict(parsed_node, config_dict)
+        # This updates the node database/schema/alias
         self.update_parsed_node_name(parsed_node, config_dict)
 
+        # tests don't have hooks
+        if parsed_node.resource_type == NodeType.Test:
+            return
+
         # at this point, we've collected our hooks. Use the node context to
         # render each hook and collect refs/sources
         hooks = list(itertools.chain(parsed_node.config.pre_hook,
@@ -323,9 +344,8 @@ class ConfiguredParser(
         # skip context rebuilding if there aren't any hooks
         if not hooks:
             return
-        # we could cache the original context from parsing this node. Is that
-        # worth the cost in memory/complexity?
-        context = self._context_for(parsed_node, config)
+        if not context:
+            context = self._context_for(parsed_node, config)
         for hook in hooks:
             get_rendered(hook.sql, context, parsed_node, capture_macros=True)
 
@@ -357,8 +377,8 @@ class ConfiguredParser(
         self, node: IntermediateNode, config: ContextConfig
     ) -> None:
         try:
-            self.render_with_context(node, config)
-            self.update_parsed_node(node, config)
+            context = self.render_with_context(node, config)
+            self.update_parsed_node_config(node, config, context=context)
         except ValidationError as exc:
             # we got a ValidationError - probably bad types in config()
             msg = validator_error_message(exc)

core/dbt/parser/manifest.py

@@ -1,8 +1,9 @@
 from dataclasses import dataclass
 from dataclasses import field
 import os
+import traceback
 from typing import (
-    Dict, Optional, Mapping, Callable, Any, List, Type, Union
+    Dict, Optional, Mapping, Callable, Any, List, Type, Union, Tuple
 )
 import time
 
@@ -31,7 +32,7 @@ from dbt.parser.read_files import read_files, load_source_file
 from dbt.parser.partial import PartialParsing
 from dbt.contracts.graph.compiled import ManifestNode
 from dbt.contracts.graph.manifest import (
-    Manifest, Disabled, MacroManifest, ManifestStateCheck
+    Manifest, Disabled, MacroManifest, ManifestStateCheck, ParsingInfo
 )
 from dbt.contracts.graph.parsed import (
     ParsedSourceDefinition, ParsedNode, ParsedMacro, ColumnInfo, ParsedExposure
@@ -59,19 +60,30 @@ from dbt.parser.sources import SourcePatcher
 from dbt.ui import warning_tag
 from dbt.version import __version__
 
-from dbt.dataclass_schema import dbtClassMixin
+from dbt.dataclass_schema import StrEnum, dbtClassMixin
 
 PARTIAL_PARSE_FILE_NAME = 'partial_parse.msgpack'
 PARSING_STATE = DbtProcessState('parsing')
 DEFAULT_PARTIAL_PARSE = False
 
 
+class ReparseReason(StrEnum):
+    version_mismatch = '01_version_mismatch'
+    file_not_found = '02_file_not_found'
+    vars_changed = '03_vars_changed'
+    profile_changed = '04_profile_changed'
+    deps_changed = '05_deps_changed'
+    project_config_changed = '06_project_config_changed'
+    load_file_failure = '07_load_file_failure'
+    exception = '08_exception'
+
+
 # Part of saved performance info
 @dataclass
 class ParserInfo(dbtClassMixin):
     parser: str
     elapsed: float
-    path_count: int = 0
+    parsed_path_count: int = 0
 
 
 # Part of saved performance info
@@ -80,14 +92,18 @@ class ProjectLoaderInfo(dbtClassMixin):
     project_name: str
     elapsed: float
     parsers: List[ParserInfo] = field(default_factory=list)
-    path_count: int = 0
+    parsed_path_count: int = 0
 
 
 # Part of saved performance info
 @dataclass
 class ManifestLoaderInfo(dbtClassMixin, Writable):
     path_count: int = 0
+    parsed_path_count: int = 0
+    static_analysis_path_count: int = 0
+    static_analysis_parsed_path_count: int = 0
     is_partial_parse_enabled: Optional[bool] = None
+    is_static_analysis_enabled: Optional[bool] = None
     read_files_elapsed: Optional[float] = None
     load_macros_elapsed: Optional[float] = None
     parse_project_elapsed: Optional[float] = None
@@ -135,8 +151,6 @@ class ManifestLoader:
         # have been enabled, but not happening because of some issue.
         self.partially_parsing = False
 
-        self._perf_info = self.build_perf_info()
-
         # This is a saved manifest from a previous run that's used for partial parsing
         self.saved_manifest: Optional[Manifest] = self.read_manifest_for_partial_parse()
 
@@ -184,37 +198,71 @@ class ManifestLoader:
 
     # This is where the main action happens
     def load(self):
-
         # Read files creates a dictionary of projects to a dictionary
         # of parsers to lists of file strings. The file strings are
         # used to get the SourceFiles from the manifest files.
-        # In the future the loaded files will be used to control
-        # partial parsing, but right now we're just moving the
-        # file loading out of the individual parsers and doing it
-        # all at once.
         start_read_files = time.perf_counter()
         project_parser_files = {}
         for project in self.all_projects.values():
             read_files(project, self.manifest.files, project_parser_files)
+        self._perf_info.path_count = len(self.manifest.files)
         self._perf_info.read_files_elapsed = (time.perf_counter() - start_read_files)
 
         skip_parsing = False
         if self.saved_manifest is not None:
             partial_parsing = PartialParsing(self.saved_manifest, self.manifest.files)
             skip_parsing = partial_parsing.skip_parsing()
-            if not skip_parsing:
+            if skip_parsing:
+                # nothing changed, so we don't need to generate project_parser_files
+                self.manifest = self.saved_manifest
+            else:
                 # create child_map and parent_map
                 self.saved_manifest.build_parent_and_child_maps()
                 # files are different, we need to create a new set of
                 # project_parser_files.
-                project_parser_files = partial_parsing.get_parsing_files()
-                self.manifest = self.saved_manifest
-                self.partially_parsing = True
+                try:
+                    project_parser_files = partial_parsing.get_parsing_files()
+                    self.partially_parsing = True
+                    self.manifest = self.saved_manifest
+                except Exception:
+                    # pp_files should still be the full set and manifest is new manifest,
+                    # since get_parsing_files failed
+                    logger.info("Partial parsing enabled but an error occurred. "
+                                "Switching to a full re-parse.")
+
+                    # Get traceback info
+                    tb_info = traceback.format_exc()
+                    formatted_lines = tb_info.splitlines()
+                    (_, line, method) = formatted_lines[-3].split(', ')
+                    exc_info = {
+                        "traceback": tb_info,
+                        "exception": formatted_lines[-1],
+                        "code": formatted_lines[-2],
+                        "location": f"{line} {method}",
+                    }
+
+                    # get file info for local logs
+                    parse_file_type = None
+                    file_id = partial_parsing.processing_file
+                    if file_id and file_id in self.manifest.files:
+                        old_file = self.manifest.files[file_id]
+                        parse_file_type = old_file.parse_file_type
+                        logger.debug(f"Partial parsing exception processing file {file_id}")
+                        file_dict = old_file.to_dict()
+                        logger.debug(f"PP file: {file_dict}")
+                    exc_info['parse_file_type'] = parse_file_type
+                    logger.debug(f"PP exception info: {exc_info}")
+
+                    # Send event
+                    if dbt.tracking.active_user is not None:
+                        exc_info['full_reparse_reason'] = ReparseReason.exception
+                        dbt.tracking.track_partial_parser(exc_info)
+
+        if self.manifest._parsing_info is None:
+            self.manifest._parsing_info = ParsingInfo()
 
         if skip_parsing:
             logger.info("Partial parsing enabled, no changes found, skipping parsing")
-            self.manifest = self.saved_manifest
-
         else:
             # Load Macros
             # We need to parse the macros first, so they're resolvable when
@@ -230,6 +278,8 @@ class ManifestLoader:
                 for file_id in parser_files['MacroParser']:
                     block = FileBlock(self.manifest.files[file_id])
                     parser.parse_file(block)
+                    # increment parsed path count for performance tracking
+                    self._perf_info.parsed_path_count = self._perf_info.parsed_path_count + 1
             # Look at changed macros and update the macro.depends_on.macros
             self.macro_depends_on()
             self._perf_info.load_macros_elapsed = (time.perf_counter() - start_load_macros)
@@ -301,9 +351,17 @@ class ManifestLoader:
             self.process_sources(self.root_project.project_name)
             self.process_refs(self.root_project.project_name)
             self.process_docs(self.root_project)
+
+            # update tracking data
             self._perf_info.process_manifest_elapsed = (
                 time.perf_counter() - start_process
             )
+            self._perf_info.static_analysis_parsed_path_count = (
+                self.manifest._parsing_info.static_analysis_parsed_path_count
+            )
+            self._perf_info.static_analysis_path_count = (
+                self.manifest._parsing_info.static_analysis_path_count
+            )
 
             # write out the fully parsed manifest
             self.write_manifest_for_partial_parse()
@@ -321,7 +379,7 @@ class ManifestLoader:
 
         project_loader_info = self._perf_info._project_index[project.project_name]
         start_timer = time.perf_counter()
-        total_path_count = 0
+        total_parsed_path_count = 0
 
         # Loop through parsers with loaded files.
         for parser_cls in parser_types:
@@ -331,7 +389,7 @@ class ManifestLoader:
                 continue
 
             # Initialize timing info
-            parser_path_count = 0
+            project_parsed_path_count = 0
             parser_start_timer = time.perf_counter()
 
             # Parse the project files for this parser
@@ -347,15 +405,15 @@ class ManifestLoader:
                     parser.parse_file(block, dct=dct)
                 else:
                     parser.parse_file(block)
-                parser_path_count = parser_path_count + 1
+                project_parsed_path_count = project_parsed_path_count + 1
 
             # Save timing info
             project_loader_info.parsers.append(ParserInfo(
                 parser=parser.resource_type,
-                path_count=parser_path_count,
+                parsed_path_count=project_parsed_path_count,
                 elapsed=time.perf_counter() - parser_start_timer
             ))
-            total_path_count = total_path_count + parser_path_count
+            total_parsed_path_count = total_parsed_path_count + project_parsed_path_count
 
         # HookParser doesn't run from loaded files, just dbt_project.yml,
         # so do separately
@@ -365,17 +423,19 @@ class ManifestLoader:
         if not self.partially_parsing and HookParser in parser_types:
             hook_parser = HookParser(project, self.manifest, self.root_project)
             path = hook_parser.get_path()
-            file_block = FileBlock(
-                load_source_file(path, ParseFileType.Hook, project.project_name)
-            )
-            hook_parser.parse_file(file_block)
+            file = load_source_file(path, ParseFileType.Hook, project.project_name)
+            if file:
+                file_block = FileBlock(file)
+                hook_parser.parse_file(file_block)
 
         # Store the performance info
         elapsed = time.perf_counter() - start_timer
-        project_loader_info.path_count = project_loader_info.path_count + total_path_count
+        project_loader_info.parsed_path_count = (
+            project_loader_info.parsed_path_count + total_parsed_path_count
+        )
         project_loader_info.elapsed = project_loader_info.elapsed + elapsed
-        self._perf_info.path_count = (
-            self._perf_info.path_count + total_path_count
+        self._perf_info.parsed_path_count = (
+            self._perf_info.parsed_path_count + total_parsed_path_count
         )
 
     # Loop through macros in the manifest and statically parse
@@ -418,6 +478,12 @@ class ManifestLoader:
         path = os.path.join(self.root_project.target_path,
                             PARTIAL_PARSE_FILE_NAME)
         try:
+            # This shouldn't be necessary, but we have gotten bug reports (#3757) of the
+            # saved manifest not matching the code version.
+            if self.manifest.metadata.dbt_version != __version__:
+                logger.debug("Manifest metadata did not contain correct version. "
+                             f"Contained '{self.manifest.metadata.dbt_version}' instead.")
+                self.manifest.metadata.dbt_version = __version__
             manifest_msgpack = self.manifest.to_msgpack()
             make_directory(os.path.dirname(path))
             with open(path, 'wb') as fp:
@@ -425,24 +491,31 @@ class ManifestLoader:
         except Exception:
             raise
 
-    def matching_parse_results(self, manifest: Manifest) -> bool:
+    def is_partial_parsable(self, manifest: Manifest) -> Tuple[bool, Optional[str]]:
         """Compare the global hashes of the read-in parse results' values to
         the known ones, and return if it is ok to re-use the results.
         """
         valid = True
+        reparse_reason = None
 
         if manifest.metadata.dbt_version != __version__:
-            logger.info("Unable to do partial parsing because of a dbt version mismatch")
-            return False  # If the version is wrong, the other checks might not work
+            # #3757 log both versions because of reports of invalid cases of mismatch.
+            logger.info("Unable to do partial parsing because of a dbt version mismatch. "
+                        f"Saved manifest version: {manifest.metadata.dbt_version}. "
+                        f"Current version: {__version__}.")
+            # If the version is wrong, the other checks might not work
+            return False, ReparseReason.version_mismatch
         if self.manifest.state_check.vars_hash != manifest.state_check.vars_hash:
             logger.info("Unable to do partial parsing because config vars, "
                         "config profile, or config target have changed")
             valid = False
+            reparse_reason = ReparseReason.vars_changed
         if self.manifest.state_check.profile_hash != manifest.state_check.profile_hash:
             # Note: This should be made more granular. We shouldn't need to invalidate
             # partial parsing if a non-used profile section has changed.
             logger.info("Unable to do partial parsing because profile has changed")
             valid = False
+            reparse_reason = ReparseReason.profile_changed
 
         missing_keys = {
             k for k in self.manifest.state_check.project_hashes
@@ -451,6 +524,7 @@ class ManifestLoader:
         if missing_keys:
             logger.info("Unable to do partial parsing because a project dependency has been added")
             valid = False
+            reparse_reason = ReparseReason.deps_changed
 
         for key, new_value in self.manifest.state_check.project_hashes.items():
             if key in manifest.state_check.project_hashes:
@@ -459,7 +533,8 @@ class ManifestLoader:
                     logger.info("Unable to do partial parsing because "
                                 "a project config has changed")
                     valid = False
-        return valid
+                    reparse_reason = ReparseReason.project_config_changed
+        return valid, reparse_reason
 
     def _partial_parse_enabled(self):
         # if the CLI is set, follow that
@@ -478,6 +553,8 @@ class ManifestLoader:
         path = os.path.join(self.root_project.target_path,
                             PARTIAL_PARSE_FILE_NAME)
 
+        reparse_reason = None
+
         if os.path.exists(path):
             try:
                 with open(path, 'rb') as fp:
@@ -486,7 +563,8 @@ class ManifestLoader:
                 # keep this check inside the try/except in case something about
                 # the file has changed in weird ways, perhaps due to being a
                 # different version of dbt
-                if self.matching_parse_results(manifest):
+                is_partial_parseable, reparse_reason = self.is_partial_parsable(manifest)
+                if is_partial_parseable:
                     return manifest
             except Exception as exc:
                 logger.debug(
@@ -494,19 +572,24 @@ class ManifestLoader:
                     .format(path, exc),
                     exc_info=True
                 )
+                reparse_reason = ReparseReason.load_file_failure
         else:
             logger.info(f"Unable to do partial parsing because {path} not found")
+            reparse_reason = ReparseReason.file_not_found
+
+        # this event is only fired if a full reparse is needed
+        dbt.tracking.track_partial_parser({'full_reparse_reason': reparse_reason})
 
         return None
 
     def build_perf_info(self):
         mli = ManifestLoaderInfo(
-            is_partial_parse_enabled=self._partial_parse_enabled()
+            is_partial_parse_enabled=self._partial_parse_enabled(),
+            is_static_analysis_enabled=flags.USE_EXPERIMENTAL_PARSER
         )
         for project in self.all_projects.values():
             project_info = ProjectLoaderInfo(
                 project_name=project.project_name,
-                path_count=0,
                 elapsed=0,
             )
             mli.projects.append(project_info)
@@ -603,6 +686,7 @@ class ManifestLoader:
             "invocation_id": invocation_id,
             "project_id": self.root_project.hashed_name(),
             "path_count": self._perf_info.path_count,
+            "parsed_path_count": self._perf_info.parsed_path_count,
             "read_files_elapsed": self._perf_info.read_files_elapsed,
             "load_macros_elapsed": self._perf_info.load_macros_elapsed,
             "parse_project_elapsed": self._perf_info.parse_project_elapsed,
@@ -614,6 +698,9 @@ class ManifestLoader:
             "is_partial_parse_enabled": (
                 self._perf_info.is_partial_parse_enabled
             ),
+            "is_static_analysis_enabled": self._perf_info.is_static_analysis_enabled,
+            "static_analysis_path_count": self._perf_info.static_analysis_path_count,
+            "static_analysis_parsed_path_count": self._perf_info.static_analysis_parsed_path_count,
         })
 
     # Takes references in 'refs' array of nodes and exposures, finds the target

core/dbt/parser/models.py

@@ -1,10 +1,15 @@
 from dbt.context.context_config import ContextConfig
 from dbt.contracts.graph.parsed import ParsedModelNode
 import dbt.flags as flags
+import dbt.tracking
 from dbt.node_types import NodeType
-from dbt.parser.base import IntermediateNode, SimpleSQLParser
+from dbt.parser.base import SimpleSQLParser
 from dbt.parser.search import FileBlock
-from dbt.tree_sitter_jinja.extractor import extract_from_source
+import dbt.tracking as tracking
+from dbt import utils
+from dbt_extractor import ExtractionError, py_extract_from_source  # type: ignore
+import random
+from typing import Any, Dict, List
 
 
 class ModelParser(SimpleSQLParser[ParsedModelNode]):
@@ -22,46 +27,121 @@ class ModelParser(SimpleSQLParser[ParsedModelNode]):
         return block.path.relative_path
 
     def render_update(
-        self, node: IntermediateNode, config: ContextConfig
+        self, node: ParsedModelNode, config: ContextConfig
     ) -> None:
+        self.manifest._parsing_info.static_analysis_path_count += 1
+
+        # `True` roughly 1/100 times this function is called
+        sample: bool = random.randint(1, 101) == 100
+
+        # run the experimental parser if the flag is on or if we're sampling
+        if flags.USE_EXPERIMENTAL_PARSER or sample:
+            try:
+                experimentally_parsed: Dict[str, List[Any]] = py_extract_from_source(node.raw_sql)
+
+                # second config format
+                config_call_dict: Dict[str, Any] = {}
+                for c in experimentally_parsed['configs']:
+                    ContextConfig._add_config_call(config_call_dict, {c[0]: c[1]})
+
+                # format sources TODO change extractor to match this type
+                source_calls: List[List[str]] = []
+                for s in experimentally_parsed['sources']:
+                    source_calls.append([s[0], s[1]])
+                experimentally_parsed['sources'] = source_calls
+
+            except ExtractionError as e:
+                experimentally_parsed = e
 
         # normal dbt run
         if not flags.USE_EXPERIMENTAL_PARSER:
+            # normal rendering
             super().render_update(node, config)
+            # if we're sampling, compare for correctness
+            if sample:
+                result: List[str] = []
+                # experimental parser couldn't parse
+                if isinstance(experimentally_parsed, Exception):
+                    result += ["01_experimental_parser_cannot_parse"]
+                else:
+                    # look for false positive configs
+                    for k in config_call_dict.keys():
+                        if k not in config._config_call_dict:
+                            result += ["02_false_positive_config_value"]
+                            break
 
-        # if the --use-experimental-parser flag was set
-        else:
+                    # look for missed configs
+                    for k in config._config_call_dict.keys():
+                        if k not in config_call_dict:
+                            result += ["03_missed_config_value"]
+                            break
 
-            # run dbt-jinja extractor (powered by tree-sitter)
-            res = extract_from_source(node.raw_sql)
+                    # look for false positive sources
+                    for s in experimentally_parsed['sources']:
+                        if s not in node.sources:
+                            result += ["04_false_positive_source_value"]
+                            break
 
-            # if it doesn't need python jinja, fit the refs, sources, and configs
+                    # look for missed sources
+                    for s in node.sources:
+                        if s not in experimentally_parsed['sources']:
+                            result += ["05_missed_source_value"]
+                            break
+
+                    # look for false positive refs
+                    for r in experimentally_parsed['refs']:
+                        if r not in node.refs:
+                            result += ["06_false_positive_ref_value"]
+                            break
+
+                    # look for missed refs
+                    for r in node.refs:
+                        if r not in experimentally_parsed['refs']:
+                            result += ["07_missed_ref_value"]
+                            break
+
+                    # if there are no errors, return a success value
+                    if not result:
+                        result = ["00_exact_match"]
+
+                # fire a tracking event. this fires one event for every sample
+                # so that we have data on a per file basis. Not only can we expect
+                # no false positives or misses, we can expect the number model
+                # files parseable by the experimental parser to match our internal
+                # testing.
+                if dbt.tracking.active_user is not None:  # None in some tests
+                    tracking.track_experimental_parser_sample({
+                        "project_id": self.root_project.hashed_name(),
+                        "file_id": utils.get_hash(node),
+                        "status": result
+                    })
+
+        # if the --use-experimental-parser flag was set, and the experimental parser succeeded
+        elif not isinstance(experimentally_parsed, Exception):
+            # since it doesn't need python jinja, fit the refs, sources, and configs
             # into the node. Down the line the rest of the node will be updated with
             # this information. (e.g. depends_on etc.)
-            if not res['python_jinja']:
+            config._config_call_dict = config_call_dict
 
-                config_calls = []
-                for c in res['configs']:
-                    config_calls.append({c[0]: c[1]})
+            # this uses the updated config to set all the right things in the node.
+            # if there are hooks present, it WILL render jinja. Will need to change
+            # when the experimental parser supports hooks
+            self.update_parsed_node_config(node, config)
 
-                config._config_calls = config_calls
+            # update the unrendered config with values from the file.
+            # values from yaml files are in there already
+            node.unrendered_config.update(dict(experimentally_parsed['configs']))
 
-                # this uses the updated config to set all the right things in the node
-                # if there are hooks present, it WILL render jinja. Will need to change
-                # when we support hooks
-                self.update_parsed_node(node, config)
+            # set refs and sources on the node object
+            node.refs += experimentally_parsed['refs']
+            node.sources += experimentally_parsed['sources']
 
-                # udpate the unrendered config with values from the file
-                # values from yaml files are in there already
-                node.unrendered_config.update(dict(res['configs']))
+            # configs don't need to be merged into the node
+            # setting them in config._config_call_dict is sufficient
 
-                # set refs, sources, and configs on the node object
-                node.refs = node.refs + res['refs']
-                for sourcev in res['sources']:
-                    # TODO change extractor to match type here
-                    node.sources.append([sourcev[0], sourcev[1]])
-                for configv in res['configs']:
-                    node.config[configv[0]] = configv[1]
+            self.manifest._parsing_info.static_analysis_parsed_path_count += 1
 
-            else:
-                super().render_update(node, config)
+        # the experimental parser tried and failed on this model.
+        # fall back to python jinja rendering.
+        else:
+            super().render_update(node, config)

core/dbt/parser/partial.py

@@ -1,4 +1,4 @@
-from typing import MutableMapping, Dict
+from typing import MutableMapping, Dict, List
 from dbt.contracts.graph.manifest import Manifest
 from dbt.contracts.files import (
     AnySourceFile, ParseFileType, parse_file_type_to_parser,
@@ -44,7 +44,9 @@ class PartialParsing:
         self.saved_files = self.saved_manifest.files
         self.project_parser_files = {}
         self.deleted_manifest = Manifest()
+        self.macro_child_map: Dict[str, List[str]] = {}
         self.build_file_diff()
+        self.processing_file = None
 
     def skip_parsing(self):
         return (
@@ -63,6 +65,7 @@ class PartialParsing:
         deleted_all_files = saved_file_ids.difference(new_file_ids)
         added = new_file_ids.difference(saved_file_ids)
         common = saved_file_ids.intersection(new_file_ids)
+        changed_or_deleted_macro_file = False
 
         # separate out deleted schema files
         deleted_schema_files = []
@@ -71,6 +74,8 @@ class PartialParsing:
             if self.saved_files[file_id].parse_file_type == ParseFileType.Schema:
                 deleted_schema_files.append(file_id)
             else:
+                if self.saved_files[file_id].parse_file_type == ParseFileType.Macro:
+                    changed_or_deleted_macro_file = True
                 deleted.append(file_id)
 
         changed = []
@@ -87,6 +92,8 @@ class PartialParsing:
                         raise Exception(f"Serialization failure for {file_id}")
                     changed_schema_files.append(file_id)
                 else:
+                    if self.saved_files[file_id].parse_file_type == ParseFileType.Macro:
+                        changed_or_deleted_macro_file = True
                     changed.append(file_id)
         file_diff = {
             "deleted": deleted,
@@ -96,6 +103,8 @@ class PartialParsing:
             "changed_schema_files": changed_schema_files,
             "unchanged": unchanged,
         }
+        if changed_or_deleted_macro_file:
+            self.macro_child_map = self.saved_manifest.build_macro_child_map()
         logger.info(f"Partial parsing enabled: "
                     f"{len(deleted) + len(deleted_schema_files)} files deleted, "
                     f"{len(added)} files added, "
@@ -110,16 +119,21 @@ class PartialParsing:
         # Need to add new files first, because changes in schema files
         # might refer to them
         for file_id in self.file_diff['added']:
+            self.processing_file = file_id
             self.add_to_saved(file_id)
         # Need to process schema files next, because the dictionaries
         # need to be in place for handling SQL file changes
         for file_id in self.file_diff['changed_schema_files']:
+            self.processing_file = file_id
             self.change_schema_file(file_id)
         for file_id in self.file_diff['deleted_schema_files']:
+            self.processing_file = file_id
             self.delete_schema_file(file_id)
         for file_id in self.file_diff['deleted']:
+            self.processing_file = file_id
             self.delete_from_saved(file_id)
         for file_id in self.file_diff['changed']:
+            self.processing_file = file_id
             self.update_in_saved(file_id)
         return self.project_parser_files
 
@@ -139,6 +153,18 @@ class PartialParsing:
                 file_id not in self.file_diff['deleted']):
             self.project_parser_files[project_name][parser_name].append(file_id)
 
+    def already_scheduled_for_parsing(self, source_file):
+        file_id = source_file.file_id
+        project_name = source_file.project_name
+        if project_name not in self.project_parser_files:
+            return False
+        parser_name = parse_file_type_to_parser[source_file.parse_file_type]
+        if parser_name not in self.project_parser_files[project_name]:
+            return False
+        if file_id not in self.project_parser_files[project_name][parser_name]:
+            return False
+        return True
+
     # Add new files, including schema files
     def add_to_saved(self, file_id):
         # add file object to saved manifest.files
@@ -174,7 +200,7 @@ class PartialParsing:
 
         # macros
         if saved_source_file.parse_file_type == ParseFileType.Macro:
-            self.delete_macro_file(saved_source_file)
+            self.delete_macro_file(saved_source_file, follow_references=True)
 
         # docs
         if saved_source_file.parse_file_type == ParseFileType.Documentation:
@@ -203,6 +229,9 @@ class PartialParsing:
     # Updated schema files should have been processed already.
     def update_mssat_in_saved(self, new_source_file, old_source_file):
 
+        if self.already_scheduled_for_parsing(old_source_file):
+            return
+
         # These files only have one node.
         unique_id = old_source_file.nodes[0]
 
@@ -214,6 +243,10 @@ class PartialParsing:
         self.remove_node_in_saved(new_source_file, unique_id)
 
     def remove_node_in_saved(self, source_file, unique_id):
+        # Has already been deleted by another action
+        if unique_id not in self.saved_manifest.nodes:
+            return
+
         # delete node in saved
         node = self.saved_manifest.nodes.pop(unique_id)
         self.deleted_manifest.nodes[unique_id] = node
@@ -239,12 +272,16 @@ class PartialParsing:
                         schema_file.node_patches.remove(unique_id)
 
     def update_macro_in_saved(self, new_source_file, old_source_file):
-        self.handle_macro_file_links(old_source_file)
+        if self.already_scheduled_for_parsing(old_source_file):
+            return
+        self.handle_macro_file_links(old_source_file, follow_references=True)
         file_id = new_source_file.file_id
         self.saved_files[file_id] = new_source_file
         self.add_to_pp_files(new_source_file)
 
     def update_doc_in_saved(self, new_source_file, old_source_file):
+        if self.already_scheduled_for_parsing(old_source_file):
+            return
         self.delete_doc_node(old_source_file)
         self.saved_files[new_source_file.file_id] = new_source_file
         self.add_to_pp_files(new_source_file)
@@ -289,7 +326,7 @@ class PartialParsing:
                     source_element = self.get_schema_element(sources, source.source_name)
                     if source_element:
                         self.delete_schema_source(schema_file, source_element)
-                        self.remove_tests(schema_file, source_element['name'])
+                        self.remove_tests(schema_file, 'sources', source_element['name'])
                         self.merge_patch(schema_file, 'sources', source_element)
             elif unique_id in self.saved_manifest.exposures:
                 exposure = self.saved_manifest.exposures[unique_id]
@@ -312,41 +349,42 @@ class PartialParsing:
                     self.saved_files[file_id] = self.new_files[file_id]
                     self.add_to_pp_files(self.saved_files[file_id])
 
-    def delete_macro_file(self, source_file):
-        self.handle_macro_file_links(source_file)
+    def delete_macro_file(self, source_file, follow_references=False):
+        self.handle_macro_file_links(source_file, follow_references)
         file_id = source_file.file_id
         self.deleted_manifest.files[file_id] = self.saved_files.pop(file_id)
 
-    def handle_macro_file_links(self, source_file):
+    def recursively_gather_macro_references(self, macro_unique_id, referencing_nodes):
+        for unique_id in self.macro_child_map[macro_unique_id]:
+            if unique_id in referencing_nodes:
+                continue
+            referencing_nodes.append(unique_id)
+            if unique_id.startswith('macro.'):
+                self.recursively_gather_macro_references(unique_id, referencing_nodes)
+
+    def handle_macro_file_links(self, source_file, follow_references=False):
         # remove the macros in the 'macros' dictionary
-        for unique_id in source_file.macros:
+        macros = source_file.macros.copy()
+        for unique_id in macros:
+            if unique_id not in self.saved_manifest.macros:
+                # This happens when a macro has already been removed
+                if unique_id in source_file.macros:
+                    source_file.macros.remove(unique_id)
+                continue
+
             base_macro = self.saved_manifest.macros.pop(unique_id)
             self.deleted_manifest.macros[unique_id] = base_macro
-            # loop through all macros, finding references to this macro: macro.depends_on.macros
-            for macro in self.saved_manifest.macros.values():
-                for macro_unique_id in macro.depends_on.macros:
-                    if (macro_unique_id == unique_id and
-                            macro_unique_id in self.saved_manifest.macros):
-                        # schedule file for parsing
-                        dep_file_id = macro.file_id
-                        if dep_file_id in self.saved_files:
-                            source_file = self.saved_files[dep_file_id]
-                            dep_macro = self.saved_manifest.macros.pop(macro.unique_id)
-                            self.deleted_manifest.macros[macro.unqiue_id] = dep_macro
-                            self.add_to_pp_files(source_file)
-                        break
-            # loop through all nodes, finding references to this macro: node.depends_on.macros
-            for node in self.saved_manifest.nodes.values():
-                for macro_unique_id in node.depends_on.macros:
-                    if (macro_unique_id == unique_id and
-                            macro_unique_id in self.saved_manifest.macros):
-                        # schedule file for parsing
-                        dep_file_id = node.file_id
-                        if dep_file_id in self.saved_files:
-                            source_file = self.saved_files[dep_file_id]
-                            self.remove_node_in_saved(source_file, node.unique_id)
-                            self.add_to_pp_files(source_file)
-                        break
+
+            # Recursively check children of this macro
+            # The macro_child_map might not exist if a macro is removed by
+            # schedule_nodes_for parsing. We only want to follow
+            # references if the macro file itself has been updated or
+            # deleted, not if we're just updating referenced nodes.
+            if self.macro_child_map and follow_references:
+                referencing_nodes = []
+                self.recursively_gather_macro_references(unique_id, referencing_nodes)
+                self.schedule_macro_nodes_for_parsing(referencing_nodes)
+
             if base_macro.patch_path:
                 file_id = base_macro.patch_path
                 if file_id in self.saved_files:
@@ -357,6 +395,55 @@ class PartialParsing:
                     macro_patch = self.get_schema_element(macro_patches, base_macro.name)
                     self.delete_schema_macro_patch(schema_file, macro_patch)
                     self.merge_patch(schema_file, 'macros', macro_patch)
+            # The macro may have already been removed by handling macro children
+            if unique_id in source_file.macros:
+                source_file.macros.remove(unique_id)
+
+    # similar to schedule_nodes_for_parsing but doesn't do sources and exposures
+    # and handles schema tests
+    def schedule_macro_nodes_for_parsing(self, unique_ids):
+        for unique_id in unique_ids:
+            if unique_id in self.saved_manifest.nodes:
+                node = self.saved_manifest.nodes[unique_id]
+                if node.resource_type == NodeType.Test:
+                    schema_file_id = node.file_id
+                    schema_file = self.saved_manifest.files[schema_file_id]
+                    (key, name) = schema_file.get_key_and_name_for_test(node.unique_id)
+                    if key and name:
+                        patch_list = []
+                        if key in schema_file.dict_from_yaml:
+                            patch_list = schema_file.dict_from_yaml[key]
+                        patch = self.get_schema_element(patch_list, name)
+                        if patch:
+                            if key in ['models', 'seeds', 'snapshots']:
+                                self.delete_schema_mssa_links(schema_file, key, patch)
+                                self.merge_patch(schema_file, key, patch)
+                                if unique_id in schema_file.node_patches:
+                                    schema_file.node_patches.remove(unique_id)
+                            elif key == 'sources':
+                                # re-schedule source
+                                if 'overrides' in patch:
+                                    # This is a source patch; need to re-parse orig source
+                                    self.remove_source_override_target(patch)
+                                self.delete_schema_source(schema_file, patch)
+                                self.remove_tests(schema_file, 'sources', patch['name'])
+                                self.merge_patch(schema_file, 'sources', patch)
+                else:
+                    file_id = node.file_id
+                    if file_id in self.saved_files and file_id not in self.file_diff['deleted']:
+                        source_file = self.saved_files[file_id]
+                        self.remove_mssat_file(source_file)
+                        # content of non-schema files is only in new files
+                        self.saved_files[file_id] = self.new_files[file_id]
+                        self.add_to_pp_files(self.saved_files[file_id])
+            elif unique_id in self.saved_manifest.macros:
+                macro = self.saved_manifest.macros[unique_id]
+                file_id = macro.file_id
+                if file_id in self.saved_files and file_id not in self.file_diff['deleted']:
+                    source_file = self.saved_files[file_id]
+                    self.delete_macro_file(source_file)
+                    self.saved_files[file_id] = self.new_files[file_id]
+                    self.add_to_pp_files(self.saved_files[file_id])
 
     def delete_doc_node(self, source_file):
         # remove the nodes in the 'docs' dictionary
@@ -376,7 +463,13 @@ class PartialParsing:
         new_schema_file = self.new_files[file_id]
         saved_yaml_dict = saved_schema_file.dict_from_yaml
         new_yaml_dict = new_schema_file.dict_from_yaml
-        saved_schema_file.pp_dict = {"version": saved_yaml_dict['version']}
+        if 'version' in new_yaml_dict:
+            # despite the fact that this goes in the saved_schema_file, it
+            # should represent the new yaml dictionary, and should produce
+            # an error if the updated yaml file doesn't have a version
+            saved_schema_file.pp_dict = {"version": new_yaml_dict['version']}
+        else:
+            saved_schema_file.pp_dict = {}
         self.handle_schema_file_changes(saved_schema_file, saved_yaml_dict, new_yaml_dict)
 
         # copy from new schema_file to saved_schema_file to preserve references
@@ -424,14 +517,14 @@ class PartialParsing:
                 if 'overrides' in source:  # This is a source patch; need to re-parse orig source
                     self.remove_source_override_target(source)
                 self.delete_schema_source(schema_file, source)
-                self.remove_tests(schema_file, source['name'])
+                self.remove_tests(schema_file, 'sources', source['name'])
                 self.merge_patch(schema_file, 'sources', source)
         if source_diff['deleted']:
             for source in source_diff['deleted']:
                 if 'overrides' in source:  # This is a source patch; need to re-parse orig source
                     self.remove_source_override_target(source)
                 self.delete_schema_source(schema_file, source)
-                self.remove_tests(schema_file, source['name'])
+                self.remove_tests(schema_file, 'sources', source['name'])
         if source_diff['added']:
             for source in source_diff['added']:
                 if 'overrides' in source:  # This is a source patch; need to re-parse orig source
@@ -556,49 +649,15 @@ class PartialParsing:
         # for models, seeds, snapshots (not analyses)
         if dict_key in ['models', 'seeds', 'snapshots']:
             # find related tests and remove them
-            self.remove_tests(schema_file, elem['name'])
+            self.remove_tests(schema_file, dict_key, elem['name'])
 
-    def remove_tests(self, schema_file, name):
-        tests = self.get_tests_for(schema_file, name)
+    def remove_tests(self, schema_file, dict_key, name):
+        tests = schema_file.get_tests(dict_key, name)
         for test_unique_id in tests:
-            node = self.saved_manifest.nodes.pop(test_unique_id)
-            self.deleted_manifest.nodes[test_unique_id] = node
-            schema_file.tests.remove(test_unique_id)
-
-    # Create a pp_test_index in the schema file if it doesn't exist
-    # and look for test names related to this yaml dict element name
-    def get_tests_for(self, schema_file, name):
-        if not schema_file.pp_test_index:
-            pp_test_index = {}
-            for test_unique_id in schema_file.tests:
-                test_node = self.saved_manifest.nodes[test_unique_id]
-                if test_node.sources:
-                    for source_ref in test_node.sources:
-                        source_name = source_ref[0]
-                        if source_name in pp_test_index:
-                            pp_test_index[source_name].append(test_unique_id)
-                        else:
-                            pp_test_index[source_name] = [test_unique_id]
-                elif test_node.depends_on.nodes:
-                    tested_node_id = test_node.depends_on.nodes[0]
-                    parts = tested_node_id.split('.')
-                    elem_name = parts[-1]
-                    if elem_name in pp_test_index:
-                        pp_test_index[elem_name].append(test_unique_id)
-                    else:
-                        pp_test_index[elem_name] = [test_unique_id]
-                elif (hasattr(test_node, 'test_metadata') and
-                        'model' in test_node.test_metadata.kwargs):
-                    (_, elem_name, _) = test_node.test_metadata.kwargs['model'].split("'")
-                    if elem_name:
-                        if elem_name in pp_test_index:
-                            pp_test_index[elem_name].append(test_unique_id)
-                        else:
-                            pp_test_index[elem_name] = [test_unique_id]
-            schema_file.pp_test_index = pp_test_index
-        if name in schema_file.pp_test_index:
-            return schema_file.pp_test_index[name]
-        return []
+            if test_unique_id in self.saved_manifest.nodes:
+                node = self.saved_manifest.nodes.pop(test_unique_id)
+                self.deleted_manifest.nodes[test_unique_id] = node
+        schema_file.remove_tests(dict_key, name)
 
     def delete_schema_source(self, schema_file, source_dict):
         # both patches, tests, and source nodes
@@ -619,19 +678,17 @@ class PartialParsing:
 
     def delete_schema_macro_patch(self, schema_file, macro):
         # This is just macro patches that need to be reapplied
-        for unique_id in schema_file.macro_patches:
-            parts = unique_id.split('.')
-            macro_name = parts[-1]
-            if macro_name == macro['name']:
-                macro_unique_id = unique_id
-                break
+        macro_unique_id = None
+        if macro['name'] in schema_file.macro_patches:
+            macro_unique_id = schema_file.macro_patches[macro['name']]
+            del schema_file.macro_patches[macro['name']]
         if macro_unique_id and macro_unique_id in self.saved_manifest.macros:
             macro = self.saved_manifest.macros.pop(macro_unique_id)
             self.deleted_manifest.macros[macro_unique_id] = macro
             macro_file_id = macro.file_id
-            self.add_to_pp_files(self.saved_files[macro_file_id])
-        if macro_unique_id in schema_file.macro_patches:
-            schema_file.macro_patches.remove(macro_unique_id)
+            if macro_file_id in self.new_files:
+                self.saved_files[macro_file_id] = self.new_files[macro_file_id]
+                self.add_to_pp_files(self.saved_files[macro_file_id])
 
     # exposures are created only from schema files, so just delete
     # the exposure.
@@ -675,6 +732,6 @@ class PartialParsing:
         (orig_file, orig_source) = self.get_source_override_file_and_dict(source_dict)
         if orig_source:
             self.delete_schema_source(orig_file, orig_source)
-            self.remove_tests(orig_file, orig_source['name'])
+            self.remove_tests(orig_file, 'sources', orig_source['name'])
             self.merge_patch(orig_file, 'sources', orig_source)
             self.add_to_pp_files(orig_file)

core/dbt/parser/read_files.py

@@ -3,25 +3,55 @@ from dbt.contracts.files import (
     FilePath, ParseFileType, SourceFile, FileHash, AnySourceFile, SchemaSourceFile
 )
 
-from dbt.parser.schemas import yaml_from_file
+from dbt.parser.schemas import yaml_from_file, schema_file_keys, check_format_version
+from dbt.exceptions import CompilationException
 from dbt.parser.search import FilesystemSearcher
+from typing import Optional
 
 
 # This loads the files contents and creates the SourceFile object
 def load_source_file(
         path: FilePath, parse_file_type: ParseFileType,
-        project_name: str) -> AnySourceFile:
+        project_name: str) -> Optional[AnySourceFile]:
     file_contents = load_file_contents(path.absolute_path, strip=False)
     checksum = FileHash.from_contents(file_contents)
     sf_cls = SchemaSourceFile if parse_file_type == ParseFileType.Schema else SourceFile
     source_file = sf_cls(path=path, checksum=checksum,
                          parse_file_type=parse_file_type, project_name=project_name)
     source_file.contents = file_contents.strip()
-    if parse_file_type == ParseFileType.Schema:
-        source_file.dfy = yaml_from_file(source_file)
+    if parse_file_type == ParseFileType.Schema and source_file.contents:
+        dfy = yaml_from_file(source_file)
+        if dfy:
+            validate_yaml(source_file.path.original_file_path, dfy)
+            source_file.dfy = dfy
+        else:
+            source_file = None
     return source_file
 
 
+# Do some minimal validation of the yaml in a schema file.
+# Check version, that key values are lists and that each element in
+# the lists has a 'name' key
+def validate_yaml(file_path, dct):
+    check_format_version(file_path, dct)
+    for key in schema_file_keys:
+        if key in dct:
+            if not isinstance(dct[key], list):
+                msg = (f"The schema file at {file_path} is "
+                       f"invalid because the value of '{key}' is not a list")
+                raise CompilationException(msg)
+            for element in dct[key]:
+                if not isinstance(element, dict):
+                    msg = (f"The schema file at {file_path} is "
+                           f"invalid because a list element for '{key}' is not a dictionary")
+                    raise CompilationException(msg)
+                if 'name' not in element:
+                    msg = (f"The schema file at {file_path} is "
+                           f"invalid because a list element for '{key}' does not have a "
+                           "name attribute.")
+                    raise CompilationException(msg)
+
+
 # Special processing for big seed files
 def load_seed_source_file(match: FilePath, project_name) -> SourceFile:
     if match.seed_too_large():
@@ -50,8 +80,10 @@ def get_source_files(project, paths, extension, parse_file_type):
         if parse_file_type == ParseFileType.Seed:
             fb_list.append(load_seed_source_file(fp, project.project_name))
         else:
-            fb_list.append(load_source_file(
-                fp, parse_file_type, project.project_name))
+            file = load_source_file(fp, parse_file_type, project.project_name)
+            # only append the list if it has contents. added to fix #3568
+            if file:
+                fb_list.append(file)
     return fb_list
 
 

core/dbt/parser/schema_test_builders.py

@@ -190,9 +190,9 @@ class TestBuilder(Generic[Testable]):
         r'(?P<test_name>([a-zA-Z_][0-9a-zA-Z_]*))'
     )
     # kwargs representing test configs
-    MODIFIER_ARGS = (
+    CONFIG_ARGS = (
         'severity', 'tags', 'enabled', 'where', 'limit', 'warn_if', 'error_if',
-        'fail_calc', 'store_failures'
+        'fail_calc', 'store_failures', 'meta', 'database', 'schema', 'alias',
     )
 
     def __init__(
@@ -224,13 +224,24 @@ class TestBuilder(Generic[Testable]):
         groups = match.groupdict()
         self.name: str = groups['test_name']
         self.namespace: str = groups['test_namespace']
-        self.modifiers: Dict[str, Any] = {}
-        for key in self.MODIFIER_ARGS:
+        self.config: Dict[str, Any] = {}
+
+        for key in self.CONFIG_ARGS:
             value = self.args.pop(key, None)
+            # 'modifier' config could be either top level arg or in config
+            if value and 'config' in self.args and key in self.args['config']:
+                raise_compiler_error(
+                    'Test cannot have the same key at the top-level and in config'
+                )
+            if not value and 'config' in self.args:
+                value = self.args['config'].pop(key, None)
             if isinstance(value, str):
                 value = get_rendered(value, render_ctx, native=True)
             if value is not None:
-                self.modifiers[key] = value
+                self.config[key] = value
+
+        if 'config' in self.args:
+            del self.args['config']
 
         if self.namespace is not None:
             self.package_name = self.namespace
@@ -240,8 +251,8 @@ class TestBuilder(Generic[Testable]):
         self.fqn_name: str = fqn_name
 
         # use hashed name as alias if too long
-        if compiled_name != fqn_name:
-            self.modifiers['alias'] = compiled_name
+        if compiled_name != fqn_name and 'alias' not in self.config:
+            self.config['alias'] = compiled_name
 
     def _bad_type(self) -> TypeError:
         return TypeError('invalid target type "{}"'.format(type(self.target)))
@@ -282,15 +293,15 @@ class TestBuilder(Generic[Testable]):
 
     @property
     def enabled(self) -> Optional[bool]:
-        return self.modifiers.get('enabled')
+        return self.config.get('enabled')
 
     @property
     def alias(self) -> Optional[str]:
-        return self.modifiers.get('alias')
+        return self.config.get('alias')
 
     @property
     def severity(self) -> Optional[str]:
-        sev = self.modifiers.get('severity')
+        sev = self.config.get('severity')
         if sev:
             return sev.upper()
         else:
@@ -298,30 +309,72 @@ class TestBuilder(Generic[Testable]):
 
     @property
     def store_failures(self) -> Optional[bool]:
-        return self.modifiers.get('store_failures')
+        return self.config.get('store_failures')
 
     @property
     def where(self) -> Optional[str]:
-        return self.modifiers.get('where')
+        return self.config.get('where')
 
     @property
     def limit(self) -> Optional[int]:
-        return self.modifiers.get('limit')
+        return self.config.get('limit')
 
     @property
     def warn_if(self) -> Optional[str]:
-        return self.modifiers.get('warn_if')
+        return self.config.get('warn_if')
 
     @property
     def error_if(self) -> Optional[str]:
-        return self.modifiers.get('error_if')
+        return self.config.get('error_if')
 
     @property
     def fail_calc(self) -> Optional[str]:
-        return self.modifiers.get('fail_calc')
+        return self.config.get('fail_calc')
+
+    @property
+    def meta(self) -> Optional[dict]:
+        return self.config.get('meta')
+
+    @property
+    def database(self) -> Optional[str]:
+        return self.config.get('database')
+
+    @property
+    def schema(self) -> Optional[str]:
+        return self.config.get('schema')
+
+    def get_static_config(self):
+        config = {}
+        if self.alias is not None:
+            config['alias'] = self.alias
+        if self.severity is not None:
+            config['severity'] = self.severity
+        if self.enabled is not None:
+            config['enabled'] = self.enabled
+        if self.where is not None:
+            config['where'] = self.where
+        if self.limit is not None:
+            config['limit'] = self.limit
+        if self.warn_if is not None:
+            config['warn_if'] = self.warn_if
+        if self.error_if is not None:
+            config['error_id'] = self.error_if
+        if self.fail_calc is not None:
+            config['fail_calc'] = self.fail_calc
+        if self.store_failures is not None:
+            config['store_failures'] = self.store_failures
+        if self.meta is not None:
+            config['meta'] = self.meta
+        if self.database is not None:
+            config['database'] = self.database
+        if self.schema is not None:
+            config['schema'] = self.schema
+        if self.alias is not None:
+            config['alias'] = self.alias
+        return config
 
     def tags(self) -> List[str]:
-        tags = self.modifiers.get('tags', [])
+        tags = self.config.get('tags', [])
         if isinstance(tags, str):
             tags = [tags]
         if not isinstance(tags, list):
@@ -355,10 +408,12 @@ class TestBuilder(Generic[Testable]):
 
     def construct_config(self) -> str:
         configs = ",".join([
-            f"{key}=" + (f"'{value}'" if isinstance(value, str)
-                         else str(value))
+            f"{key}=" + (
+                ("\"" + value.replace('\"', '\\\"') + "\"") if isinstance(value, str)
+                else str(value)
+            )
             for key, value
-            in self.modifiers.items()
+            in self.config.items()
         ])
         if configs:
             return f"{{{{ config({configs}) }}}}"
@@ -378,12 +433,8 @@ class TestBuilder(Generic[Testable]):
 
     def build_model_str(self):
         targ = self.target
-        cfg_where = "config.get('where')"
         if isinstance(self.target, UnparsedNodeUpdate):
-            identifier = self.target.name
-            target_str = f"{{{{ ref('{targ.name}') }}}}"
+            target_str = f"ref('{targ.name}')"
         elif isinstance(self.target, UnpatchedSourceDefinition):
-            identifier = self.target.table.name
-            target_str = f"{{{{ source('{targ.source.name}', '{targ.table.name}') }}}}"
-        filtered = f"(select * from {target_str} where {{{{{cfg_where}}}}}) {identifier}"
-        return f"{{% if {cfg_where} %}}{filtered}{{% else %}}{target_str}{{% endif %}}"
+            target_str = f"source('{targ.source.name}', '{targ.table.name}')"
+        return f"{{{{ get_where_subquery({target_str}) }}}}"

core/dbt/parser/schemas.py

@@ -22,8 +22,7 @@ from dbt.context.providers import (
     generate_parse_exposure, generate_test_context
 )
 from dbt.context.macro_resolver import MacroResolver
-from dbt.contracts.files import FileHash
-from dbt.contracts.graph.manifest import SchemaSourceFile
+from dbt.contracts.files import FileHash, SchemaSourceFile
 from dbt.contracts.graph.parsed import (
     ParsedNodePatch,
     ColumnInfo,
@@ -47,7 +46,10 @@ from dbt.contracts.graph.unparsed import (
 from dbt.exceptions import (
     validator_error_message, JSONValidationException,
     raise_invalid_schema_yml_version, ValidationException,
-    CompilationException,
+    CompilationException, raise_duplicate_patch_name,
+    raise_duplicate_macro_patch_name, InternalException,
+    raise_duplicate_source_patch_name,
+    warn_or_error,
 )
 from dbt.node_types import NodeType
 from dbt.parser.base import SimpleParser
@@ -70,6 +72,11 @@ UnparsedSchemaYaml = Union[
 
 TestDef = Union[str, Dict[str, Any]]
 
+schema_file_keys = (
+    'models', 'seeds', 'snapshots', 'sources',
+    'macros', 'analyses', 'exposures',
+)
+
 
 def error_context(
     path: str,
@@ -93,10 +100,10 @@ def error_context(
 
 def yaml_from_file(
     source_file: SchemaSourceFile
-) -> Optional[Dict[str, Any]]:
+) -> Dict[str, Any]:
     """If loading the yaml fails, raise an exception.
     """
-    path: str = source_file.path.relative_path
+    path = source_file.path.relative_path
     try:
         return load_yaml_text(source_file.contents)
     except ValidationException as e:
@@ -105,7 +112,6 @@ def yaml_from_file(
             'Error reading {}: {} - {}'
             .format(source_file.project_name, path, reason)
         )
-    return None
 
 
 class ParserRef:
@@ -167,15 +173,15 @@ class SchemaParser(SimpleParser[SchemaTestBlock, ParsedSchemaTestNode]):
             self.project.config_version == 2
         )
         if all_v_2:
-            ctx = generate_schema_yml(
+            self.render_ctx = generate_schema_yml(
                 self.root_project, self.project.project_name
             )
         else:
-            ctx = generate_target_context(
+            self.render_ctx = generate_target_context(
                 self.root_project, self.root_project.cli_vars
             )
 
-        self.raw_renderer = SchemaYamlRenderer(ctx)
+        self.raw_renderer = SchemaYamlRenderer(self.render_ctx)
 
         internal_package_names = get_adapter_package_names(
             self.root_project.credentials.type
@@ -200,25 +206,6 @@ class SchemaParser(SimpleParser[SchemaTestBlock, ParsedSchemaTestNode]):
             ParsedSchemaTestNode.validate(dct)
         return ParsedSchemaTestNode.from_dict(dct)
 
-    def _check_format_version(
-        self, yaml: YamlBlock
-    ) -> None:
-        path = yaml.path.relative_path
-        if 'version' not in yaml.data:
-            raise_invalid_schema_yml_version(path, 'no version is specified')
-
-        version = yaml.data['version']
-        # if it's not an integer, the version is malformed, or not
-        # set. Either way, only 'version: 2' is supported.
-        if not isinstance(version, int):
-            raise_invalid_schema_yml_version(
-                path, 'the version is not an integer'
-            )
-        if version != 2:
-            raise_invalid_schema_yml_version(
-                path, 'version {} is not supported'.format(version)
-            )
-
     def parse_column_tests(
         self, block: TestBlock, column: UnparsedColumn
     ) -> None:
@@ -302,17 +289,13 @@ class SchemaParser(SimpleParser[SchemaTestBlock, ParsedSchemaTestNode]):
         tags: List[str],
         column_name: Optional[str],
     ) -> ParsedSchemaTestNode:
-
-        render_ctx = generate_target_context(
-            self.root_project, self.root_project.cli_vars
-        )
         try:
             builder = TestBuilder(
                 test=test,
                 target=target,
                 column_name=column_name,
                 package_name=target.package_name,
-                render_ctx=render_ctx,
+                render_ctx=self.render_ctx,
             )
         except CompilationException as exc:
             context = _trimmed(str(target))
@@ -333,8 +316,8 @@ class SchemaParser(SimpleParser[SchemaTestBlock, ParsedSchemaTestNode]):
         # is not necessarily this package's name
         fqn = self.get_fqn(fqn_path, builder.fqn_name)
 
-        # this is the config that is used in render_update
-        config = self.initial_config(fqn)
+        # this is the ContextConfig that is used in render_update
+        config: ContextConfig = self.initial_config(fqn)
 
         metadata = {
             'namespace': builder.namespace,
@@ -375,34 +358,10 @@ class SchemaParser(SimpleParser[SchemaTestBlock, ParsedSchemaTestNode]):
         node.depends_on.add_macro(macro_unique_id)
         if (macro_unique_id in
                 ['macro.dbt.test_not_null', 'macro.dbt.test_unique']):
-            self.update_parsed_node(node, config)
-            # manually set configs
-            # note: this does not respect generate_alias_name() macro
-            if builder.alias is not None:
-                node.unrendered_config['alias'] = builder.alias
-                node.config['alias'] = builder.alias
-                node.alias = builder.alias
-            if builder.severity is not None:
-                node.unrendered_config['severity'] = builder.severity
-                node.config['severity'] = builder.severity
-            if builder.enabled is not None:
-                node.unrendered_config['enabled'] = builder.enabled
-                node.config['enabled'] = builder.enabled
-            if builder.where is not None:
-                node.unrendered_config['where'] = builder.where
-                node.config['where'] = builder.where
-            if builder.limit is not None:
-                node.unrendered_config['limit'] = builder.limit
-                node.config['limit'] = builder.limit
-            if builder.warn_if is not None:
-                node.unrendered_config['warn_if'] = builder.warn_if
-                node.config['warn_if'] = builder.warn_if
-            if builder.error_if is not None:
-                node.unrendered_config['error_if'] = builder.error_if
-                node.config['error_if'] = builder.error_if
-            if builder.fail_calc is not None:
-                node.unrendered_config['fail_calc'] = builder.fail_calc
-                node.config['fail_calc'] = builder.fail_calc
+            config_call_dict = builder.get_static_config()
+            config._config_call_dict = config_call_dict
+            # This sets the config from dbt_project
+            self.update_parsed_node_config(node, config)
             # source node tests are processed at patch_source time
             if isinstance(builder.target, UnpatchedSourceDefinition):
                 sources = [builder.target.fqn[-2], builder.target.fqn[-1]]
@@ -422,7 +381,7 @@ class SchemaParser(SimpleParser[SchemaTestBlock, ParsedSchemaTestNode]):
                 get_rendered(
                     node.raw_sql, context, node, capture_macros=True
                 )
-                self.update_parsed_node(node, config)
+                self.update_parsed_node_config(node, config)
             except ValidationError as exc:
                 # we got a ValidationError - probably bad types in config()
                 msg = validator_error_message(exc)
@@ -439,9 +398,16 @@ class SchemaParser(SimpleParser[SchemaTestBlock, ParsedSchemaTestNode]):
             tags=block.tags,
             column_name=block.column_name,
         )
-        self.add_result_node(block, node)
+        self.add_test_node(block, node)
         return node
 
+    def add_test_node(self, block: SchemaTestBlock, node: ParsedSchemaTestNode):
+        test_from = {"key": block.target.yaml_key, "name": block.target.name}
+        if node.config.enabled:
+            self.manifest.add_node(block.file, node, test_from)
+        else:
+            self.manifest.add_disabled(block.file, node, test_from)
+
     def render_with_context(
         self, node: ParsedSchemaTestNode, config: ContextConfig,
     ) -> None:
@@ -514,9 +480,6 @@ class SchemaParser(SimpleParser[SchemaTestBlock, ParsedSchemaTestNode]):
             # contains the FileBlock and the data (dictionary)
             yaml_block = YamlBlock.from_file_block(block, dct)
 
-            # checks version
-            self._check_format_version(yaml_block)
-
             parser: YamlDocsReader
 
             # There are 7 kinds of parsers:
@@ -565,6 +528,25 @@ class SchemaParser(SimpleParser[SchemaTestBlock, ParsedSchemaTestNode]):
                     self.manifest.add_exposure(yaml_block.file, node)
 
 
+def check_format_version(
+    file_path, yaml_dct
+) -> None:
+    if 'version' not in yaml_dct:
+        raise_invalid_schema_yml_version(file_path, 'no version is specified')
+
+    version = yaml_dct['version']
+    # if it's not an integer, the version is malformed, or not
+    # set. Either way, only 'version: 2' is supported.
+    if not isinstance(version, int):
+        raise_invalid_schema_yml_version(
+            file_path, 'the version is not an integer'
+        )
+    if version != 2:
+        raise_invalid_schema_yml_version(
+            file_path, 'version {} is not supported'.format(version)
+        )
+
+
 Parsed = TypeVar(
     'Parsed',
     UnpatchedSourceDefinition, ParsedNodePatch, ParsedMacroPatch
@@ -667,7 +649,14 @@ class SourceParser(YamlDocsReader):
             if is_override:
                 data['path'] = self.yaml.path.original_file_path
                 patch = self._target_from_dict(SourcePatch, data)
-                self.manifest.add_source_patch(self.yaml.file, patch)
+                assert isinstance(self.yaml.file, SchemaSourceFile)
+                source_file = self.yaml.file
+                # source patches must be unique
+                key = (patch.overrides, patch.name)
+                if key in self.manifest.source_patches:
+                    raise_duplicate_source_patch_name(patch, self.manifest.source_patches[key])
+                self.manifest.source_patches[key] = patch
+                source_file.source_patches.append(key)
             else:
                 source = self._target_from_dict(UnparsedSourceDefinition, data)
                 self.add_source_definitions(source)
@@ -764,6 +753,9 @@ class NonSourceParser(YamlDocsReader, Generic[NonSourceTarget, Parsed]):
                 # target_type: UnparsedNodeUpdate, UnparsedAnalysisUpdate,
                 # or UnparsedMacroUpdate
                 self._target_type().validate(data)
+                if self.key != 'macros':
+                    # macros don't have the 'config' key support yet
+                    self.normalize_meta_attribute(data, path)
                 node = self._target_type().from_dict(data)
             except (ValidationError, JSONValidationException) as exc:
                 msg = error_context(path, self.key, data, exc)
@@ -771,6 +763,33 @@ class NonSourceParser(YamlDocsReader, Generic[NonSourceTarget, Parsed]):
             else:
                 yield node
 
+    # We want to raise an error if 'meta' is in two places, and move 'meta'
+    # from toplevel to config if necessary
+    def normalize_meta_attribute(self, data, path):
+        if 'meta' in data:
+            if 'config' in data and 'meta' in data['config']:
+                raise CompilationException(f"""
+                    In {path}: found meta dictionary in 'config' dictionary and as top-level key.
+                    Remove the top-level key and define it under 'config' dictionary only.
+                """.strip())
+            else:
+                if 'config' not in data:
+                    data['config'] = {}
+                data['config']['meta'] = data.pop('meta')
+
+    def patch_node_config(self, node, patch):
+        # Get the ContextConfig that's used in calculating the config
+        # This must match the model resource_type that's being patched
+        config = ContextConfig(
+            self.schema_parser.root_project,
+            node.fqn,
+            node.resource_type,
+            self.schema_parser.project.project_name,
+        )
+        # We need to re-apply the config_call_dict after the patch config
+        config._config_call_dict = node.config_call_dict
+        self.schema_parser.update_parsed_node_config(node, config, patch_config_dict=patch.config)
+
 
 class NodePatchParser(
     NonSourceParser[NodeTarget, ParsedNodePatch],
@@ -779,6 +798,9 @@ class NodePatchParser(
     def parse_patch(
         self, block: TargetBlock[NodeTarget], refs: ParserRef
     ) -> None:
+        # We're not passing the ParsedNodePatch around anymore, so we
+        # could possibly skip creating one. Leaving here for now for
+        # code consistency.
         patch = ParsedNodePatch(
             name=block.target.name,
             original_file_path=block.target.original_file_path,
@@ -788,8 +810,35 @@ class NodePatchParser(
             columns=refs.column_info,
             meta=block.target.meta,
             docs=block.target.docs,
+            config=block.target.config,
         )
-        self.manifest.add_patch(self.yaml.file, patch)
+        assert isinstance(self.yaml.file, SchemaSourceFile)
+        source_file: SchemaSourceFile = self.yaml.file
+        if patch.yaml_key in ['models', 'seeds', 'snapshots']:
+            unique_id = self.manifest.ref_lookup.get_unique_id(patch.name, None)
+        elif patch.yaml_key == 'analyses':
+            unique_id = self.manifest.analysis_lookup.get_unique_id(patch.name, None)
+        else:
+            raise InternalException(
+                f'Unexpected yaml_key {patch.yaml_key} for patch in '
+                f'file {source_file.path.original_file_path}'
+            )
+        if unique_id is None:
+            # This will usually happen when a node is disabled
+            return
+
+        # patches can't be overwritten
+        node = self.manifest.nodes.get(unique_id)
+        if node:
+            if node.patch_path:
+                package_name, existing_file_path = node.patch_path.split('://')
+                raise_duplicate_patch_name(patch, existing_file_path)
+            source_file.append_patch(patch.yaml_key, unique_id)
+            # If this patch has config changes, re-calculate the node config
+            # with the patch config
+            if patch.config:
+                self.patch_node_config(node, patch)
+            node.patch(patch)
 
 
 class TestablePatchParser(NodePatchParser[UnparsedNodeUpdate]):
@@ -827,8 +876,24 @@ class MacroPatchParser(NonSourceParser[UnparsedMacroUpdate, ParsedMacroPatch]):
             description=block.target.description,
             meta=block.target.meta,
             docs=block.target.docs,
+            config=block.target.config,
         )
-        self.manifest.add_macro_patch(self.yaml.file, patch)
+        assert isinstance(self.yaml.file, SchemaSourceFile)
+        source_file = self.yaml.file
+        # macros are fully namespaced
+        unique_id = f'macro.{patch.package_name}.{patch.name}'
+        macro = self.manifest.macros.get(unique_id)
+        if not macro:
+            warn_or_error(
+                f'WARNING: Found patch for macro "{patch.name}" '
+                f'which was not found'
+            )
+            return
+        if macro.patch_path:
+            package_name, existing_file_path = macro.patch_path.split('://')
+            raise_duplicate_macro_patch_name(patch, existing_file_path)
+        source_file.macro_patches[patch.name] = unique_id
+        macro.patch(patch)
 
 
 class ExposureParser(YamlReader):

core/dbt/parser/sources.py

@@ -77,7 +77,8 @@ class SourcePatcher:
                     self.manifest.add_disabled_nofile(test)
                 # save the test unique_id in the schema_file, so we can
                 # process in partial parsing
-                schema_file.tests.append(test.unique_id)
+                test_from = {"key": 'sources', "name": patched.source.name}
+                schema_file.add_test(test.unique_id, test_from)
 
             # Convert UnpatchedSourceDefinition to a ParsedSourceDefinition
             parsed = self.parse_source(patched)
@@ -285,7 +286,7 @@ class SourcePatcher:
             )
 
         return generator.calculate_node_config(
-            config_calls=[],
+            config_call_dict={},
             fqn=fqn,
             resource_type=NodeType.Source,
             project_name=project_name,

core/dbt/rpc/method.py

@@ -1,6 +1,5 @@
 import inspect
 from abc import abstractmethod
-from copy import deepcopy
 from typing import List, Optional, Type, TypeVar, Generic, Dict, Any
 
 from dbt.dataclass_schema import dbtClassMixin, ValidationError
@@ -21,7 +20,7 @@ class RemoteMethod(Generic[Parameters, Result]):
     METHOD_NAME: Optional[str] = None
 
     def __init__(self, args, config):
-        self.args = deepcopy(args)
+        self.args = args
         self.config = config
 
     @classmethod

core/dbt/rpc/task_manager.py

@@ -1,3 +1,4 @@
+from copy import deepcopy
 import threading
 import uuid
 from datetime import datetime
@@ -155,7 +156,7 @@ class TaskManager:
                     f'Manifest should not be None if the last parse state is '
                     f'{state}'
                 )
-            return task(self.args, self.config, self.manifest)
+            return task(deepcopy(self.args), self.config, self.manifest)
 
     def rpc_task(
         self, method_name: str
@@ -167,7 +168,7 @@ class TaskManager:
             elif issubclass(task, RemoteManifestMethod):
                 return self._get_manifest_callable(task)
             elif issubclass(task, RemoteMethod):
-                return task(self.args, self.config)
+                return task(deepcopy(self.args), self.config)
             else:
                 raise dbt.exceptions.InternalException(
                     f'Got a task with an invalid type! {task} with method '