add more future work

* Update werkzeug, refresh-token script

* Add changelog note

.bumpversion.cfg

@@ -1,23 +1,27 @@
 [bumpversion]
-current_version = 0.19.0
+current_version = 0.21.0a1
 parse = (?P<major>\d+)
 	\.(?P<minor>\d+)
 	\.(?P<patch>\d+)
-	((?P<prerelease>[a-z]+)(?P<num>\d+))?
+	((?P<prekind>a|b|rc)
+	(?P<pre>\d+)  # pre-release version num
+	)?
 serialize = 
-	{major}.{minor}.{patch}{prerelease}{num}
+	{major}.{minor}.{patch}{prekind}{pre}
 	{major}.{minor}.{patch}
 commit = False
 tag = False
 
-[bumpversion:part:prerelease]
+[bumpversion:part:prekind]
 first_value = a
+optional_value = final
 values = 
 	a
 	b
 	rc
+	final
 
-[bumpversion:part:num]
+[bumpversion:part:pre]
 first_value = 1
 
 [bumpversion:file:setup.py]
@@ -26,6 +30,8 @@ first_value = 1
 
 [bumpversion:file:core/dbt/version.py]
 
+[bumpversion:file:core/scripts/create_adapter_plugins.py]
+
 [bumpversion:file:plugins/postgres/setup.py]
 
 [bumpversion:file:plugins/redshift/setup.py]
@@ -41,4 +47,3 @@ first_value = 1
 [bumpversion:file:plugins/snowflake/dbt/adapters/snowflake/__version__.py]
 
 [bumpversion:file:plugins/bigquery/dbt/adapters/bigquery/__version__.py]
-

.circleci/config.yml

@@ -2,12 +2,19 @@ version: 2.1
 jobs:
   unit:
     docker: &test_only
-      - image: fishtownanalytics/test-container:9
+      - image: fishtownanalytics/test-container:12
         environment:
           DBT_INVOCATION_ENV: circle
+          DOCKER_TEST_DATABASE_HOST: "database"
+          TOX_PARALLEL_NO_SPINNER: 1
     steps:
       - checkout
-      - run: tox -e flake8,mypy,unit-py36,unit-py38
+      - run: tox -p -e py36,py37,py38
+  lint:
+    docker: *test_only
+    steps:
+      - checkout
+      - run: tox -e mypy,flake8 -- -v
   build-wheels:
     docker: *test_only
     steps:
@@ -19,7 +26,7 @@ jobs:
             export PYTHON_BIN="${PYTHON_ENV}/bin/python"
             $PYTHON_BIN -m pip install -U pip setuptools
             $PYTHON_BIN -m pip install -r requirements.txt
-            $PYTHON_BIN -m pip install -r dev_requirements.txt
+            $PYTHON_BIN -m pip install -r dev-requirements.txt
             /bin/bash ./scripts/build-wheels.sh
             $PYTHON_BIN ./scripts/collect-dbt-contexts.py > ./dist/context_metadata.json
             $PYTHON_BIN ./scripts/collect-artifact-schema.py > ./dist/artifact_schemas.json
@@ -28,20 +35,22 @@ jobs:
       - store_artifacts:
           path: ./dist
           destination: dist
-  integration-postgres-py36:
-    docker: &test_and_postgres
-      - image: fishtownanalytics/test-container:9
+  integration-postgres:
+    docker:
+      - image: fishtownanalytics/test-container:12
         environment:
           DBT_INVOCATION_ENV: circle
+          DOCKER_TEST_DATABASE_HOST: "database"
+          TOX_PARALLEL_NO_SPINNER: 1
       - image: postgres
         name: database
-        environment: &pgenv
+        environment:
           POSTGRES_USER: "root"
           POSTGRES_PASSWORD: "password"
           POSTGRES_DB: "dbt"
     steps:
       - checkout
-      - run: &setupdb
+      - run:
           name: Setup postgres
           command: bash test/setup_db.sh
           environment:
@@ -50,113 +59,39 @@ jobs:
             PGPASSWORD: password
             PGDATABASE: postgres
       - run:
-          name: Run tests
-          command: tox -e integration-postgres-py36
+          name: Postgres integration tests
+          command: tox -p -e py36-postgres,py38-postgres -- -v -n4
+          no_output_timeout: 30m
       - store_artifacts:
           path: ./logs
-  integration-snowflake-py36:
+  integration-snowflake:
     docker: *test_only
     steps:
       - checkout
       - run:
-          name: Run tests
-          command: tox -e integration-snowflake-py36
-          no_output_timeout: 1h
+          name: Snowflake integration tests
+          command: tox -p -e py36-snowflake,py38-snowflake -- -v -n4
+          no_output_timeout: 30m
       - store_artifacts:
           path: ./logs
-  integration-redshift-py36:
+  integration-redshift:
     docker: *test_only
     steps:
       - checkout
       - run:
-          name: Run tests
-          command: tox -e integration-redshift-py36
+          name: Redshift integration tests
+          command: tox -p -e py36-redshift,py38-redshift -- -v -n4
+          no_output_timeout: 30m
       - store_artifacts:
           path: ./logs
-  integration-bigquery-py36:
+  integration-bigquery:
     docker: *test_only
     steps:
       - checkout
       - run:
-          name: Run tests
-          command: tox -e integration-bigquery-py36
-      - store_artifacts:
-          path: ./logs
-  integration-postgres-py38:
-    docker: *test_and_postgres
-    steps:
-      - checkout
-      - run: *setupdb
-      - run:
-          name: Run tests
-          command: tox -e integration-postgres-py38
-      - store_artifacts:
-          path: ./logs
-  integration-snowflake-py38:
-    docker: *test_only
-    steps:
-      - checkout
-      - run:
-          name: Run tests
-          command: tox -e integration-snowflake-py38
-          no_output_timeout: 1h
-      - store_artifacts:
-          path: ./logs
-  integration-redshift-py38:
-    docker: *test_only
-    steps:
-      - checkout
-      - run:
-          name: Run tests
-          command: tox -e integration-redshift-py38
-      - store_artifacts:
-          path: ./logs
-  integration-bigquery-py38:
-    docker: *test_only
-    steps:
-      - checkout
-      - run:
-          name: Run tests
-          command: tox -e integration-bigquery-py38
-      - store_artifacts:
-          path: ./logs
-
-  integration-postgres-py39:
-    docker: *test_and_postgres
-    steps:
-      - checkout
-      - run: *setupdb
-      - run:
-          name: Run tests
-          command: tox -e integration-postgres-py39
-      - store_artifacts:
-          path: ./logs
-  integration-snowflake-py39:
-    docker: *test_only
-    steps:
-      - checkout
-      - run:
-          name: Run tests
-          command: tox -e integration-snowflake-py39
-          no_output_timeout: 1h
-      - store_artifacts:
-          path: ./logs
-  integration-redshift-py39:
-    docker: *test_only
-    steps:
-      - checkout
-      - run:
-          name: Run tests
-          command: tox -e integration-redshift-py39
-      - store_artifacts:
-          path: ./logs
-  integration-bigquery-py39:
-    docker: *test_only
-    steps:
-      - checkout
-      - run:
-          name: Run tests
-          command: tox -e integration-bigquery-py39
+          name: Bigquery integration test
+          command: tox -p -e py36-bigquery,py38-bigquery -- -v -n4
+          no_output_timeout: 30m
       - store_artifacts:
           path: ./logs
 
@@ -164,55 +99,25 @@ workflows:
   version: 2
   test-everything:
     jobs:
+      - lint
       - unit
-      - integration-postgres-py36:
+      - integration-postgres:
           requires:
             - unit
-      - integration-redshift-py36:
-          requires:
-            - integration-postgres-py36
-      - integration-bigquery-py36:
-          requires:
-            - integration-postgres-py36
-      - integration-snowflake-py36:
-          requires:
-            - integration-postgres-py36
-      - integration-postgres-py38:
+      - integration-redshift:
           requires:
             - unit
-      - integration-redshift-py38:
-          requires:
-            - integration-postgres-py38
-      - integration-bigquery-py38:
-          requires:
-            - integration-postgres-py38
-      - integration-snowflake-py38:
-          requires:
-            - integration-postgres-py38
-      - integration-postgres-py39:
+      - integration-bigquery:
           requires:
             - unit
-      - integration-redshift-py39:
+      - integration-snowflake:
           requires:
-            - integration-postgres-py39
-      - integration-bigquery-py39:
-          requires:
-            - integration-postgres-py39
-      # - integration-snowflake-py39:
-      #     requires:
-      #       - integration-postgres-py39
+            - unit
       - build-wheels:
           requires:
+            - lint
             - unit
-            - integration-postgres-py36
-            - integration-redshift-py36
-            - integration-bigquery-py36
-            - integration-snowflake-py36
-            - integration-postgres-py38
-            - integration-redshift-py38
-            - integration-bigquery-py38
-            - integration-snowflake-py38
-            - integration-postgres-py39
-            - integration-redshift-py39
-            - integration-bigquery-py39
-            # - integration-snowflake-py39
+            - integration-postgres
+            - integration-redshift
+            - integration-bigquery
+            - integration-snowflake

.github/ISSUE_TEMPLATE/minor-version-release.md

@@ -0,0 +1,29 @@
+---
+name: Minor version release
+about: Creates a tracking checklist of items for a minor version release
+title: "[Tracking] v#.##.# release "
+labels: ''
+assignees: ''
+
+---
+
+### Release Core
+- [ ] [Engineering] dbt-release workflow 
+- [ ] [Engineering] Create new protected `x.latest` branch 
+- [ ] [Product] Finalize migration guide (next.docs.getdbt.com)
+
+### Release Cloud
+- [ ] [Engineering] Create a platform issue to update dbt Cloud and verify it is completed
+- [ ] [Engineering] Determine if schemas have changed. If so, generate new schemas and push to schemas.getdbt.com
+
+### Announce
+- [ ] [Product] Publish discourse
+- [ ] [Product] Announce in dbt Slack
+
+### Post-release
+- [ ] [Engineering] [Bump plugin versions](https://www.notion.so/fishtownanalytics/Releasing-b97c5ea9a02949e79e81db3566bbc8ef#59571f5bc1a040d9a8fd096e23d2c7db) (dbt-spark + dbt-presto), add compatibility as needed
+   - [ ]  Spark 
+   - [ ]  Presto
+- [ ] [Engineering] Create a platform issue to update dbt-spark versions to dbt Cloud 
+- [ ] [Product] Release new version of dbt-utils with new dbt version compatibility. If there are breaking changes requiring a minor version, plan upgrades of other packages that depend on dbt-utils.
+- [ ] [Engineering] If this isn't a final release, create an epic for the next release

.github/workflows/performance.yml

@@ -0,0 +1,181 @@
+
+name: Performance Regression Testing
+# Schedule triggers
+on:
+  # TODO this is just while developing
+  pull_request:
+    branches:
+      - 'develop' 
+      - 'performance-regression-testing'
+  schedule:
+    # runs twice a day at 10:05am and 10:05pm
+    - cron:  '5 10,22 * * *'
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+jobs:
+
+  # checks fmt of runner code
+  # purposefully not a dependency of any other job
+  # will block merging, but not prevent developing
+  fmt:
+    name: Cargo fmt
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: stable
+          override: true
+      - run: rustup component add rustfmt
+      - uses: actions-rs/cargo@v1
+        with:
+          command: fmt
+          args: --manifest-path performance/runner/Cargo.toml --all -- --check
+
+  # runs any tests associated with the runner
+  # these tests make sure the runner logic is correct
+  test-runner:
+    name: Test Runner
+    runs-on: ubuntu-latest
+    env:
+      # turns errors into warnings
+      RUSTFLAGS: "-D warnings"
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: stable
+          override: true
+      - uses: actions-rs/cargo@v1
+        with:
+          command: test
+          args: --manifest-path performance/runner/Cargo.toml
+
+  # build an optimized binary to be used as the runner in later steps
+  build-runner:
+    needs: [test-runner]
+    name: Build Runner
+    runs-on: ubuntu-latest
+    env:
+      RUSTFLAGS: "-D warnings"
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: stable
+          override: true
+      - uses: actions-rs/cargo@v1
+        with:
+          command: build
+          args: --release --manifest-path performance/runner/Cargo.toml
+      - uses: actions/upload-artifact@v2
+        with:
+          name: runner
+          path: performance/runner/target/release/runner
+
+  # run the performance measurements on the current or default branch
+  measure-dev:
+    needs: [build-runner]
+    name: Measure Dev Branch
+    runs-on: ubuntu-latest
+    steps:
+      - name: checkout dev
+        uses: actions/checkout@v2
+      - name: Setup Python
+        uses: actions/setup-python@v2.2.2
+        with:
+          python-version: '3.8'
+      - name: install dbt
+        run: pip install -r dev-requirements.txt -r editable-requirements.txt
+      - name: install hyperfine
+        run: wget https://github.com/sharkdp/hyperfine/releases/download/v1.11.0/hyperfine_1.11.0_amd64.deb && sudo dpkg -i hyperfine_1.11.0_amd64.deb
+      - uses: actions/download-artifact@v2
+        with:
+          name: runner
+      - name: change permissions
+        run: chmod +x ./runner
+      - name: run
+        run: ./runner measure -b dev -p ${{ github.workspace }}/performance/projects/
+      - uses: actions/upload-artifact@v2
+        with:
+          name: dev-results
+          path: performance/results/
+
+  # run the performance measurements on the release branch which we use
+  # as a performance baseline. This part takes by far the longest, so
+  # we do everything we can first so the job fails fast.
+  # -----
+  # we need to checkout dbt twice in this job: once for the baseline dbt
+  # version, and once to get the latest regression testing projects,
+  # metrics, and runner code from the develop or current branch so that
+  # the calculations match for both versions of dbt we are comparing.
+  measure-baseline:
+    needs: [build-runner]
+    name: Measure Baseline Branch
+    runs-on: ubuntu-latest
+    steps:
+      - name: checkout latest
+        uses: actions/checkout@v2
+        with:
+          ref: '0.20.latest'
+      - name: Setup Python
+        uses: actions/setup-python@v2.2.2
+        with:
+          python-version: '3.8'
+      - name: move repo up a level
+        run: mkdir ${{ github.workspace }}/../baseline/ && cp -r ${{ github.workspace }} ${{ github.workspace }}/../baseline
+      - name: "[debug] ls new dbt location"
+        run: ls ${{ github.workspace }}/../baseline/dbt/
+      # installation creates egg-links so we have to preserve source
+      - name: install dbt from new location
+        run: cd ${{ github.workspace }}/../baseline/dbt/ && pip install -r dev-requirements.txt -r editable-requirements.txt
+      # checkout the current branch to get all the target projects
+      # this deletes the old checked out code which is why we had to copy before
+      - name: checkout dev
+        uses: actions/checkout@v2
+      - name: install hyperfine
+        run: wget https://github.com/sharkdp/hyperfine/releases/download/v1.11.0/hyperfine_1.11.0_amd64.deb && sudo dpkg -i hyperfine_1.11.0_amd64.deb
+      - uses: actions/download-artifact@v2
+        with:
+          name: runner
+      - name: change permissions
+        run: chmod +x ./runner
+      - name: run runner
+        run: ./runner measure -b baseline -p ${{ github.workspace }}/performance/projects/
+      - uses: actions/upload-artifact@v2
+        with:
+          name: baseline-results
+          path: performance/results/
+
+  # detect regressions on the output generated from measuring
+  # the two branches. Exits with non-zero code if a regression is detected.
+  calculate-regressions:
+    needs: [measure-dev, measure-baseline]
+    name: Compare Results
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/download-artifact@v2
+        with:
+          name: dev-results
+      - uses: actions/download-artifact@v2
+        with:
+          name: baseline-results
+      - name: "[debug] ls result files"
+        run: ls
+      - uses: actions/download-artifact@v2
+        with:
+          name: runner
+      - name: change permissions
+        run: chmod +x ./runner
+      - name: run calculation
+        run: ./runner calculate -r ./
+        # always attempt to upload the results even if there were regressions found
+      - uses: actions/upload-artifact@v2
+        if: ${{ always() }}
+        with:
+          name: final-calculations
+          path: ./final_calculations.json

.github/workflows/tests.yml

@@ -0,0 +1,178 @@
+# This is a workflow to run our unit and integration tests for windows and mac
+
+name: dbt Tests
+
+# Triggers
+on:
+  # Triggers the workflow on push or pull request events and also adds a manual trigger
+  push:
+    branches: 
+      - 'develop' 
+      - '*.latest' 
+      - 'releases/*'
+  pull_request_target:
+    branches: 
+      - 'develop' 
+      - '*.latest' 
+      - 'pr/*'
+      - 'releases/*'
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+jobs:
+  Linting:
+    runs-on: ubuntu-latest #no need to run on every OS
+    steps:
+    - uses: actions/checkout@v2
+    - name: Setup Python
+      uses: actions/setup-python@v2.2.2
+      with:
+        python-version: '3.8'
+        architecture: 'x64'
+
+    - name: 'Install dependencies'
+      run: python -m pip install --upgrade pip && pip install tox
+      
+    - name: 'Linting'
+      run: tox -e mypy,flake8 -- -v
+      
+  UnitTest:
+    strategy:
+      matrix:
+        os: [windows-latest, ubuntu-latest, macos-latest]
+    runs-on: ${{ matrix.os }}
+    steps:
+    - uses: actions/checkout@v2
+    - name: Setup Python
+      uses: actions/setup-python@v2.2.2
+      with:
+        python-version: '3.8'
+        architecture: 'x64'
+
+    - name: 'Install dependencies'
+      run: python -m pip install --upgrade pip && pip install tox
+
+    - name: 'Run unit tests'
+      run: python -m tox -e py -- -v
+      
+  PostgresIntegrationTest:
+    runs-on: 'windows-latest' #TODO: Add Mac support
+    environment: 'Postgres'
+    needs: UnitTest
+    steps:
+    - uses: actions/checkout@v2
+    - name: 'Install postgresql and set up database'
+      shell: pwsh
+      run: |
+          $serviceName = Get-Service -Name postgresql*
+          Set-Service -InputObject $serviceName -StartupType Automatic
+          Start-Service -InputObject $serviceName
+          & $env:PGBIN\createdb.exe -U postgres dbt
+          & $env:PGBIN\psql.exe  -U postgres -c "CREATE ROLE root WITH PASSWORD '$env:ROOT_PASSWORD';"
+          & $env:PGBIN\psql.exe  -U postgres -c "ALTER ROLE root WITH LOGIN;"
+          & $env:PGBIN\psql.exe  -U postgres -c "GRANT CREATE, CONNECT ON DATABASE dbt TO root WITH GRANT OPTION;"
+          & $env:PGBIN\psql.exe  -U postgres -c "CREATE ROLE noaccess WITH PASSWORD '$env:NOACCESS_PASSWORD' NOSUPERUSER;"
+          & $env:PGBIN\psql.exe  -U postgres -c "ALTER ROLE noaccess WITH LOGIN;"
+          & $env:PGBIN\psql.exe  -U postgres -c "GRANT CONNECT ON DATABASE dbt TO noaccess;"
+      env:
+          ROOT_PASSWORD: ${{ secrets.ROOT_PASSWORD }}
+          NOACCESS_PASSWORD: ${{ secrets.NOACCESS_PASSWORD }}
+      
+    - name: Setup Python
+      uses: actions/setup-python@v2.2.2
+      with:
+        python-version: '3.7'
+        architecture: 'x64'
+        
+    - name: 'Install dependencies'
+      run: python -m pip install --upgrade pip && pip install tox
+      
+    - name: 'Run integration tests'
+      run: python -m tox -e py-postgres -- -v -n4
+      
+  # These three are all similar except secure environment variables, which MUST be passed along to their tasks,
+  # but there's probably a better way to do this!
+  SnowflakeIntegrationTest:
+    strategy:
+      matrix:
+        os: [windows-latest, macos-latest]
+    runs-on: ${{ matrix.os }}
+    environment: 'Snowflake'
+    needs: UnitTest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Setup Python
+        uses: actions/setup-python@v2.2.2
+        with:
+          python-version: '3.7'
+          architecture: 'x64'
+
+      - name: 'Install dependencies'
+        run: python -m pip install --upgrade pip && pip install tox
+
+      - name: 'Run integration tests'
+        run: python -m tox -e py-snowflake -- -v -n4
+        env:
+          SNOWFLAKE_TEST_ACCOUNT: ${{ secrets.SNOWFLAKE_TEST_ACCOUNT }}
+          SNOWFLAKE_TEST_PASSWORD: ${{ secrets.SNOWFLAKE_TEST_PASSWORD }}
+          SNOWFLAKE_TEST_USER: ${{ secrets.SNOWFLAKE_TEST_USER }}
+          SNOWFLAKE_TEST_WAREHOUSE: ${{ secrets.SNOWFLAKE_TEST_WAREHOUSE }}
+          SNOWFLAKE_TEST_OAUTH_REFRESH_TOKEN: ${{ secrets.SNOWFLAKE_TEST_OAUTH_REFRESH_TOKEN }}
+          SNOWFLAKE_TEST_OAUTH_CLIENT_ID: ${{ secrets.SNOWFLAKE_TEST_OAUTH_CLIENT_ID }}
+          SNOWFLAKE_TEST_OAUTH_CLIENT_SECRET: ${{ secrets.SNOWFLAKE_TEST_OAUTH_CLIENT_SECRET }}
+          SNOWFLAKE_TEST_ALT_DATABASE: ${{ secrets.SNOWFLAKE_TEST_ALT_DATABASE }}
+          SNOWFLAKE_TEST_ALT_WAREHOUSE: ${{ secrets.SNOWFLAKE_TEST_ALT_WAREHOUSE }}
+          SNOWFLAKE_TEST_DATABASE: ${{ secrets.SNOWFLAKE_TEST_DATABASE }}
+          SNOWFLAKE_TEST_QUOTED_DATABASE: ${{ secrets.SNOWFLAKE_TEST_QUOTED_DATABASE }}
+          SNOWFLAKE_TEST_ROLE: ${{ secrets.SNOWFLAKE_TEST_ROLE }}
+
+  BigQueryIntegrationTest:
+    strategy:
+      matrix:
+        os: [windows-latest, macos-latest]
+    runs-on: ${{ matrix.os }}
+    environment: 'Bigquery'
+    needs: UnitTest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Setup Python
+        uses: actions/setup-python@v2.2.2
+        with:
+          python-version: '3.7'
+          architecture: 'x64'
+        
+      - name: 'Install dependencies'
+        run: python -m pip install --upgrade pip && pip install tox
+      
+      - name: 'Run integration tests'
+        run: python -m tox -e py-bigquery -- -v -n4
+        env:
+          BIGQUERY_SERVICE_ACCOUNT_JSON: ${{ secrets.BIGQUERY_SERVICE_ACCOUNT_JSON }}
+          BIGQUERY_TEST_ALT_DATABASE: ${{ secrets.BIGQUERY_TEST_ALT_DATABASE }}
+
+  RedshiftIntegrationTest:
+    strategy:
+      matrix:
+        os: [windows-latest, macos-latest]
+    runs-on: ${{ matrix.os }}
+    environment: 'Redshift'
+    needs: UnitTest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Setup Python
+        uses: actions/setup-python@v2.2.2
+        with:
+          python-version: '3.7'
+          architecture: 'x64'
+
+      - name: 'Install dependencies'
+        run: python -m pip install --upgrade pip && pip install tox
+
+      - name: 'Run integration tests' 
+        run: python -m tox -e py-redshift -- -v -n4
+        env:
+          REDSHIFT_TEST_DBNAME: ${{ secrets.REDSHIFT_TEST_DBNAME }}
+          REDSHIFT_TEST_PASS: ${{ secrets.REDSHIFT_TEST_PASS }}
+          REDSHIFT_TEST_USER: ${{ secrets.REDSHIFT_TEST_USER }}
+          REDSHIFT_TEST_PORT: ${{ secrets.REDSHIFT_TEST_PORT }}
+          REDSHIFT_TEST_HOST: ${{ secrets.REDSHIFT_TEST_HOST }}

.gitignore

@@ -85,6 +85,7 @@ target/
 
 # pycharm
 .idea/
+venv/
 
 # AWS credentials
 .aws/

CHANGELOG.md

@@ -1,32 +1,236 @@
-## dbt 0.20.0 (Release TBD)
+## dbt 0.21.0 (Release TBD)
+
+### Features
+- Add `dbt build` command to run models, tests, seeds, and snapshots in DAG order. ([#2743] (https://github.com/dbt-labs/dbt/issues/2743), [#3490] (https://github.com/dbt-labs/dbt/issues/3490))
 
 ### Fixes
-- Fix exit code from dbt debug not returning a failure when one of the tests fail ([#3017](https://github.com/fishtown-analytics/dbt/issues/3017))
+- Fix docs generation for cross-db sources in REDSHIFT RA3 node ([#3236](https://github.com/fishtown-analytics/dbt/issues/3236), [#3408](https://github.com/fishtown-analytics/dbt/pull/3408))
+- Fix type coercion issues when fetching query result sets ([#2984](https://github.com/fishtown-analytics/dbt/issues/2984), [#3499](https://github.com/fishtown-analytics/dbt/pull/3499))
+- Handle whitespace after a plus sign on the project config ([#3526](https://github.com/dbt-labs/dbt/pull/3526))
+
+### Under the hood
+- Add performance regression testing [#3602](https://github.com/dbt-labs/dbt/pull/3602)
+- Improve default view and table materialization performance by checking relational cache before attempting to drop temp relations ([#3112](https://github.com/fishtown-analytics/dbt/issues/3112), [#3468](https://github.com/fishtown-analytics/dbt/pull/3468))
+- Add optional `sslcert`, `sslkey`, and `sslrootcert` profile arguments to the Postgres connector. ([#3472](https://github.com/fishtown-analytics/dbt/pull/3472), [#3473](https://github.com/fishtown-analytics/dbt/pull/3473))
+- Move the example project used by `dbt init` into `dbt` repository, to avoid cloning an external repo ([#3005](https://github.com/fishtown-analytics/dbt/pull/3005), [#3474](https://github.com/fishtown-analytics/dbt/pull/3474), [#3536](https://github.com/fishtown-analytics/dbt/pull/3536))
+- Better interaction between `dbt init` and adapters. Avoid raising errors while initializing a project ([#2814](https://github.com/fishtown-analytics/dbt/pull/2814), [#3483](https://github.com/fishtown-analytics/dbt/pull/3483))
+- Update `create_adapter_plugins` script to include latest accessories, and stay up to date with latest dbt-core version ([#3002](https://github.com/fishtown-analytics/dbt/issues/3002), [#3509](https://github.com/fishtown-analytics/dbt/pull/3509))
+
+### Dependencies
+- Require `werkzeug>=1`
+
+Contributors:
+- [@kostek-pl](https://github.com/kostek-pl) ([#3236](https://github.com/fishtown-analytics/dbt/pull/3408))
+- [@tconbeer](https://github.com/tconbeer) [#3468](https://github.com/fishtown-analytics/dbt/pull/3468))
+- [@JLDLaughlin](https://github.com/JLDLaughlin) ([#3473](https://github.com/fishtown-analytics/dbt/pull/3473))
+- [@jmriego](https://github.com/jmriego) ([#3526](https://github.com/dbt-labs/dbt/pull/3526))
+
+
+## dbt 0.20.1 (Release TBD)
+
+### Fixes
+- Fix `store_failures` config when defined as a modifier for `unique` and `not_null` tests ([#3575](https://github.com/fishtown-analytics/dbt/issues/3575), [#3577](https://github.com/fishtown-analytics/dbt/pull/3577))
+
+
+## dbt 0.20.0 (July 12, 2021)
+
+### Fixes
+
+- Avoid slowdown in column-level `persist_docs` on Snowflake, while preserving the error-avoidance from [#3149](https://github.com/fishtown-analytics/dbt/issues/3149) ([#3541](https://github.com/fishtown-analytics/dbt/issues/3541), [#3543](https://github.com/fishtown-analytics/dbt/pull/3543))
+- Partial parsing: handle already deleted nodes when schema block also deleted ([#3516](http://github.com/fishown-analystics/dbt/issues/3516), [#3522](http://github.com/fishown-analystics/dbt/issues/3522))
+
+### Docs
+
+- Update dbt logo and links ([docs#197](https://github.com/fishtown-analytics/dbt-docs/issues/197))
+
+### Under the hood
+
+- Add tracking for experimental parser accuracy ([3503](https://github.com/dbt-labs/dbt/pull/3503), [3553](https://github.com/dbt-labs/dbt/pull/3553))
+
+## dbt 0.20.0rc2 (June 30, 2021)
+
+### Fixes
+
+- Handle quoted values within test configs, such as `where` ([#3458](https://github.com/fishtown-analytics/dbt/issues/3458), [#3459](https://github.com/fishtown-analytics/dbt/pull/3459))
+
+### Docs
+
+- Display `tags` on exposures ([docs#194](https://github.com/fishtown-analytics/dbt-docs/issues/194), [docs#195](https://github.com/fishtown-analytics/dbt-docs/issues/195))
+
+### Under the hood
+
+- Swap experimental parser implementation to use Rust [#3497](https://github.com/fishtown-analytics/dbt/pull/3497)
+- Dispatch the core SQL statement of the new test materialization, to benefit adapter maintainers ([#3465](https://github.com/fishtown-analytics/dbt/pull/3465), [#3461](https://github.com/fishtown-analytics/dbt/pull/3461))
+- Minimal validation of yaml dictionaries prior to partial parsing ([#3246](https://github.com/fishtown-analytics/dbt/issues/3246), [#3460](https://github.com/fishtown-analytics/dbt/pull/3460))
+- Add partial parsing tests and improve partial parsing handling of macros ([#3449](https://github.com/fishtown-analytics/dbt/issues/3449), [#3505](https://github.com/fishtown-analytics/dbt/pull/3505))
+- Update project loading event data to include experimental parser information. ([#3438](https://github.com/fishtown-analytics/dbt/issues/3438), [#3495](https://github.com/fishtown-analytics/dbt/pull/3495))
+
+Contributors:
+- [@swanderz](https://github.com/swanderz) ([#3461](https://github.com/fishtown-analytics/dbt/pull/3461))
+- [@stkbailey](https://github.com/stkbailey) ([docs#195](https://github.com/fishtown-analytics/dbt-docs/issues/195))
+
+## dbt 0.20.0rc1 (June 04, 2021)
+
+
+### Breaking changes
+- Fix adapter.dispatch macro resolution when statically extracting macros. Introduce new project-level `dispatch` config. The `packages` argument to `dispatch` no longer supports macro calls; there is backwards compatibility for existing packages. The argument will no longer be supported in a future release, instead provide the `macro_namespace` argument. ([#3362](https://github.com/fishtown-analytics/dbt/issues/3362), [#3363](https://github.com/fishtown-analytics/dbt/pull/3363), [#3383](https://github.com/fishtown-analytics/dbt/pull/3383), [#3403](https://github.com/fishtown-analytics/dbt/pull/3403))
+
+### Features
+- Support optional `updated_at` config parameter with `check` strategy snapshots. If not supplied, will use current timestamp (default). ([#1844](https://github.com/fishtown-analytics/dbt/issues/1844), [#3376](https://github.com/fishtown-analytics/dbt/pull/3376))
+- Add the opt-in `--use-experimental-parser` flag ([#3307](https://github.com/fishtown-analytics/dbt/issues/3307), [#3374](https://github.com/fishtown-analytics/dbt/issues/3374))
+- Store test failures in the database ([#517](https://github.com/fishtown-analytics/dbt/issues/517), [#903](https://github.com/fishtown-analytics/dbt/issues/903), [#2593](https://github.com/fishtown-analytics/dbt/issues/2593), [#3316](https://github.com/fishtown-analytics/dbt/issues/3316))
+- Add new test configs: `where`, `limit`, `warn_if`, `error_if`, `fail_calc` ([#3258](https://github.com/fishtown-analytics/dbt/issues/3258), [#3321](https://github.com/fishtown-analytics/dbt/issues/3321), [#3336](https://github.com/fishtown-analytics/dbt/pull/3336))
+- Move partial parsing to end of parsing and implement new partial parsing method. ([#3217](https://github.com/fishtown-analytics/dbt/issues/3217), [#3364](https://github.com/fishtown-analytics/dbt/pull/3364))
+- Save doc file node references and use in partial parsing. ([#3425](https://github.com/fishtown-analytics/dbt/issues/3425), [#3432](https://github.com/fishtown-analytics/dbt/pull/3432))
+
+### Fixes
+- Fix compiled sql for ephemeral models ([#3317](https://github.com/fishtown-analytics/dbt/issues/3317), [#3318](https://github.com/fishtown-analytics/dbt/pull/3318))
+- Now generating `run_results.json` even when no nodes are selected ([#3313](https://github.com/fishtown-analytics/dbt/issues/3313), [#3315](https://github.com/fishtown-analytics/dbt/pull/3315))
+- Add missing `packaging` dependency ([#3312](https://github.com/fishtown-analytics/dbt/issues/3312), [#3339](https://github.com/fishtown-analytics/dbt/pull/3339))
+- Fix references to macros with package names when rendering schema tests ([#3324](https://github.com/fishtown-analytics/dbt/issues/3324), [#3345](https://github.com/fishtown-analytics/dbt/pull/3345))
+- Stop clobbering default keyword arguments for jinja test definitions ([#3329](https://github.com/fishtown-analytics/dbt/issues/3329), [#3340](https://github.com/fishtown-analytics/dbt/pull/3340))
+- Fix unique_id generation for generic tests so tests with the same FQN but different configuration will run. ([#3254](https://github.com/fishtown-analytics/dbt/issues/3254), [#3335](https://github.com/fishtown-analytics/dbt/issues/3335))
+- Update the snowflake adapter to only comment on a column if it exists when using the persist_docs config ([#3039](https://github.com/fishtown-analytics/dbt/issues/3039), [#3149](https://github.com/fishtown-analytics/dbt/pull/3149))
+- Add a better error messages for undefined macros and when there are less packages installed than specified in `packages.yml`. ([#2999](https://github.com/fishtown-analytics/dbt/issues/2999))
+- Separate `compiled_path` from `build_path`, and print the former alongside node error messages ([#1985](https://github.com/fishtown-analytics/dbt/issues/1985), [#3327](https://github.com/fishtown-analytics/dbt/pull/3327))
+- Fix exception caused when running `dbt debug` with BigQuery connections ([#3314](https://github.com/fishtown-analytics/dbt/issues/3314), [#3351](https://github.com/fishtown-analytics/dbt/pull/3351))
+- Raise better error if snapshot is missing required configurations ([#3381](https://github.com/fishtown-analytics/dbt/issues/3381), [#3385](https://github.com/fishtown-analytics/dbt/pull/3385))
+- Fix `dbt run` errors caused from receiving non-JSON responses from Snowflake with Oauth ([#3350](https://github.com/fishtown-analytics/dbt/issues/3350))
+- Fix deserialization of Manifest lock attribute ([#3435](https://github.com/fishtown-analytics/dbt/issues/3435), [#3445](https://github.com/fishtown-analytics/dbt/pull/3445))
+- Fix `dbt run` errors caused from receiving non-JSON responses from Snowflake with Oauth ([#3350](https://github.com/fishtown-analytics/dbt/issues/3350)
+- Fix infinite recursion when parsing schema tests due to loops in macro calls ([#3444](https://github.com/fishtown-analytics/dbt/issues/3344), [#3454](https://github.com/fishtown-analytics/dbt/pull/3454))
+
+### Docs
+- Reversed the rendering direction of relationship tests so that the test renders in the model it is defined in ([docs#181](https://github.com/fishtown-analytics/dbt-docs/issues/181), [docs#183](https://github.com/fishtown-analytics/dbt-docs/pull/183))
+- Support dots in model names: display them in the graphs ([docs#184](https://github.com/fishtown-analytics/dbt-docs/issues/184), [docs#185](https://github.com/fishtown-analytics/dbt-docs/issues/185))
+- Render meta tags for sources ([docs#192](https://github.com/fishtown-analytics/dbt-docs/issues/192), [docs#193](https://github.com/fishtown-analytics/dbt-docs/issues/193))
+
+### Under the hood
+- Added logic for registry requests to raise a timeout error after a response hangs out for 30 seconds and 5 attempts have been made to reach the endpoint ([#3177](https://github.com/fishtown-analytics/dbt/issues/3177), [#3275](https://github.com/fishtown-analytics/dbt/pull/3275))
+- Added support for invoking the `list` task via the RPC server ([#3311](https://github.com/fishtown-analytics/dbt/issues/3311), [#3384](https://github.com/fishtown-analytics/dbt/pull/3384))
+- Added `unique_id` and `original_file_path` as keys to json responses from the `list` task ([#3356](https://github.com/fishtown-analytics/dbt/issues/3356), [#3384](https://github.com/fishtown-analytics/dbt/pull/3384))
+- Use shutil.which so Windows can pick up git.bat as a git executable ([#3035](https://github.com/fishtown-analytics/dbt/issues/3035), [#3134](https://github.com/fishtown-analytics/dbt/issues/3134))
+- Add `ssh-client` and update `git` version (using buster backports) in Docker image ([#3337](https://github.com/fishtown-analytics/dbt/issues/3337), [#3338](https://github.com/fishtown-analytics/dbt/pull/3338))
+- Add `tags` and `meta` properties to the exposure resource schema. ([#3404](https://github.com/fishtown-analytics/dbt/issues/3404), [#3405](https://github.com/fishtown-analytics/dbt/pull/3405))
+- Update test sub-query alias ([#3398](https://github.com/fishtown-analytics/dbt/issues/3398), [#3414](https://github.com/fishtown-analytics/dbt/pull/3414)) 
+- Bump schema versions for run results and manifest artifacts ([#3422](https://github.com/fishtown-analytics/dbt/issues/3422), [#3421](https://github.com/fishtown-analytics/dbt/pull/3421))
+- Add deprecation warning for using `packages` argument with `adapter.dispatch` ([#3419](https://github.com/fishtown-analytics/dbt/issues/3419), [#3420](https://github.com/fishtown-analytics/dbt/pull/3420))
+
+Contributors:
+- [@TeddyCr](https://github.com/TeddyCr) ([#3275](https://github.com/fishtown-analytics/dbt/pull/3275))
+- [@panasenco](https://github.com/panasenco) ([#3315](https://github.com/fishtown-analytics/dbt/pull/3315))
+- [@dmateusp](https://github.com/dmateusp) ([#3338](https://github.com/fishtown-analytics/dbt/pull/3338))
+- [@peiwangdb](https://github.com/peiwangdb) ([#3344](https://github.com/fishtown-analytics/dbt/pull/3344))
+- [@elikastelein](https://github.com/elikastelein) ([#3149](https://github.com/fishtown-analytics/dbt/pull/3149))
+- [@majidaldo](https://github.com/majidaldo) ([#3134](https://github.com/fishtown-analytics/dbt/issues/3134))
+- [@jaypeedevlin](https://github.com/jaypeedevlin) ([#2999](https://github.com/fishtown-analytics/dbt/issues/2999))
+- [@PJGaetan](https://github.com/PJGaetan) ([#3315](https://github.com/fishtown-analytics/dbt/pull/3376))
+- [@jnatkins](https://github.com/jnatkins) ([#3385](https://github.com/fishtown-analytics/dbt/pull/3385))
+- [@matt-winkler](https://github.com/matt-winkler) ([#3365](https://github.com/fishtown-analytics/dbt/pull/3365))
+- [@stkbailey](https://github.com/stkbailey) ([#3404](https://github.com/fishtown-analytics/dbt/pull/3405))
+- [@mascah](https://github.com/mascah) ([docs#181](https://github.com/fishtown-analytics/dbt-docs/issues/181), [docs#183](https://github.com/fishtown-analytics/dbt-docs/pull/183))
+- [@monti-python](https://github.com/monti-python) ([docs#184](https://github.com/fishtown-analytics/dbt-docs/issues/184))
+- [@diegodewilde](https://github.com/diegodewilde) ([docs#193](https://github.com/fishtown-analytics/dbt-docs/issues/193))
+
+## dbt 0.20.0b1 (May 03, 2021)
+
+### Breaking changes
+
+- Add Jinja tag for generic test definitions. Replacement for macros prefixed `test_` ([#1173](https://github.com/fishtown-analytics/dbt/issues/1173), [#3261](https://github.com/fishtown-analytics/dbt/pull/3261))
+- Update schema/generic tests to expect a set of rows instead of a single numeric value, and to use test materialization when executing. ([#3192](https://github.com/fishtown-analytics/dbt/issues/3192), [#3286](https://github.com/fishtown-analytics/dbt/pull/3286))
+- **Plugin maintainers:** For adapters that inherit from other adapters (e.g. `dbt-postgres` → `dbt-redshift`), `adapter.dispatch()` will now include parent macro implementations as viable candidates ([#2923](https://github.com/fishtown-analytics/dbt/issues/2923), [#3296](https://github.com/fishtown-analytics/dbt/pull/3296))
+
+### Features
+- Support commit hashes in dbt deps package revision ([#3268](https://github.com/fishtown-analytics/dbt/issues/3268), [#3270](https://github.com/fishtown-analytics/dbt/pull/3270))
+- Add optional `subdirectory` key to install dbt packages that are not hosted at the root of a Git repository ([#275](https://github.com/fishtown-analytics/dbt/issues/275), [#3267](https://github.com/fishtown-analytics/dbt/pull/3267))
+- Add optional configs for `require_partition_filter` and `partition_expiration_days` in BigQuery ([#1843](https://github.com/fishtown-analytics/dbt/issues/1843), [#2928](https://github.com/fishtown-analytics/dbt/pull/2928))
+- Fix for EOL SQL comments prevent entire line execution ([#2731](https://github.com/fishtown-analytics/dbt/issues/2731), [#2974](https://github.com/fishtown-analytics/dbt/pull/2974))
+- Add optional `merge_update_columns` config to specify columns to update for `merge` statements in BigQuery and Snowflake ([#1862](https://github.com/fishtown-analytics/dbt/issues/1862), [#3100](https://github.com/fishtown-analytics/dbt/pull/3100))
+- Use query comment JSON as job labels for BigQuery adapter when `query-comment.job-label` is set to `true` ([#2483](https://github.com/fishtown-analytics/dbt/issues/2483)), ([#3145](https://github.com/fishtown-analytics/dbt/pull/3145))
+- Set application_name for Postgres connections ([#885](https://github.com/fishtown-analytics/dbt/issues/885), [#3182](https://github.com/fishtown-analytics/dbt/pull/3182))
+- Support disabling schema tests, and configuring tests from `dbt_project.yml` ([#3252](https://github.com/fishtown-analytics/dbt/issues/3252),
+[#3253](https://github.com/fishtown-analytics/dbt/issues/3253), [#3257](https://github.com/fishtown-analytics/dbt/pull/3257))
+- Add native support for Postgres index creation ([#804](https://github.com/fishtown-analytics/dbt/issues/804), [3106](https://github.com/fishtown-analytics/dbt/pull/3106))
+- Less greedy test selection: expand to select unselected tests if and only if all parents are selected ([#2891](https://github.com/fishtown-analytics/dbt/issues/2891), [#3235](https://github.com/fishtown-analytics/dbt/pull/3235))
+- Prevent locks in Redshift during full refresh in incremental materialization. ([#2426](https://github.com/fishtown-analytics/dbt/issues/2426), [#2998](https://github.com/fishtown-analytics/dbt/pull/2998))
+
+### Fixes
+- Fix exit code from dbt debug not returning a failure when one of the tests fail ([#3017](https://github.com/fishtown-analytics/dbt/issues/3017), [#3018](https://github.com/fishtown-analytics/dbt/issues/3018))
 - Auto-generated CTEs in tests and ephemeral models have lowercase names to comply with dbt coding conventions ([#3027](https://github.com/fishtown-analytics/dbt/issues/3027), [#3028](https://github.com/fishtown-analytics/dbt/issues/3028))
 - Fix incorrect error message when a selector does not match any node [#3036](https://github.com/fishtown-analytics/dbt/issues/3036))
 - Fix variable `_dbt_max_partition` declaration and initialization for BigQuery incremental models ([#2940](https://github.com/fishtown-analytics/dbt/issues/2940), [#2976](https://github.com/fishtown-analytics/dbt/pull/2976))
 - Moving from 'master' to 'HEAD' default branch in git ([#3057](https://github.com/fishtown-analytics/dbt/issues/3057), [#3104](https://github.com/fishtown-analytics/dbt/issues/3104), [#3117](https://github.com/fishtown-analytics/dbt/issues/3117)))
 - Requirement on `dataclasses` is relaxed to be between `>=0.6,<0.9` allowing dbt to cohabit with other libraries which required higher versions. ([#3150](https://github.com/fishtown-analytics/dbt/issues/3150), [#3151](https://github.com/fishtown-analytics/dbt/pull/3151))
-
-### Features
-- Add optional configs for `require_partition_filter` and `partition_expiration_days` in BigQuery ([#1843](https://github.com/fishtown-analytics/dbt/issues/1843), [#2928](https://github.com/fishtown-analytics/dbt/pull/2928))
-- Fix for EOL SQL comments prevent entire line execution ([#2731](https://github.com/fishtown-analytics/dbt/issues/2731), [#2974](https://github.com/fishtown-analytics/dbt/pull/2974))
+- Add feature to add `_n` alias to same column names in SQL query ([#3147](https://github.com/fishtown-analytics/dbt/issues/3147), [#3158](https://github.com/fishtown-analytics/dbt/pull/3158))
+- Raise a proper error message if dbt parses a macro twice due to macro duplication or misconfiguration. ([#2449](https://github.com/fishtown-analytics/dbt/issues/2449), [#3165](https://github.com/fishtown-analytics/dbt/pull/3165))
+- Fix exposures missing in graph context variable. ([#3241](https://github.com/fishtown-analytics/dbt/issues/3241), [#3243](https://github.com/fishtown-analytics/dbt/issues/3243))
+- Ensure that schema test macros are properly processed ([#3229](https://github.com/fishtown-analytics/dbt/issues/3229), [#3272](https://github.com/fishtown-analytics/dbt/pull/3272))
+- Use absolute path for profiles directory instead of a path relative to the project directory. Note: If a user supplies a relative path to the profiles directory, the value of `args.profiles_dir` will still be absolute. ([#3133](https://github.com/fishtown-analytics/dbt/issues/3133), [#3176](https://github.com/fishtown-analytics/dbt/issues/3176))
+- Fix FQN selector unable to find models whose name contains dots ([#3246](https://github.com/fishtown-analytics/dbt/issues/3246), [#3247](https://github.com/fishtown-analytics/dbt/issues/3247))
 
 ### Under the hood
 - Add dependabot configuration for alerting maintainers about keeping dependencies up to date and secure. ([#3061](https://github.com/fishtown-analytics/dbt/issues/3061), [#3062](https://github.com/fishtown-analytics/dbt/pull/3062))
 - Update script to collect and write json schema for dbt artifacts ([#2870](https://github.com/fishtown-analytics/dbt/issues/2870), [#3065](https://github.com/fishtown-analytics/dbt/pull/3065))
+- Relax Google Cloud dependency pins to major versions. ([#3155](https://github.com/fishtown-analytics/dbt/pull/3156), [#3155](https://github.com/fishtown-analytics/dbt/pull/3156))
+- Bump `snowflake-connector-python` and releated dependencies, support Python 3.9 ([#2985](https://github.com/fishtown-analytics/dbt/issues/2985), [#3148](https://github.com/fishtown-analytics/dbt/pull/3148))
+- General development environment clean up and improve experience running tests locally ([#3194](https://github.com/fishtown-analytics/dbt/issues/3194), [#3204](https://github.com/fishtown-analytics/dbt/pull/3204), [#3228](https://github.com/fishtown-analytics/dbt/pull/3228))
+- Add a new materialization for tests, update data tests to use test materialization when executing. ([#3154](https://github.com/fishtown-analytics/dbt/issues/3154), [#3181](https://github.com/fishtown-analytics/dbt/pull/3181))
+- Switch from externally storing parsing state in ParseResult object to using Manifest ([#3163](http://github.com/fishtown-analytics/dbt/issues/3163), [#3219](https://github.com/fishtown-analytics/dbt/pull/3219))
+- Switch from loading project files in separate parsers to loading in one place([#3244](http://github.com/fishtown-analytics/dbt/issues/3244), [#3248](https://github.com/fishtown-analytics/dbt/pull/3248))
 
 Contributors:
 - [@yu-iskw](https://github.com/yu-iskw) ([#2928](https://github.com/fishtown-analytics/dbt/pull/2928))
-- [@sdebruyn](https://github.com/sdebruyn) / [@lynxcare](https://github.com/lynxcare) ([#3018](https://github.com/fishtown-analytics/dbt/pull/3018))
+- [@sdebruyn](https://github.com/sdebruyn) ([#3018](https://github.com/fishtown-analytics/dbt/pull/3018))
 - [@rvacaru](https://github.com/rvacaru) ([#2974](https://github.com/fishtown-analytics/dbt/pull/2974))
 - [@NiallRees](https://github.com/NiallRees) ([#3028](https://github.com/fishtown-analytics/dbt/pull/3028))
-- [ran-eh](https://github.com/ran-eh) ([#3036](https://github.com/fishtown-analytics/dbt/pull/3036))
+- [@ran-eh](https://github.com/ran-eh) ([#3036](https://github.com/fishtown-analytics/dbt/pull/3036))
 - [@pcasteran](https://github.com/pcasteran) ([#2976](https://github.com/fishtown-analytics/dbt/pull/2976))
 - [@VasiliiSurov](https://github.com/VasiliiSurov) ([#3104](https://github.com/fishtown-analytics/dbt/pull/3104))
+- [@jmcarp](https://github.com/jmcarp) ([#3145](https://github.com/fishtown-analytics/dbt/pull/3145))
 - [@bastienboutonnet](https://github.com/bastienboutonnet) ([#3151](https://github.com/fishtown-analytics/dbt/pull/3151))
+- [@max-sixty](https://github.com/max-sixty) ([#3156](https://github.com/fishtown-analytics/dbt/pull/3156)
+- [@prratek](https://github.com/prratek) ([#3100](https://github.com/fishtown-analytics/dbt/pull/3100))
+- [@techytushar](https://github.com/techytushar) ([#3158](https://github.com/fishtown-analytics/dbt/pull/3158))
+- [@cgopalan](https://github.com/cgopalan) ([#3165](https://github.com/fishtown-analytics/dbt/pull/3165), [#3182](https://github.com/fishtown-analytics/dbt/pull/3182))
+- [@fux](https://github.com/fuchsst) ([#3243](https://github.com/fishtown-analytics/dbt/issues/3243))
+- [@arzavj](https://github.com/arzavj) ([3106](https://github.com/fishtown-analytics/dbt/pull/3106))
+- [@JCZuurmond](https://github.com/JCZuurmond) ([#3176](https://github.com/fishtown-analytics/dbt/pull/3176))
+- [@dmateusp](https://github.com/dmateusp) ([#3270](https://github.com/fishtown-analytics/dbt/pull/3270), [#3267](https://github.com/fishtown-analytics/dbt/pull/3267))
+- [@monti-python](https://github.com/monti-python) ([#3247](https://github.com/fishtown-analytics/dbt/issues/3247))
+- [@drkarthi](https://github.com/drkarthi) ([#2426](https://github.com/fishtown-analytics/dbt/issues/2426), [#2998](https://github.com/fishtown-analytics/dbt/pull/2998))
 
-## dbt 0.19.1 (Release TBD)
+## dbt 0.19.2 (Release TBD)
+
+### Breaking changes
+- Fix adapter.dispatch macro resolution when statically extracting macros. Introduce new project-level `dispatch` config. The `packages` argument to `dispatch` no longer supports macro calls; there is backwards compatibility for existing packages. The argument will no longer be supported in a future release, instead provide the `macro_namespace` argument. ([#3362](https://github.com/fishtown-analytics/dbt/issues/3362), [#3363](https://github.com/fishtown-analytics/dbt/pull/3363), [#3383](https://github.com/fishtown-analytics/dbt/pull/3383), [#3403](https://github.com/fishtown-analytics/dbt/pull/3403))
+
+### Fixes
+- Fix references to macros with package names when rendering schema tests ([#3324](https://github.com/fishtown-analytics/dbt/issues/3324), [#3345](https://github.com/fishtown-analytics/dbt/pull/3345))
+
+## dbt 0.19.1 (March 31, 2021)
+
+## dbt 0.19.1rc2 (March 25, 2021)
+
+### Fixes
+- Pass service-account scopes to gcloud-based oauth ([#3040](https://github.com/fishtown-analytics/dbt/issues/3040), [#3041](https://github.com/fishtown-analytics/dbt/pull/3041))
+
+Contributors:
+- [@yu-iskw](https://github.com/yu-iskw) ([#3041](https://github.com/fishtown-analytics/dbt/pull/3041))
+
+## dbt 0.19.1rc1 (March 15, 2021)
+
+### Under the hood
+- Update code to use Mashumaro 2.0 ([#3138](https://github.com/fishtown-analytics/dbt/pull/3138))
+- Pin `agate<1.6.2` to avoid installation errors relating to its new dependency `PyICU` ([#3160](https://github.com/fishtown-analytics/dbt/issues/3160), [#3161](https://github.com/fishtown-analytics/dbt/pull/3161))
+- Add an event to track resource counts ([#3050](https://github.com/fishtown-analytics/dbt/issues/3050), [#3157](https://github.com/fishtown-analytics/dbt/pull/3157))
+
+### Fixes
+
+- Fix compiled sql for ephemeral models ([#3139](https://github.com/fishtown-analytics/dbt/pull/3139), [#3056](https://github.com/fishtown-analytics/dbt/pull/3056))
+
+## dbt 0.19.1b2 (February 15, 2021)
+
+## dbt 0.19.1b1 (February 12, 2021)
 
 ### Fixes
 
@@ -35,13 +239,11 @@ Contributors:
 ### Under the hood
 - Bump werkzeug upper bound dependency to `<v2.0` ([#3011](https://github.com/fishtown-analytics/dbt/pull/3011))
 - Performance fixes for many different things ([#2862](https://github.com/fishtown-analytics/dbt/issues/2862), [#3034](https://github.com/fishtown-analytics/dbt/pull/3034))
-- Update code to use Mashumaro 2.0 ([#3138](https://github.com/fishtown-analytics/dbt/pull/3138))
-- Add an event to track resource counts ([#3050](https://github.com/fishtown-analytics/dbt/issues/3050), [#3156](https://github.com/fishtown-analytics/dbt/pull/3156))
-- Pin `agate<1.6.2` to avoid installation errors relating to its new dependency `PyICU` ([#3160](https://github.com/fishtown-analytics/dbt/issues/3160), [#3161](https://github.com/fishtown-analytics/dbt/pull/3161))
 
 Contributors:
 - [@Bl3f](https://github.com/Bl3f) ([#3011](https://github.com/fishtown-analytics/dbt/pull/3011))
 
+
 ## dbt 0.19.0 (January 27, 2021)
 
 ## dbt 0.19.0rc3 (January 27, 2021)
@@ -156,6 +358,15 @@ Contributors:
 - [@Mr-Nobody99](https://github.com/Mr-Nobody99) ([docs#138](https://github.com/fishtown-analytics/dbt-docs/pull/138))
 - [@jplynch77](https://github.com/jplynch77) ([docs#139](https://github.com/fishtown-analytics/dbt-docs/pull/139))
 
+## dbt 0.18.2 (March 22, 2021)
+
+## dbt 0.18.2rc1 (March 12, 2021)
+
+### Under the hood
+- Pin `agate<1.6.2` to avoid installation errors relating to its new dependency
+  `PyICU` ([#3160](https://github.com/fishtown-analytics/dbt/issues/3160),
+  [#3161](https://github.com/fishtown-analytics/dbt/pull/3161))
+
 ## dbt 0.18.1 (October 13, 2020)
 
 ## dbt 0.18.1rc1 (October 01, 2020)

CONTRIBUTING.md

@@ -1,86 +1,86 @@
-# Contributing to dbt
+# Contributing to `dbt`
 
 1. [About this document](#about-this-document)
 2. [Proposing a change](#proposing-a-change)
 3. [Getting the code](#getting-the-code)
 4. [Setting up an environment](#setting-up-an-environment)
-5. [Running dbt in development](#running-dbt-in-development)
+5. [Running `dbt` in development](#running-dbt-in-development)
 6. [Testing](#testing)
 7. [Submitting a Pull Request](#submitting-a-pull-request)
 
 ## About this document
 
-This document is a guide intended for folks interested in contributing to dbt. Below, we document the process by which members of the community should create issues and submit pull requests (PRs) in this repository. It is not intended as a guide for using dbt, and it assumes a certain level of familiarity with Python concepts such as virtualenvs, `pip`, python modules, filesystems, and so on. This guide assumes you are using macOS or Linux and are comfortable with the command line.
+This document is a guide intended for folks interested in contributing to `dbt`. Below, we document the process by which members of the community should create issues and submit pull requests (PRs) in this repository. It is not intended as a guide for using `dbt`, and it assumes a certain level of familiarity with Python concepts such as virtualenvs, `pip`, python modules, filesystems, and so on. This guide assumes you are using macOS or Linux and are comfortable with the command line.
 
-If you're new to python development or contributing to open-source software, we encourage you to read this document from start to finish. If you get stuck, drop us a line in the #development channel on [slack](community.getdbt.com).
+If you're new to python development or contributing to open-source software, we encourage you to read this document from start to finish. If you get stuck, drop us a line in the `#dbt-core-development` channel on [slack](https://community.getdbt.com).
 
 ### Signing the CLA
 
-Please note that all contributors to dbt must sign the [Contributor License Agreement](https://docs.getdbt.com/docs/contributor-license-agreements) to have their Pull Request merged into the dbt codebase. If you are unable to sign the CLA, then the dbt maintainers will unfortunately be unable to merge your Pull Request. You are, however, welcome to open issues and comment on existing ones.
+Please note that all contributors to `dbt` must sign the [Contributor License Agreement](https://docs.getdbt.com/docs/contributor-license-agreements) to have their Pull Request merged into the `dbt` codebase. If you are unable to sign the CLA, then the `dbt` maintainers will unfortunately be unable to merge your Pull Request. You are, however, welcome to open issues and comment on existing ones.
 
 ## Proposing a change
 
-dbt is Apache 2.0-licensed open source software. dbt is what it is today because community members like you have opened issues, provided feedback, and contributed to the knowledge loop for the entire communtiy. Whether you are a seasoned open source contributor or a first-time committer, we welcome and encourage you to contribute code, documentation, ideas, or problem statements to this project.
+`dbt` is Apache 2.0-licensed open source software. `dbt` is what it is today because community members like you have opened issues, provided feedback, and contributed to the knowledge loop for the entire communtiy. Whether you are a seasoned open source contributor or a first-time committer, we welcome and encourage you to contribute code, documentation, ideas, or problem statements to this project.
 
 ### Defining the problem
 
-If you have an idea for a new feature or if you've discovered a bug in dbt, the first step is to open an issue. Please check the list of [open issues](https://github.com/fishtown-analytics/dbt/issues) before creating a new one. If you find a relevant issue, please add a comment to the open issue instead of creating a new one. There are hundreds of open issues in this repository and it can be hard to know where to look for a relevant open issue. **The dbt maintainers are always happy to point contributors in the right direction**, so please err on the side of documenting your idea in a new issue if you are unsure where a problem statement belongs.
+If you have an idea for a new feature or if you've discovered a bug in `dbt`, the first step is to open an issue. Please check the list of [open issues](https://github.com/fishtown-analytics/dbt/issues) before creating a new one. If you find a relevant issue, please add a comment to the open issue instead of creating a new one. There are hundreds of open issues in this repository and it can be hard to know where to look for a relevant open issue. **The `dbt` maintainers are always happy to point contributors in the right direction**, so please err on the side of documenting your idea in a new issue if you are unsure where a problem statement belongs.
 
-**Note:** All community-contributed Pull Requests _must_ be associated with an open issue. If you submit a Pull Request that does not pertain to an open issue, you will be asked to create an issue describing the problem before the Pull Request can be reviewed.
+> **Note:** All community-contributed Pull Requests _must_ be associated with an open issue. If you submit a Pull Request that does not pertain to an open issue, you will be asked to create an issue describing the problem before the Pull Request can be reviewed.
 
 ### Discussing the idea
 
-After you open an issue, a dbt maintainer will follow up by commenting on your issue (usually within 1-3 days) to explore your idea further and advise on how to implement the suggested changes. In many cases, community members will chime in with their own thoughts on the problem statement. If you as the issue creator are interested in submitting a Pull Request to address the issue, you should indicate this in the body of the issue. The dbt maintainers are _always_ happy to help contributors with the implementation of fixes and features, so please also indicate if there's anything you're unsure about or could use guidance around in the issue.
+After you open an issue, a `dbt` maintainer will follow up by commenting on your issue (usually within 1-3 days) to explore your idea further and advise on how to implement the suggested changes. In many cases, community members will chime in with their own thoughts on the problem statement. If you as the issue creator are interested in submitting a Pull Request to address the issue, you should indicate this in the body of the issue. The `dbt` maintainers are _always_ happy to help contributors with the implementation of fixes and features, so please also indicate if there's anything you're unsure about or could use guidance around in the issue.
 
 ### Submitting a change
 
-If an issue is appropriately well scoped and describes a beneficial change to the dbt codebase, then anyone may submit a Pull Request to implement the functionality described in the issue. See the sections below on how to do this.
+If an issue is appropriately well scoped and describes a beneficial change to the `dbt` codebase, then anyone may submit a Pull Request to implement the functionality described in the issue. See the sections below on how to do this.
 
-The dbt maintainers will add a `good first issue` label if an issue is suitable for a first-time contributor. This label often means that the required code change is small, limited to one database adapter, or a net-new addition that does not impact existing functionality. You can see the list of currently open issues on the [Contribute](https://github.com/fishtown-analytics/dbt/contribute) page.
+The `dbt` maintainers will add a `good first issue` label if an issue is suitable for a first-time contributor. This label often means that the required code change is small, limited to one database adapter, or a net-new addition that does not impact existing functionality. You can see the list of currently open issues on the [Contribute](https://github.com/fishtown-analytics/dbt/contribute) page.
 
 Here's a good workflow:
 - Comment on the open issue, expressing your interest in contributing the required code change
 - Outline your planned implementation. If you want help getting started, ask!
-- Follow the steps outlined below to develop locally. Once you have opened a PR, one of the dbt maintainers will work with you to review your code.
-- Add a test! Tests are crucial for both fixes and new features alike. We want to make sure that code works as intended, and that it avoids any bugs  previously encountered. Currently, the best resource for understanding dbt's [unit](test/unit) and [integration](test/integration) tests is the tests themselves. One of the maintainers can help by pointing out relevant examples.
+- Follow the steps outlined below to develop locally. Once you have opened a PR, one of the `dbt` maintainers will work with you to review your code.
+- Add a test! Tests are crucial for both fixes and new features alike. We want to make sure that code works as intended, and that it avoids any bugs  previously encountered. Currently, the best resource for understanding `dbt`'s [unit](test/unit) and [integration](test/integration) tests is the tests themselves. One of the maintainers can help by pointing out relevant examples.
 
-In some cases, the right resolution to an open issue might be tangential to the dbt codebase. The right path forward might be a documentation update or a change that can be made in user-space. In other cases, the issue might describe functionality that the dbt maintainers are unwilling or unable to incorporate into the dbt codebase. When it is determined that an open issue describes functionality that will not translate to a code change in the dbt repository, the issue will be tagged with the `wontfix` label (see below) and closed.
+In some cases, the right resolution to an open issue might be tangential to the `dbt` codebase. The right path forward might be a documentation update or a change that can be made in user-space. In other cases, the issue might describe functionality that the `dbt` maintainers are unwilling or unable to incorporate into the `dbt` codebase. When it is determined that an open issue describes functionality that will not translate to a code change in the `dbt` repository, the issue will be tagged with the `wontfix` label (see below) and closed.
 
 ### Using issue labels
 
-The dbt maintainers use labels to categorize open issues. Some labels indicate the databases impacted by the issue, while others describe the domain in the dbt codebase germane to the discussion. While most of these labels are self-explanatory (eg. `snowflake` or `bigquery`), there are others that are worth describing.
+The `dbt` maintainers use labels to categorize open issues. Some labels indicate the databases impacted by the issue, while others describe the domain in the `dbt` codebase germane to the discussion. While most of these labels are self-explanatory (eg. `snowflake` or `bigquery`), there are others that are worth describing.
 
 | tag | description |
 | --- | ----------- |
-| [triage](https://github.com/fishtown-analytics/dbt/labels/triage) | This is a new issue which has not yet been reviewed by a dbt maintainer. This label is removed when a maintainer reviews and responds to the issue. |
-| [bug](https://github.com/fishtown-analytics/dbt/labels/bug) | This issue represents a defect or regression in dbt |
-| [enhancement](https://github.com/fishtown-analytics/dbt/labels/enhancement) | This issue represents net-new functionality in dbt |
-| [good first issue](https://github.com/fishtown-analytics/dbt/labels/good%20first%20issue) | This issue does not require deep knowledge of the dbt codebase to implement. This issue is appropriate for a first-time contributor. |
-| [help wanted](https://github.com/fishtown-analytics/dbt/labels/help%20wanted) / [discussion](https://github.com/fishtown-analytics/dbt/labels/discussion) | Conversation around this issue in ongoing, and there isn't yet a clear path forward. Input from community members is most welcome. |
-| [duplicate](https://github.com/fishtown-analytics/dbt/issues/duplicate) | This issue is functionally identical to another open issue. The dbt maintainers will close this issue and encourage community members to focus conversation on the other one. |
-| [snoozed](https://github.com/fishtown-analytics/dbt/labels/snoozed) | This issue describes a good idea, but one which will probably not be addressed in a six-month time horizon. The dbt maintainers will revist these issues periodically and re-prioritize them accordingly. |
-| [stale](https://github.com/fishtown-analytics/dbt/labels/stale) | This is an old issue which has not recently been updated. Stale issues will periodically be closed by dbt maintainers, but they can be re-opened if the discussion is restarted. |
-| [wontfix](https://github.com/fishtown-analytics/dbt/labels/wontfix) | This issue does not require a code change in the dbt repository, or the maintainers are unwilling/unable to merge a Pull Request which implements the behavior described in the issue. |
+| [triage](https://github.com/fishtown-analytics/dbt/labels/triage) | This is a new issue which has not yet been reviewed by a `dbt` maintainer. This label is removed when a maintainer reviews and responds to the issue. |
+| [bug](https://github.com/fishtown-analytics/dbt/labels/bug) | This issue represents a defect or regression in `dbt` |
+| [enhancement](https://github.com/fishtown-analytics/dbt/labels/enhancement) | This issue represents net-new functionality in `dbt` |
+| [good first issue](https://github.com/fishtown-analytics/dbt/labels/good%20first%20issue) | This issue does not require deep knowledge of the `dbt` codebase to implement. This issue is appropriate for a first-time contributor. |
+| [help wanted](https://github.com/fishtown-analytics/`dbt`/labels/help%20wanted) / [discussion](https://github.com/fishtown-analytics/dbt/labels/discussion) | Conversation around this issue in ongoing, and there isn't yet a clear path forward. Input from community members is most welcome. |
+| [duplicate](https://github.com/fishtown-analytics/dbt/issues/duplicate) | This issue is functionally identical to another open issue. The `dbt` maintainers will close this issue and encourage community members to focus conversation on the other one. |
+| [snoozed](https://github.com/fishtown-analytics/dbt/labels/snoozed) | This issue describes a good idea, but one which will probably not be addressed in a six-month time horizon. The `dbt` maintainers will revist these issues periodically and re-prioritize them accordingly. |
+| [stale](https://github.com/fishtown-analytics/dbt/labels/stale) | This is an old issue which has not recently been updated. Stale issues will periodically be closed by `dbt` maintainers, but they can be re-opened if the discussion is restarted. |
+| [wontfix](https://github.com/fishtown-analytics/dbt/labels/wontfix) | This issue does not require a code change in the `dbt` repository, or the maintainers are unwilling/unable to merge a Pull Request which implements the behavior described in the issue. |
 
 #### Branching Strategy
 
-dbt has three types of branches:
+`dbt` has three types of branches:
 
 - **Trunks** are where active development of the next release takes place. There is one trunk named `develop` at the time of writing this, and will be the default branch of the repository.
-- **Release Branches** track a specific, not yet complete release of dbt. Each minor version release has a corresponding release branch. For example, the `0.11.x` series of releases has a branch called `0.11.latest`. This allows us to release new patch versions under `0.11` without necessarily needing to pull them into the latest version of dbt.
+- **Release Branches** track a specific, not yet complete release of `dbt`. Each minor version release has a corresponding release branch. For example, the `0.11.x` series of releases has a branch called `0.11.latest`. This allows us to release new patch versions under `0.11` without necessarily needing to pull them into the latest version of `dbt`.
 - **Feature Branches** track individual features and fixes. On completion they should be merged into the trunk brnach or a specific release branch.
 
 ## Getting the code
 
 ### Installing git
 
-You will need `git` in order to download and modify the dbt source code. On macOS, the best way to download git is to just install [Xcode](https://developer.apple.com/support/xcode/).
+You will need `git` in order to download and modify the `dbt` source code. On macOS, the best way to download git is to just install [Xcode](https://developer.apple.com/support/xcode/).
 
 ### External contributors
 
-If you are not a member of the `fishtown-analytics` GitHub organization, you can contribute to dbt by forking the dbt repository. For a detailed overview on forking, check out the [GitHub docs on forking](https://help.github.com/en/articles/fork-a-repo). In short, you will need to:
+If you are not a member of the `fishtown-analytics` GitHub organization, you can contribute to `dbt` by forking the `dbt` repository. For a detailed overview on forking, check out the [GitHub docs on forking](https://help.github.com/en/articles/fork-a-repo). In short, you will need to:
 
-1. fork the dbt repository
+1. fork the `dbt` repository
 2. clone your fork locally
 3. check out a new branch for your proposed changes
 4. push changes to your fork
@@ -88,31 +88,30 @@ If you are not a member of the `fishtown-analytics` GitHub organization, you can
 
 ### Core contributors
 
-If you are a member of the `fishtown-analytics` GitHub organization, you will have push access to the dbt repo. Rather than 
-forking dbt to make your changes, just clone the repository, check out a new branch, and push directly to that branch.
+If you are a member of the `fishtown-analytics` GitHub organization, you will have push access to the `dbt` repo. Rather than forking `dbt` to make your changes, just clone the repository, check out a new branch, and push directly to that branch.
 
 ## Setting up an environment
 
-There are some tools that will be helpful to you in developing locally. While this is the list relevant for dbt development, many of these tools are used commonly across open-source python projects.
+There are some tools that will be helpful to you in developing locally. While this is the list relevant for `dbt` development, many of these tools are used commonly across open-source python projects.
 
 ### Tools
 
-A short list of tools used in dbt testing that will be helpful to your understanding:
+A short list of tools used in `dbt` testing that will be helpful to your understanding:
 
-- [virtualenv](https://virtualenv.pypa.io/en/stable/) to manage dependencies
-- [tox](https://tox.readthedocs.io/en/latest/) to manage virtualenvs across python versions
-- [pytest](https://docs.pytest.org/en/latest/) to discover/run tests
-- [make](https://users.cs.duke.edu/~ola/courses/programming/Makefiles/Makefiles.html) - but don't worry too much, nobody _really_ understands how make works and our Makefile is super simple
-- [flake8](https://gitlab.com/pycqa/flake8) for code linting
+- [`tox`](https://tox.readthedocs.io/en/latest/) to manage virtualenvs across python versions. We currently target the latest patch releases for Python 3.6, Python 3.7, Python 3.8, and Python 3.9
+- [`pytest`](https://docs.pytest.org/en/latest/) to discover/run tests
+- [`make`](https://users.cs.duke.edu/~ola/courses/programming/Makefiles/Makefiles.html) - but don't worry too much, nobody _really_ understands how make works and our Makefile is super simple
+- [`flake8`](https://flake8.pycqa.org/en/latest/) for code linting
+- [`mypy`](https://mypy.readthedocs.io/en/stable/) for static type checking
 - [CircleCI](https://circleci.com/product/) and [Azure Pipelines](https://azure.microsoft.com/en-us/services/devops/pipelines/)
 
-A deep understanding of these tools in not required to effectively contribute to dbt, but we recommend checking out the attached documentation if you're interested in learning more about them.
+A deep understanding of these tools in not required to effectively contribute to `dbt`, but we recommend checking out the attached documentation if you're interested in learning more about them.
 
 #### virtual environments
 
-We strongly recommend using virtual environments when developing code in dbt. We recommend creating this virtualenv
-in the root of the dbt repository. To create a new virtualenv, run:
-```
+We strongly recommend using virtual environments when developing code in `dbt`. We recommend creating this virtualenv
+in the root of the `dbt` repository. To create a new virtualenv, run:
+```sh
 python3 -m venv env
 source env/bin/activate
 ```
@@ -128,23 +127,25 @@ Docker and docker-compose are both used in testing. Specific instructions for yo
 
 For testing, and later in the examples in this document, you may want to have `psql` available so you can poke around in the database and see what happened. We recommend that you use [homebrew](https://brew.sh/) for that on macOS, and your package manager on Linux. You can install any version of the postgres client that you'd like. On macOS, with homebrew setup, you can run:
 
-```
+```sh
 brew install postgresql
 ```
 
-## Running dbt in development
+## Running `dbt` in development
 
 ### Installation
 
-First make sure that you set up your `virtualenv` as described in section _Setting up an environment_. Next, install dbt (and its dependencies) with:
+First make sure that you set up your `virtualenv` as described in [Setting up an environment](#setting-up-an-environment). Next, install `dbt` (and its dependencies) with:
 
-```
-pip install -r editable_requirements.txt
+```sh
+make dev
+# or
+pip install -r dev-requirements.txt -r editable-requirements.txt
 ```
 
-When dbt is installed from source in this way, any changes you make to the dbt source code will be reflected immediately in your next `dbt` run.
+When `dbt` is installed this way, any changes you make to the `dbt` source code will be reflected immediately in your next `dbt` run.
 
-### Running dbt
+### Running `dbt`
 
 With your virtualenv activated, the `dbt` script should point back to the source code you've cloned on your machine. You can verify this by running `which dbt`. This command should show you a path to an executable in your virtualenv.
 
@@ -152,76 +153,79 @@ Configure your [profile](https://docs.getdbt.com/docs/configure-your-profile) as
 
 ## Testing
 
-Getting the dbt integration tests set up in your local environment will be very helpful as you start to make changes to your local version of dbt. The section that follows outlines some helpful tips for setting up the test environment.
+Getting the `dbt` integration tests set up in your local environment will be very helpful as you start to make changes to your local version of `dbt`. The section that follows outlines some helpful tips for setting up the test environment.
 
-### Running tests via Docker
+Since `dbt` works with a number of different databases, you will need to supply credentials for one or more of these databases in your test environment. Most organizations don't have access to each of a BigQuery, Redshift, Snowflake, and Postgres database, so it's likely that you will be unable to run every integration test locally. Fortunately, Fishtown Analytics provides a CI environment with access to sandboxed Redshift, Snowflake, BigQuery, and Postgres databases. See the section on [_Submitting a Pull Request_](#submitting-a-pull-request) below for more information on this CI setup.
 
-dbt's unit and integration tests run in Docker. Because dbt works with a number of different databases, you will need to supply credentials for one or more of these databases in your test environment. Most organizations don't have access to each of a BigQuery, Redshift, Snowflake, and Postgres database, so it's likely that you will be unable to run every integration test locally. Fortunately, Fishtown Analytics provides a CI environment with access to sandboxed Redshift, Snowflake, BigQuery, and Postgres databases. See the section on [_Submitting a Pull Request_](#submitting-a-pull-request) below for more information on this CI setup.
+### Initial setup
 
+We recommend starting with `dbt`'s Postgres tests. These tests cover most of the functionality in `dbt`, are the fastest to run, and are the easiest to set up. To run the Postgres integration tests, you'll have to do one extra step of setting up the test database:
 
-### Specifying your test credentials
-
-dbt uses test credentials specified in a `test.env` file in the root of the repository. This `test.env` file is git-ignored, but please be _extra_ careful to never check in credentials or other sensitive information when developing against dbt. To create your `test.env` file, copy the provided sample file, then supply your relevant credentials:
-
-```
-cp test.env.sample test.env
-```
-
-We recommend starting with dbt's Postgres tests. These tests cover most of the functionality in dbt, are the fastest to run, and are the easiest to set up. dbt's test suite runs Postgres in a Docker container, so no setup should be required to run these tests.
-
-If you additionally want to test Snowflake, Bigquery, or Redshift, locally you'll need to get credentials and add them to the `test.env` file. In general, it's most important to have successful unit and Postgres tests. Once you open a PR, dbt will automatically run integration tests for the other three core database adapters. Of course, if you are a BigQuery user, contributing a BigQuery-only feature, it's important to run BigQuery tests as well.
-
-### Test commands
-
-dbt's unit tests and Python linter can be run with:
-
-```
-make test-unit
-```
-
-To run the Postgres + Python 3.6 integration tests, you'll have to do one extra step of setting up the test database:
-
+```sh
+make setup-db
 ```
+or, alternatively:
+```sh
 docker-compose up -d database
 PGHOST=localhost PGUSER=root PGPASSWORD=password PGDATABASE=postgres bash test/setup_db.sh
 ```
 
-To run a quick test for Python3 integration tests on Postgres, you can run:
+`dbt` uses test credentials specified in a `test.env` file in the root of the repository for non-Postgres databases. This `test.env` file is git-ignored, but please be _extra_ careful to never check in credentials or other sensitive information when developing against `dbt`. To create your `test.env` file, copy the provided sample file, then supply your relevant credentials. This step is only required to use non-Postgres databases.
 
 ```
-make test-quick
+cp test.env.sample test.env
+$EDITOR test.env
 ```
 
-To run tests for a specific database, invoke `tox` directly with the required flags:
-```
-# Run Postgres py36 tests
-docker-compose run test tox -e integration-postgres-py36 -- -x
+> In general, it's most important to have successful unit and Postgres tests. Once you open a PR, `dbt` will automatically run integration tests for the other three core database adapters. Of course, if you are a BigQuery user, contributing a BigQuery-only feature, it's important to run BigQuery tests as well.
 
-# Run Snowflake py36 tests
-docker-compose run test tox -e integration-snowflake-py36 -- -x
+### Test commands
 
-# Run BigQuery py36 tests
-docker-compose run test tox -e integration-bigquery-py36 -- -x
+There are a few methods for running tests locally.
 
-# Run Redshift py36 tests
-docker-compose run test tox -e integration-redshift-py36 -- -x
-```
+#### Makefile
 
-To run a specific test by itself:
-```
-docker-compose run test tox -e explicit-py36 -- -s -x -m profile_{adapter} {path_to_test_file_or_folder}
-```
-E.g.
-```
-docker-compose run test tox -e explicit-py36 -- -s -x -m profile_snowflake test/integration/001_simple_copy_test
-```
+There are multiple targets in the Makefile to run common test suites and code
+checks, most notably:
 
-See the `Makefile` contents for more some other examples of ways to run `tox`.
+```sh
+# Runs unit tests with py38 and code checks in parallel.
+make test
+# Runs postgres integration tests with py38 in "fail fast" mode.
+make integration
+```
+> These make targets assume you have a recent version of [`tox`](https://tox.readthedocs.io/en/latest/) installed locally,
+> unless you use choose a Docker container to run tests. Run `make help` for more info.
 
+Check out the other targets in the Makefile to see other commonly used test
+suites.
+
+#### `tox`
+
+[`tox`](https://tox.readthedocs.io/en/latest/) takes care of managing virtualenvs and install dependencies in order to run
+tests. You can also run tests in parallel, for example, you can run unit tests
+for Python 3.6, Python 3.7, Python 3.8, `flake8` checks, and `mypy` checks in
+parallel with `tox -p`. Also, you can run unit tests for specific python versions
+with `tox -e py36`. The configuration for these tests in located in `tox.ini`.
+
+#### `pytest`
+
+Finally, you can also run a specific test or group of tests using [`pytest`](https://docs.pytest.org/en/latest/) directly. With a virtualenv
+active and dev dependencies installed you can do things like:
+```sh
+# run specific postgres integration tests
+python -m pytest -m profile_postgres test/integration/001_simple_copy_test
+# run all unit tests in a file
+python -m pytest test/unit/test_graph.py
+# run a specific unit test
+python -m pytest test/unit/test_graph.py::GraphTest::test__dependency_list
+```
+> [Here](https://docs.pytest.org/en/reorganize-docs/new-docs/user/commandlineuseful.html)
+> is a list of useful command-line options for `pytest` to use while developing.
 ## Submitting a Pull Request
 
 Fishtown Analytics provides a sandboxed Redshift, Snowflake, and BigQuery database for use in a CI environment. When pull requests are submitted to the `fishtown-analytics/dbt` repo, GitHub will trigger automated tests in CircleCI and Azure Pipelines.
 
-A dbt maintainer will review your PR. They may suggest code revision for style or clarity, or request that you add unit or integration test(s). These are good things! We believe that, with a little bit of help, anyone can contribute high-quality code.
+A `dbt` maintainer will review your PR. They may suggest code revision for style or clarity, or request that you add unit or integration test(s). These are good things! We believe that, with a little bit of help, anyone can contribute high-quality code.
 
-Once all tests are passing and your PR has been approved, a dbt maintainer will merge your changes into the active development branch. And that's it! Happy developing :tada:
+Once all tests are passing and your PR has been approved, a `dbt` maintainer will merge your changes into the active development branch. And that's it! Happy developing :tada:

Dockerfile.test

@@ -3,6 +3,9 @@ FROM ubuntu:18.04
 ENV DEBIAN_FRONTEND noninteractive
 
 RUN apt-get update \
+    && apt-get install -y --no-install-recommends \
+    software-properties-common \
+    && add-apt-repository ppa:git-core/ppa -y \
     && apt-get dist-upgrade -y \
     && apt-get install -y --no-install-recommends \
     netcat \
@@ -46,9 +49,7 @@ RUN curl -LO https://github.com/jwilder/dockerize/releases/download/$DOCKERIZE_V
     && tar -C /usr/local/bin -xzvf dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz \
     && rm dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz
 
-RUN pip3 install -U "tox==3.14.4" wheel "six>=1.14.0,<1.15.0" "virtualenv==20.0.3" setuptools
-# tox fails if the 'python' interpreter (python2) doesn't have `tox` installed
-RUN pip install -U "tox==3.14.4" "six>=1.14.0,<1.15.0" "virtualenv==20.0.3" setuptools
+RUN pip3 install -U tox wheel six setuptools
 
 # These args are passed in via docker-compose, which reads then from the .env file.
 # On Linux, run `make .env` to create the .env file for the current user.

License.md

@@ -186,7 +186,7 @@
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright {yyyy} {name of copyright owner}
+   Copyright 2021 dbt Labs, Inc.
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

Makefile

@@ -1,29 +1,81 @@
-.PHONY: install test test-unit test-integration
+.DEFAULT_GOAL:=help
 
-changed_tests := `git status --porcelain | grep '^\(M\| M\|A\| A\)' | awk '{ print $$2 }' | grep '\/test_[a-zA-Z_\-\.]\+.py'`
+# Optional flag to run target in a docker container.
+# (example `make test USE_DOCKER=true`)
+ifeq ($(USE_DOCKER),true)
+	DOCKER_CMD := docker-compose run --rm test
+endif
 
-install:
-	pip install -e .
+.PHONY: dev
+dev: ## Installs dbt-* packages in develop mode along with development dependencies.
+	pip install -r dev-requirements.txt -r editable-requirements.txt
 
-test: .env
-	@echo "Full test run starting..."
-	@time docker-compose run --rm test tox
+.PHONY: mypy
+mypy: .env ## Runs mypy for static type checking.
+	$(DOCKER_CMD) tox -e mypy
 
-test-unit: .env
-	@echo "Unit test run starting..."
-	@time docker-compose run --rm test tox -e unit-py36,flake8
+.PHONY: flake8
+flake8: .env ## Runs flake8 to enforce style guide.
+	$(DOCKER_CMD) tox -e flake8
 
-test-integration: .env
-	@echo "Integration test run starting..."
-	@time docker-compose run --rm test tox -e integration-postgres-py36,integration-redshift-py36,integration-snowflake-py36,integration-bigquery-py36
+.PHONY: lint
+lint: .env ## Runs all code checks in parallel.
+	$(DOCKER_CMD) tox -p -e flake8,mypy
 
-test-quick: .env
-	@echo "Integration test run starting..."
-	@time docker-compose run --rm test tox -e integration-postgres-py36 -- -x
+.PHONY: unit
+unit: .env ## Runs unit tests with py38.
+	$(DOCKER_CMD) tox -e py38
+
+.PHONY: test
+test: .env ## Runs unit tests with py38 and code checks in parallel.
+	$(DOCKER_CMD) tox -p -e py38,flake8,mypy
+
+.PHONY: integration
+integration: .env integration-postgres ## Alias for integration-postgres.
+
+.PHONY: integration-fail-fast
+integration-fail-fast: .env integration-postgres-fail-fast ## Alias for integration-postgres-fail-fast.
+
+.PHONY: integration-postgres
+integration-postgres: .env ## Runs postgres integration tests with py38.
+	$(DOCKER_CMD) tox -e py38-postgres -- -nauto
+
+.PHONY: integration-postgres-fail-fast
+integration-postgres-fail-fast: .env ## Runs postgres integration tests with py38 in "fail fast" mode.
+	$(DOCKER_CMD) tox -e py38-postgres -- -x -nauto
+
+.PHONY: integration-redshift
+integration-redshift: .env ## Runs redshift integration tests with py38.
+	$(DOCKER_CMD) tox -e py38-redshift -- -nauto
+
+.PHONY: integration-redshift-fail-fast
+integration-redshift-fail-fast: .env ## Runs redshift integration tests with py38 in "fail fast" mode.
+	$(DOCKER_CMD) tox -e py38-redshift -- -x -nauto
+
+.PHONY: integration-snowflake
+integration-snowflake: .env ## Runs snowflake integration tests with py38.
+	$(DOCKER_CMD) tox -e py38-snowflake -- -nauto
+
+.PHONY: integration-snowflake-fail-fast
+integration-snowflake-fail-fast: .env ## Runs snowflake integration tests with py38 in "fail fast" mode.
+	$(DOCKER_CMD) tox -e py38-snowflake -- -x -nauto
+
+.PHONY: integration-bigquery
+integration-bigquery: .env ## Runs bigquery integration tests with py38.
+	$(DOCKER_CMD) tox -e py38-bigquery -- -nauto
+
+.PHONY: integration-bigquery-fail-fast
+integration-bigquery-fail-fast: .env ## Runs bigquery integration tests with py38 in "fail fast" mode.
+	$(DOCKER_CMD) tox -e py38-bigquery -- -x -nauto
+
+.PHONY: setup-db
+setup-db: ## Setup Postgres database with docker-compose for system testing.
+	docker-compose up -d database
+	PGHOST=localhost PGUSER=root PGPASSWORD=password PGDATABASE=postgres bash test/setup_db.sh
 
 # This rule creates a file named .env that is used by docker-compose for passing
 # the USER_ID and GROUP_ID arguments to the Docker image.
-.env:
+.env: ## Setup step for using using docker-compose with make target.
 	@touch .env
 ifneq ($(OS),Windows_NT)
 ifneq ($(shell uname -s), Darwin)
@@ -31,9 +83,9 @@ ifneq ($(shell uname -s), Darwin)
 	@echo GROUP_ID=$(shell id -g) >> .env
 endif
 endif
-	@time docker-compose build
 
-clean:
+.PHONY: clean
+clean: ## Resets development environment.
 	rm -f .coverage
 	rm -rf .eggs/
 	rm -f .env
@@ -47,3 +99,14 @@ clean:
 	rm -rf target/
 	find . -type f -name '*.pyc' -delete
 	find . -type d -name '__pycache__' -depth -delete
+
+.PHONY: help
+help: ## Show this help message.
+	@echo 'usage: make [target] [USE_DOCKER=true]'
+	@echo
+	@echo 'targets:'
+	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
+	@echo
+	@echo 'options:'
+	@echo 'use USE_DOCKER=true to run target in a docker container'
+

README.md

@@ -1,28 +1,21 @@
 <p align="center">
-  <img src="https://raw.githubusercontent.com/fishtown-analytics/dbt/6c6649f9129d5d108aa3b0526f634cd8f3a9d1ed/etc/dbt-logo-full.svg" alt="dbt logo" width="500"/>
+  <img src="https://raw.githubusercontent.com/dbt-labs/dbt/ec7dee39f793aa4f7dd3dae37282cc87664813e4/etc/dbt-logo-full.svg" alt="dbt logo" width="500"/>
 </p>
 <p align="center">
-  <a href="https://codeclimate.com/github/fishtown-analytics/dbt">
-    <img src="https://codeclimate.com/github/fishtown-analytics/dbt/badges/gpa.svg" alt="Code Climate"/>
+  <a href="https://github.com/dbt-labs/dbt/actions/workflows/tests.yml?query=branch%3Adevelop">
+    <img src="https://github.com/dbt-labs/dbt/actions/workflows/tests.yml/badge.svg" alt="GitHub Actions"/>
   </a>
-  <a href="https://circleci.com/gh/fishtown-analytics/dbt/tree/master">
-    <img src="https://circleci.com/gh/fishtown-analytics/dbt/tree/master.svg?style=svg"  alt="CircleCI" />
+  <a href="https://circleci.com/gh/dbt-labs/dbt/tree/develop">
+    <img src="https://circleci.com/gh/dbt-labs/dbt/tree/develop.svg?style=svg"  alt="CircleCI" />
   </a>
-  <a href="https://ci.appveyor.com/project/DrewBanin/dbt/branch/development">
-    <img src="https://ci.appveyor.com/api/projects/status/v01rwd3q91jnwp9m/branch/development?svg=true" alt="AppVeyor" />
-  </a>
-  <a href="https://community.getdbt.com">
-    <img src="https://community.getdbt.com/badge.svg" alt="Slack" />
+  <a href="https://dev.azure.com/fishtown-analytics/dbt/_build?definitionId=1&_a=summary&repositoryFilter=1&branchFilter=789%2C789%2C789%2C789">
+    <img src="https://dev.azure.com/fishtown-analytics/dbt/_apis/build/status/fishtown-analytics.dbt?branchName=develop" alt="Azure Pipelines" />
   </a>
 </p>
 
-**[dbt](https://www.getdbt.com/)** (data build tool) enables data analysts and engineers to transform their data using the same practices that software engineers use to build applications.
+**[dbt](https://www.getdbt.com/)** enables data analysts and engineers to transform their data using the same practices that software engineers use to build applications.
 
-dbt is the T in ELT. Organize, cleanse, denormalize, filter, rename, and pre-aggregate the raw data in your warehouse so that it's ready for analysis.
-
-![dbt architecture](https://raw.githubusercontent.com/fishtown-analytics/dbt/6c6649f9129d5d108aa3b0526f634cd8f3a9d1ed/etc/dbt-arch.png)
-
-dbt can be used to [aggregate pageviews into sessions](https://github.com/fishtown-analytics/snowplow), calculate [ad spend ROI](https://github.com/fishtown-analytics/facebook-ads), or report on [email campaign performance](https://github.com/fishtown-analytics/mailchimp).
+![dbt architecture](https://raw.githubusercontent.com/dbt-labs/dbt/6c6649f9129d5d108aa3b0526f634cd8f3a9d1ed/etc/dbt-arch.png)
 
 ## Understanding dbt
 
@@ -30,28 +23,22 @@ Analysts using dbt can transform their data by simply writing select statements,
 
 These select statements, or "models", form a dbt project. Models frequently build on top of one another – dbt makes it easy to [manage relationships](https://docs.getdbt.com/docs/ref) between models, and [visualize these relationships](https://docs.getdbt.com/docs/documentation), as well as assure the quality of your transformations through [testing](https://docs.getdbt.com/docs/testing).
 
-![dbt dag](https://raw.githubusercontent.com/fishtown-analytics/dbt/6c6649f9129d5d108aa3b0526f634cd8f3a9d1ed/etc/dbt-dag.png)
+![dbt dag](https://raw.githubusercontent.com/dbt-labs/dbt/6c6649f9129d5d108aa3b0526f634cd8f3a9d1ed/etc/dbt-dag.png)
 
 ## Getting started
 
--   [Install dbt](https://docs.getdbt.com/docs/installation)
--   Read the [documentation](https://docs.getdbt.com/).
--   Productionize your dbt project with [dbt Cloud](https://www.getdbt.com)
+- [Install dbt](https://docs.getdbt.com/docs/installation)
+- Read the [introduction](https://docs.getdbt.com/docs/introduction/) and [viewpoint](https://docs.getdbt.com/docs/about/viewpoint/)
 
-## Find out more
+## Join the dbt Community
 
--   Check out the [Introduction to dbt](https://docs.getdbt.com/docs/introduction/).
--   Read the [dbt Viewpoint](https://docs.getdbt.com/docs/about/viewpoint/).
-
-## Join thousands of analysts in the dbt community
-
--   Join the [chat](http://community.getdbt.com/) on Slack.
--   Find community posts on [dbt Discourse](https://discourse.getdbt.com).
+- Be part of the conversation in the [dbt Community Slack](http://community.getdbt.com/)
+- Read more on the [dbt Community Discourse](https://discourse.getdbt.com)
 
 ## Reporting bugs and contributing code
 
--   Want to report a bug or request a feature? Let us know on [Slack](http://community.getdbt.com/), or open [an issue](https://github.com/fishtown-analytics/dbt/issues/new).
--   Want to help us build dbt? Check out the [Contributing Getting Started Guide](https://github.com/fishtown-analytics/dbt/blob/HEAD/CONTRIBUTING.md)
+- Want to report a bug or request a feature? Let us know on [Slack](http://community.getdbt.com/), or open [an issue](https://github.com/dbt-labs/dbt/issues/new)
+- Want to help us build dbt? Check out the [Contributing Guide](https://github.com/dbt-labs/dbt/blob/HEAD/CONTRIBUTING.md)
 
 ## Code of Conduct
 

azure-pipelines.yml

@@ -6,8 +6,8 @@
 trigger:
   branches:
     include:
-    - master
-    - dev/*
+    - develop
+    - '*.latest'
     - pr/*
 
 jobs:
@@ -23,7 +23,7 @@ jobs:
   - script: python -m pip install --upgrade pip && pip install tox
     displayName: 'Install dependencies'
 
-  - script: python -m tox -e pywin-unit
+  - script: python -m tox -e py -- -v
     displayName: Run unit tests
 
 - job: PostgresIntegrationTest
@@ -54,7 +54,7 @@ jobs:
   - script: python -m pip install --upgrade pip && pip install tox
     displayName: 'Install dependencies'
 
-  - script: python -m tox -e pywin-postgres
+  - script: python -m tox -e py-postgres -- -v -n4
     displayName: Run integration tests
 
 # These three are all similar except secure environment variables, which MUST be passed along to their tasks,
@@ -62,7 +62,7 @@ jobs:
 - job: SnowflakeIntegrationTest
   pool:
     vmImage: 'vs2017-win2016'
-  dependsOn: PostgresIntegrationTest
+  dependsOn: UnitTest
   condition: succeeded()
   steps:
   - task: UsePythonVersion@0
@@ -73,7 +73,7 @@ jobs:
   - script: python -m pip install --upgrade pip && pip install tox
     displayName: 'Install dependencies'
 
-  - script: python -m tox -e pywin-snowflake
+  - script: python -m tox -e py-snowflake -- -v -n4
     env:
       SNOWFLAKE_TEST_ACCOUNT: $(SNOWFLAKE_TEST_ACCOUNT)
       SNOWFLAKE_TEST_PASSWORD: $(SNOWFLAKE_TEST_PASSWORD)
@@ -87,7 +87,7 @@ jobs:
 - job: BigQueryIntegrationTest
   pool:
     vmImage: 'vs2017-win2016'
-  dependsOn: PostgresIntegrationTest
+  dependsOn: UnitTest
   condition: succeeded()
   steps:
   - task: UsePythonVersion@0
@@ -96,7 +96,7 @@ jobs:
       architecture: 'x64'
   - script: python -m pip install --upgrade pip && pip install tox
     displayName: 'Install dependencies'
-  - script: python -m tox -e pywin-bigquery
+  - script: python -m tox -e py-bigquery -- -v -n4
     env:
       BIGQUERY_SERVICE_ACCOUNT_JSON: $(BIGQUERY_SERVICE_ACCOUNT_JSON)
     displayName: Run integration tests
@@ -104,7 +104,7 @@ jobs:
 - job: RedshiftIntegrationTest
   pool:
     vmImage: 'vs2017-win2016'
-  dependsOn: PostgresIntegrationTest
+  dependsOn: UnitTest
   condition: succeeded()
   steps:
   - task: UsePythonVersion@0
@@ -115,7 +115,7 @@ jobs:
   - script: python -m pip install --upgrade pip && pip install tox
     displayName: 'Install dependencies'
 
-  - script: python -m tox -e pywin-redshift
+  - script: python -m tox -e py-redshift -- -v -n4
     env:
       REDSHIFT_TEST_DBNAME: $(REDSHIFT_TEST_DBNAME)
       REDSHIFT_TEST_PASS: $(REDSHIFT_TEST_PASS)
@@ -139,7 +139,7 @@ jobs:
       inputs:
         versionSpec: '3.7'
         architecture: 'x64'
-    - script: python -m pip install --upgrade pip setuptools && python -m pip install -r requirements.txt && python -m pip install -r dev_requirements.txt
+    - script: python -m pip install --upgrade pip setuptools && python -m pip install -r requirements.txt && python -m pip install -r dev-requirements.txt
       displayName: Install dependencies
     - task: ShellScript@2
       inputs:

core/MANIFEST.in

@@ -1 +1 @@
-recursive-include dbt/include *.py *.sql *.yml *.html *.md
+recursive-include dbt/include *.py *.sql *.yml *.html *.md .gitkeep .gitignore

core/dbt/adapters/base/impl.py

@@ -273,8 +273,8 @@ class BaseAdapter(metaclass=AdapterMeta):
     def load_macro_manifest(self) -> MacroManifest:
         if self._macro_manifest_lazy is None:
             # avoid a circular import
-            from dbt.parser.manifest import load_macro_manifest
-            manifest = load_macro_manifest(
+            from dbt.parser.manifest import ManifestLoader
+            manifest = ManifestLoader.load_macros(
                 self.config, self.connections.set_query_header
             )
             self._macro_manifest_lazy = manifest

core/dbt/adapters/base/relation.py

@@ -433,13 +433,14 @@ class SchemaSearchMap(Dict[InformationSchema, Set[Optional[str]]]):
             for schema in schemas:
                 yield information_schema_name, schema
 
-    def flatten(self):
+    def flatten(self, allow_multiple_databases: bool = False):
         new = self.__class__()
 
-        # make sure we don't have duplicates
-        seen = {r.database.lower() for r in self if r.database}
-        if len(seen) > 1:
-            dbt.exceptions.raise_compiler_error(str(seen))
+        # make sure we don't have multiple databases if allow_multiple_databases is set to False
+        if not allow_multiple_databases:
+            seen = {r.database.lower() for r in self if r.database}
+            if len(seen) > 1:
+                dbt.exceptions.raise_compiler_error(str(seen))
 
         for information_schema_name, schema in self.search():
             path = {

core/dbt/adapters/sql/connections.py

@@ -99,7 +99,14 @@ class SQLConnectionManager(BaseConnectionManager):
         column_names: Iterable[str],
         rows: Iterable[Any]
     ) -> List[Dict[str, Any]]:
-
+        unique_col_names = dict()
+        for idx in range(len(column_names)):
+            col_name = column_names[idx]
+            if col_name in unique_col_names:
+                unique_col_names[col_name] += 1
+                column_names[idx] = f'{col_name}_{unique_col_names[col_name]}'
+            else:
+                unique_col_names[column_names[idx]] = 1
         return [dict(zip(column_names, row)) for row in rows]
 
     @classmethod

core/dbt/clients/agate_helper.py

@@ -35,7 +35,11 @@ class ISODateTime(agate.data_types.DateTime):
         )
 
 
-def build_type_tester(text_columns: Iterable[str]) -> agate.TypeTester:
+def build_type_tester(
+    text_columns: Iterable[str],
+    string_null_values: Optional[Iterable[str]] = ('null', '')
+) -> agate.TypeTester:
+
     types = [
         agate.data_types.Number(null_values=('null', '')),
         agate.data_types.Date(null_values=('null', ''),
@@ -46,10 +50,10 @@ def build_type_tester(text_columns: Iterable[str]) -> agate.TypeTester:
         agate.data_types.Boolean(true_values=('true',),
                                  false_values=('false',),
                                  null_values=('null', '')),
-        agate.data_types.Text(null_values=('null', ''))
+        agate.data_types.Text(null_values=string_null_values)
     ]
     force = {
-        k: agate.data_types.Text(null_values=('null', ''))
+        k: agate.data_types.Text(null_values=string_null_values)
         for k in text_columns
     }
     return agate.TypeTester(force=force, types=types)
@@ -66,7 +70,13 @@ def table_from_rows(
     if text_only_columns is None:
         column_types = DEFAULT_TYPE_TESTER
     else:
-        column_types = build_type_tester(text_only_columns)
+        # If text_only_columns are present, prevent coercing empty string or
+        # literal 'null' strings to a None representation.
+        column_types = build_type_tester(
+            text_only_columns,
+            string_null_values=()
+        )
+
     return agate.Table(rows, column_names, column_types=column_types)
 
 
@@ -86,19 +96,34 @@ def table_from_data(data, column_names: Iterable[str]) -> agate.Table:
 
 
 def table_from_data_flat(data, column_names: Iterable[str]) -> agate.Table:
-    "Convert list of dictionaries into an Agate table"
+    """
+    Convert a list of dictionaries into an Agate table. This method does not
+    coerce string values into more specific types (eg. '005' will not be
+    coerced to '5'). Additionally, this method does not coerce values to
+    None (eg. '' or 'null' will retain their string literal representations).
+    """
 
     rows = []
+    text_only_columns = set()
     for _row in data:
         row = []
-        for value in list(_row.values()):
+        for col_name in column_names:
+            value = _row[col_name]
             if isinstance(value, (dict, list, tuple)):
-                row.append(json.dumps(value, cls=dbt.utils.JSONEncoder))
-            else:
-                row.append(value)
+                # Represent container types as json strings
+                value = json.dumps(value, cls=dbt.utils.JSONEncoder)
+                text_only_columns.add(col_name)
+            elif isinstance(value, str):
+                text_only_columns.add(col_name)
+            row.append(value)
+
         rows.append(row)
 
-    return table_from_rows(rows=rows, column_names=column_names)
+    return table_from_rows(
+        rows=rows,
+        column_names=column_names,
+        text_only_columns=text_only_columns
+    )
 
 
 def empty_table():

core/dbt/clients/git.py

@@ -4,21 +4,43 @@ import os.path
 from dbt.clients.system import run_cmd, rmdir
 from dbt.logger import GLOBAL_LOGGER as logger
 import dbt.exceptions
+from packaging import version
 
 
-def clone(repo, cwd, dirname=None, remove_git_dir=False, branch=None):
+def _is_commit(revision: str) -> bool:
+    # match SHA-1 git commit
+    return bool(re.match(r"\b[0-9a-f]{40}\b", revision))
+
+
+def clone(repo, cwd, dirname=None, remove_git_dir=False, revision=None, subdirectory=None):
+    has_revision = revision is not None
+    is_commit = _is_commit(revision or "")
+
     clone_cmd = ['git', 'clone', '--depth', '1']
+    if subdirectory:
+        logger.debug('  Subdirectory specified: {}, using sparse checkout.'.format(subdirectory))
+        out, _ = run_cmd(cwd, ['git', '--version'], env={'LC_ALL': 'C'})
+        git_version = version.parse(re.search(r"\d+\.\d+\.\d+", out.decode("utf-8")).group(0))
+        if not git_version >= version.parse("2.25.0"):
+            # 2.25.0 introduces --sparse
+            raise RuntimeError(
+                "Please update your git version to pull a dbt package "
+                "from a subdirectory: your version is {}, >= 2.25.0 needed".format(git_version)
+            )
+        clone_cmd.extend(['--filter=blob:none', '--sparse'])
 
-    if branch is not None:
-        clone_cmd.extend(['--branch', branch])
+    if has_revision and not is_commit:
+        clone_cmd.extend(['--branch', revision])
 
     clone_cmd.append(repo)
 
     if dirname is not None:
         clone_cmd.append(dirname)
-
     result = run_cmd(cwd, clone_cmd, env={'LC_ALL': 'C'})
 
+    if subdirectory:
+        run_cmd(os.path.join(cwd, dirname or ''), ['git', 'sparse-checkout', 'set', subdirectory])
+
     if remove_git_dir:
         rmdir(os.path.join(dirname, '.git'))
 
@@ -31,33 +53,38 @@ def list_tags(cwd):
     return tags
 
 
-def _checkout(cwd, repo, branch):
-    logger.debug('  Checking out branch {}.'.format(branch))
+def _checkout(cwd, repo, revision):
+    logger.debug('  Checking out revision {}.'.format(revision))
 
-    run_cmd(cwd, ['git', 'remote', 'set-branches', 'origin', branch])
-    run_cmd(cwd, ['git', 'fetch', '--tags', '--depth', '1', 'origin', branch])
+    fetch_cmd = ["git", "fetch", "origin", "--depth", "1"]
 
-    tags = list_tags(cwd)
-
-    # Prefer tags to branches if one exists
-    if branch in tags:
-        spec = 'tags/{}'.format(branch)
+    if _is_commit(revision):
+        run_cmd(cwd, fetch_cmd + [revision])
     else:
-        spec = 'origin/{}'.format(branch)
+        run_cmd(cwd, ['git', 'remote', 'set-branches', 'origin', revision])
+        run_cmd(cwd, fetch_cmd + ["--tags", revision])
+
+    if _is_commit(revision):
+        spec = revision
+    # Prefer tags to branches if one exists
+    elif revision in list_tags(cwd):
+        spec = 'tags/{}'.format(revision)
+    else:
+        spec = 'origin/{}'.format(revision)
 
     out, err = run_cmd(cwd, ['git', 'reset', '--hard', spec],
                        env={'LC_ALL': 'C'})
     return out, err
 
 
-def checkout(cwd, repo, branch=None):
-    if branch is None:
-        branch = 'HEAD'
+def checkout(cwd, repo, revision=None):
+    if revision is None:
+        revision = 'HEAD'
     try:
-        return _checkout(cwd, repo, branch)
+        return _checkout(cwd, repo, revision)
     except dbt.exceptions.CommandResultError as exc:
         stderr = exc.stderr.decode('utf-8').strip()
-    dbt.exceptions.bad_package_spec(repo, branch, stderr)
+    dbt.exceptions.bad_package_spec(repo, revision, stderr)
 
 
 def get_current_sha(cwd):
@@ -71,11 +98,16 @@ def remove_remote(cwd):
 
 
 def clone_and_checkout(repo, cwd, dirname=None, remove_git_dir=False,
-                       branch=None):
+                       revision=None, subdirectory=None):
     exists = None
     try:
-        _, err = clone(repo, cwd, dirname=dirname,
-                       remove_git_dir=remove_git_dir)
+        _, err = clone(
+            repo,
+            cwd,
+            dirname=dirname,
+            remove_git_dir=remove_git_dir,
+            subdirectory=subdirectory,
+        )
     except dbt.exceptions.CommandResultError as exc:
         err = exc.stderr.decode('utf-8')
         exists = re.match("fatal: destination path '(.+)' already exists", err)
@@ -97,7 +129,7 @@ def clone_and_checkout(repo, cwd, dirname=None, remove_git_dir=False,
         logger.debug('Pulling new dependency {}.', directory)
     full_path = os.path.join(cwd, directory)
     start_sha = get_current_sha(full_path)
-    checkout(full_path, repo, branch)
+    checkout(full_path, repo, revision)
     end_sha = get_current_sha(full_path)
     if exists:
         if start_sha == end_sha:
@@ -107,4 +139,4 @@ def clone_and_checkout(repo, cwd, dirname=None, remove_git_dir=False,
                          start_sha[:7], end_sha[:7])
     else:
         logger.debug('  Checked out at {}.', end_sha[:7])
-    return directory
+    return os.path.join(directory, subdirectory or '')

core/dbt/clients/jinja.py

@@ -21,7 +21,7 @@ import jinja2.sandbox
 
 from dbt.utils import (
     get_dbt_macro_name, get_docs_macro_name, get_materialization_macro_name,
-    deep_map
+    get_test_macro_name, deep_map
 )
 
 from dbt.clients._jinja_blocks import BlockIterator, BlockData, BlockTag
@@ -29,7 +29,8 @@ from dbt.contracts.graph.compiled import CompiledSchemaTestNode
 from dbt.contracts.graph.parsed import ParsedSchemaTestNode
 from dbt.exceptions import (
     InternalException, raise_compiler_error, CompilationException,
-    invalid_materialization_argument, MacroReturn, JinjaRenderingException
+    invalid_materialization_argument, MacroReturn, JinjaRenderingException,
+    UndefinedMacroException
 )
 from dbt import flags
 from dbt.logger import GLOBAL_LOGGER as logger  # noqa
@@ -408,6 +409,20 @@ class DocumentationExtension(jinja2.ext.Extension):
         return node
 
 
+class TestExtension(jinja2.ext.Extension):
+    tags = ['test']
+
+    def parse(self, parser):
+        node = jinja2.nodes.Macro(lineno=next(parser.stream).lineno)
+        test_name = parser.parse_assign_target(name_only=True).name
+
+        parser.parse_signature(node)
+        node.name = get_test_macro_name(test_name)
+        node.body = parser.parse_statements(('name:endtest',),
+                                            drop_needle=True)
+        return node
+
+
 def _is_dunder_name(name):
     return name.startswith('__') and name.endswith('__')
 
@@ -479,6 +494,7 @@ def get_environment(
 
     args['extensions'].append(MaterializationExtension)
     args['extensions'].append(DocumentationExtension)
+    args['extensions'].append(TestExtension)
 
     env_cls: Type[jinja2.Environment]
     text_filter: Type
@@ -503,7 +519,7 @@ def catch_jinja(node=None) -> Iterator[None]:
         e.translated = False
         raise CompilationException(str(e), node) from e
     except jinja2.exceptions.UndefinedError as e:
-        raise CompilationException(str(e), node) from e
+        raise UndefinedMacroException(str(e), node) from e
     except CompilationException as exc:
         exc.add_node(node)
         raise

core/dbt/clients/jinja_static.py

@@ -0,0 +1,225 @@
+import jinja2
+from dbt.clients.jinja import get_environment
+from dbt.exceptions import raise_compiler_error
+
+
+def statically_extract_macro_calls(string, ctx, db_wrapper=None):
+    # set 'capture_macros' to capture undefined
+    env = get_environment(None, capture_macros=True)
+    parsed = env.parse(string)
+
+    standard_calls = ['source', 'ref', 'config']
+    possible_macro_calls = []
+    for func_call in parsed.find_all(jinja2.nodes.Call):
+        func_name = None
+        if hasattr(func_call, 'node') and hasattr(func_call.node, 'name'):
+            func_name = func_call.node.name
+        else:
+            # func_call for dbt_utils.current_timestamp macro
+            # Call(
+            #   node=Getattr(
+            #     node=Name(
+            #       name='dbt_utils',
+            #       ctx='load'
+            #     ),
+            #     attr='current_timestamp',
+            #     ctx='load
+            #   ),
+            #   args=[],
+            #   kwargs=[],
+            #   dyn_args=None,
+            #   dyn_kwargs=None
+            # )
+            if (hasattr(func_call, 'node') and
+                    hasattr(func_call.node, 'node') and
+                    type(func_call.node.node).__name__ == 'Name' and
+                    hasattr(func_call.node, 'attr')):
+                package_name = func_call.node.node.name
+                macro_name = func_call.node.attr
+                if package_name == 'adapter':
+                    if macro_name == 'dispatch':
+                        ad_macro_calls = statically_parse_adapter_dispatch(
+                            func_call, ctx, db_wrapper)
+                        possible_macro_calls.extend(ad_macro_calls)
+                    else:
+                        # This skips calls such as adapter.parse_index
+                        continue
+                else:
+                    func_name = f'{package_name}.{macro_name}'
+            else:
+                continue
+        if not func_name:
+            continue
+        if func_name in standard_calls:
+            continue
+        elif ctx.get(func_name):
+            continue
+        else:
+            if func_name not in possible_macro_calls:
+                possible_macro_calls.append(func_name)
+
+    return possible_macro_calls
+
+
+# Call(
+#   node=Getattr(
+#     node=Name(
+#       name='adapter',
+#       ctx='load'
+#     ),
+#     attr='dispatch',
+#     ctx='load'
+#   ),
+#   args=[
+#     Const(value='test_pkg_and_dispatch')
+#   ],
+#   kwargs=[
+#     Keyword(
+#       key='packages',
+#       value=Call(node=Getattr(node=Name(name='local_utils', ctx='load'),
+#          attr='_get_utils_namespaces', ctx='load'), args=[], kwargs=[],
+#          dyn_args=None, dyn_kwargs=None)
+#     )
+#   ],
+#   dyn_args=None,
+#   dyn_kwargs=None
+# )
+def statically_parse_adapter_dispatch(func_call, ctx, db_wrapper):
+    possible_macro_calls = []
+    # This captures an adapter.dispatch('<macro_name>') call.
+
+    func_name = None
+    # macro_name positional argument
+    if len(func_call.args) > 0:
+        func_name = func_call.args[0].value
+    if func_name:
+        possible_macro_calls.append(func_name)
+
+    # packages positional argument
+    packages = None
+    macro_namespace = None
+    packages_arg = None
+    packages_arg_type = None
+
+    if len(func_call.args) > 1:
+        packages_arg = func_call.args[1]
+        # This can be a List or a Call
+        packages_arg_type = type(func_call.args[1]).__name__
+
+    # keyword arguments
+    if func_call.kwargs:
+        for kwarg in func_call.kwargs:
+            if kwarg.key == 'packages':
+                # The packages keyword will be deprecated and
+                # eventually removed
+                packages_arg = kwarg.value
+                # This can be a List or a Call
+                packages_arg_type = type(kwarg.value).__name__
+            elif kwarg.key == 'macro_name':
+                # This will remain to enable static resolution
+                if type(kwarg.value).__name__ == 'Const':
+                    func_name = kwarg.value.value
+                    possible_macro_calls.append(func_name)
+                else:
+                    raise_compiler_error(f"The macro_name parameter ({kwarg.value.value}) "
+                                         "to adapter.dispatch was not a string")
+            elif kwarg.key == 'macro_namespace':
+                # This will remain to enable static resolution
+                kwarg_type = type(kwarg.value).__name__
+                if kwarg_type == 'Const':
+                    macro_namespace = kwarg.value.value
+                else:
+                    raise_compiler_error("The macro_namespace parameter to adapter.dispatch "
+                                         f"is a {kwarg_type}, not a string")
+
+    # positional arguments
+    if packages_arg:
+        if packages_arg_type == 'List':
+            # This will remain to enable static resolution
+            packages = []
+            for item in packages_arg.items:
+                packages.append(item.value)
+        elif packages_arg_type == 'Const':
+            # This will remain to enable static resolution
+            macro_namespace = packages_arg.value
+        elif packages_arg_type == 'Call':
+            # This is deprecated and should be removed eventually.
+            # It is here to support (hackily) common ways of providing
+            # a packages list to adapter.dispatch
+            if (hasattr(packages_arg, 'node') and
+                    hasattr(packages_arg.node, 'node') and
+                    hasattr(packages_arg.node.node, 'name') and
+                    hasattr(packages_arg.node, 'attr')):
+                package_name = packages_arg.node.node.name
+                macro_name = packages_arg.node.attr
+                if (macro_name.startswith('_get') and 'namespaces' in macro_name):
+                    # noqa: https://github.com/fishtown-analytics/dbt-utils/blob/9e9407b/macros/cross_db_utils/_get_utils_namespaces.sql
+                    var_name = f'{package_name}_dispatch_list'
+                    # hard code compatibility for fivetran_utils, just a teensy bit different
+                    # noqa: https://github.com/fivetran/dbt_fivetran_utils/blob/0978ba2/macros/_get_utils_namespaces.sql
+                    if package_name == 'fivetran_utils':
+                        default_packages = ['dbt_utils', 'fivetran_utils']
+                    else:
+                        default_packages = [package_name]
+
+                    namespace_names = get_dispatch_list(ctx, var_name, default_packages)
+                    packages = []
+                    if namespace_names:
+                        packages.extend(namespace_names)
+                else:
+                    msg = (
+                        f"As of v0.19.2, custom macros, such as '{macro_name}', are no longer "
+                        "supported in the 'packages' argument of 'adapter.dispatch()'.\n"
+                        f"See https://docs.getdbt.com/reference/dbt-jinja-functions/dispatch "
+                        "for details."
+                    ).strip()
+                    raise_compiler_error(msg)
+        elif packages_arg_type == 'Add':
+            # This logic is for when there is a variable and an addition of a list,
+            # like: packages = (var('local_utils_dispatch_list', []) + ['local_utils2'])
+            # This is deprecated and should be removed eventually.
+            namespace_var = None
+            default_namespaces = []
+            # This might be a single call or it might be the 'left' piece in an addition
+            for var_call in packages_arg.find_all(jinja2.nodes.Call):
+                if (hasattr(var_call, 'node') and
+                        var_call.node.name == 'var' and
+                        hasattr(var_call, 'args')):
+                    namespace_var = var_call.args[0].value
+            if hasattr(packages_arg, 'right'):  # we have a default list of namespaces
+                for item in packages_arg.right.items:
+                    default_namespaces.append(item.value)
+            if namespace_var:
+                namespace_names = get_dispatch_list(ctx, namespace_var, default_namespaces)
+                packages = []
+                if namespace_names:
+                    packages.extend(namespace_names)
+
+    if db_wrapper:
+        macro = db_wrapper.dispatch(
+            func_name,
+            packages=packages,
+            macro_namespace=macro_namespace
+        ).macro
+        func_name = f'{macro.package_name}.{macro.name}'
+        possible_macro_calls.append(func_name)
+    else:  # this is only for test/unit/test_macro_calls.py
+        if macro_namespace:
+            packages = [macro_namespace]
+        if packages is None:
+            packages = []
+        for package_name in packages:
+            possible_macro_calls.append(f'{package_name}.{func_name}')
+
+    return possible_macro_calls
+
+
+def get_dispatch_list(ctx, var_name, default_packages):
+    namespace_list = None
+    try:
+        # match the logic currently used in package _get_namespaces() macro
+        namespace_list = ctx['var'](var_name) + default_packages
+    except Exception:
+        pass
+    namespace_list = namespace_list if namespace_list else default_packages
+    return namespace_list

core/dbt/clients/registry.py

@@ -28,11 +28,10 @@ def _wrap_exceptions(fn):
             attempt += 1
             try:
                 return fn(*args, **kwargs)
-            except requests.exceptions.ConnectionError as exc:
+            except (requests.exceptions.ConnectionError, requests.exceptions.Timeout) as exc:
                 if attempt < max_attempts:
                     time.sleep(1)
                     continue
-
                 raise RegistryException(
                     'Unable to connect to registry hub'
                 ) from exc
@@ -43,7 +42,7 @@ def _wrap_exceptions(fn):
 def _get(path, registry_base_url=None):
     url = _get_url(path, registry_base_url)
     logger.debug('Making package registry request: GET {}'.format(url))
-    resp = requests.get(url)
+    resp = requests.get(url, timeout=30)
     logger.debug('Response from registry: GET {} {}'.format(url,
                                                             resp.status_code))
     resp.raise_for_status()

core/dbt/clients/system.py

@@ -416,6 +416,9 @@ def run_cmd(
         full_env.update(env)
 
     try:
+        exe_pth = shutil.which(cmd[0])
+        if exe_pth:
+            cmd = [os.path.abspath(exe_pth)] + list(cmd[1:])
         proc = subprocess.Popen(
             cmd,
             cwd=cwd,

core/dbt/compilation.py

@@ -12,9 +12,8 @@ from dbt.clients.system import make_directory
 from dbt.context.providers import generate_runtime_model
 from dbt.contracts.graph.manifest import Manifest
 from dbt.contracts.graph.compiled import (
-    CompiledDataTestNode,
-    CompiledSchemaTestNode,
     COMPILED_TYPES,
+    CompiledSchemaTestNode,
     GraphMemberNode,
     InjectedCTE,
     ManifestNode,
@@ -183,8 +182,7 @@ class Compiler:
 
     def _get_relation_name(self, node: ParsedNode):
         relation_name = None
-        if (node.resource_type in NodeType.refable() and
-                not node.is_ephemeral_model):
+        if node.is_relational and not node.is_ephemeral_model:
             adapter = get_adapter(self.config)
             relation_cls = adapter.Relation
             relation_name = str(relation_cls.create_from(self.config, node))
@@ -251,22 +249,19 @@ class Compiler:
 
         return str(parsed)
 
-    def _get_dbt_test_name(self) -> str:
-        return 'dbt__cte__internal_test'
-
-    # This method is called by the 'compile_node' method. Starting
-    # from the node that it is passed in, it will recursively call
-    # itself using the 'extra_ctes'.  The 'ephemeral' models do
-    # not produce SQL that is executed directly, instead they
-    # are rolled up into the models that refer to them by
-    # inserting CTEs into the SQL.
     def _recursively_prepend_ctes(
         self,
         model: NonSourceCompiledNode,
         manifest: Manifest,
         extra_context: Optional[Dict[str, Any]],
     ) -> Tuple[NonSourceCompiledNode, List[InjectedCTE]]:
-
+        """This method is called by the 'compile_node' method. Starting
+        from the node that it is passed in, it will recursively call
+        itself using the 'extra_ctes'.  The 'ephemeral' models do
+        not produce SQL that is executed directly, instead they
+        are rolled up into the models that refer to them by
+        inserting CTEs into the SQL.
+        """
         if model.compiled_sql is None:
             raise RuntimeException(
                 'Cannot inject ctes into an unparsed node', model
@@ -284,65 +279,59 @@ class Compiler:
         # gathered and then "injected" into the model.
         prepended_ctes: List[InjectedCTE] = []
 
-        dbt_test_name = self._get_dbt_test_name()
-
         # extra_ctes are added to the model by
         # RuntimeRefResolver.create_relation, which adds an
         # extra_cte for every model relation which is an
         # ephemeral model.
         for cte in model.extra_ctes:
-            if cte.id == dbt_test_name:
-                sql = cte.sql
+            if cte.id not in manifest.nodes:
+                raise InternalException(
+                    f'During compilation, found a cte reference that '
+                    f'could not be resolved: {cte.id}'
+                )
+            cte_model = manifest.nodes[cte.id]
+
+            if not cte_model.is_ephemeral_model:
+                raise InternalException(f'{cte.id} is not ephemeral')
+
+            # This model has already been compiled, so it's been
+            # through here before
+            if getattr(cte_model, 'compiled', False):
+                assert isinstance(cte_model, tuple(COMPILED_TYPES.values()))
+                cte_model = cast(NonSourceCompiledNode, cte_model)
+                new_prepended_ctes = cte_model.extra_ctes
+
+            # if the cte_model isn't compiled, i.e. first time here
             else:
-                if cte.id not in manifest.nodes:
-                    raise InternalException(
-                        f'During compilation, found a cte reference that '
-                        f'could not be resolved: {cte.id}'
+                # This is an ephemeral parsed model that we can compile.
+                # Compile and update the node
+                cte_model = self._compile_node(
+                    cte_model, manifest, extra_context)
+                # recursively call this method
+                cte_model, new_prepended_ctes = \
+                    self._recursively_prepend_ctes(
+                        cte_model, manifest, extra_context
                     )
-                cte_model = manifest.nodes[cte.id]
+                # Save compiled SQL file and sync manifest
+                self._write_node(cte_model)
+                manifest.sync_update_node(cte_model)
 
-                if not cte_model.is_ephemeral_model:
-                    raise InternalException(f'{cte.id} is not ephemeral')
+            _extend_prepended_ctes(prepended_ctes, new_prepended_ctes)
 
-                # This model has already been compiled, so it's been
-                # through here before
-                if getattr(cte_model, 'compiled', False):
-                    assert isinstance(cte_model,
-                                      tuple(COMPILED_TYPES.values()))
-                    cte_model = cast(NonSourceCompiledNode, cte_model)
-                    new_prepended_ctes = cte_model.extra_ctes
-
-                # if the cte_model isn't compiled, i.e. first time here
-                else:
-                    # This is an ephemeral parsed model that we can compile.
-                    # Compile and update the node
-                    cte_model = self._compile_node(
-                        cte_model, manifest, extra_context)
-                    # recursively call this method
-                    cte_model, new_prepended_ctes = \
-                        self._recursively_prepend_ctes(
-                            cte_model, manifest, extra_context
-                        )
-                    # Save compiled SQL file and sync manifest
-                    self._write_node(cte_model)
-                    manifest.sync_update_node(cte_model)
-
-                _extend_prepended_ctes(prepended_ctes, new_prepended_ctes)
-
-                new_cte_name = self.add_ephemeral_prefix(cte_model.name)
-                sql = f' {new_cte_name} as (\n{cte_model.compiled_sql}\n)'
+            new_cte_name = self.add_ephemeral_prefix(cte_model.name)
+            rendered_sql = (
+                cte_model._pre_injected_sql or cte_model.compiled_sql
+            )
+            sql = f' {new_cte_name} as (\n{rendered_sql}\n)'
 
             _add_prepended_cte(prepended_ctes, InjectedCTE(id=cte.id, sql=sql))
 
-        # We don't save injected_sql into compiled sql for ephemeral models
-        # because it will cause problems with processing of subsequent models.
-        # Ephemeral models do not produce executable SQL of their own.
-        if not model.is_ephemeral_model:
-            injected_sql = self._inject_ctes_into_sql(
-                model.compiled_sql,
-                prepended_ctes,
-            )
-            model.compiled_sql = injected_sql
+        injected_sql = self._inject_ctes_into_sql(
+            model.compiled_sql,
+            prepended_ctes,
+        )
+        model._pre_injected_sql = model.compiled_sql
+        model.compiled_sql = injected_sql
         model.extra_ctes_injected = True
         model.extra_ctes = prepended_ctes
         model.validate(model.to_dict(omit_none=True))
@@ -351,34 +340,6 @@ class Compiler:
 
         return model, prepended_ctes
 
-    def _add_ctes(
-        self,
-        compiled_node: NonSourceCompiledNode,
-        manifest: Manifest,
-        extra_context: Dict[str, Any],
-    ) -> NonSourceCompiledNode:
-        """Wrap the data test SQL in a CTE."""
-
-        # for data tests, we need to insert a special CTE at the end of the
-        # list containing the test query, and then have the "real" query be a
-        # select count(*) from that model.
-        # the benefit of doing it this way is that _add_ctes() can be
-        # rewritten for different adapters to handle databases that don't
-        # support CTEs, or at least don't have full support.
-        if isinstance(compiled_node, CompiledDataTestNode):
-            # the last prepend (so last in order) should be the data test body.
-            # then we can add our select count(*) from _that_ cte as the "real"
-            # compiled_sql, and do the regular prepend logic from CTEs.
-            name = self._get_dbt_test_name()
-            cte = InjectedCTE(
-                id=name,
-                sql=f' {name} as (\n{compiled_node.compiled_sql}\n)'
-            )
-            compiled_node.extra_ctes.append(cte)
-            compiled_node.compiled_sql = f'\nselect count(*) from {name}'
-
-        return compiled_node
-
     # creates a compiled_node from the ManifestNode passed in,
     # creates a "context" dictionary for jinja rendering,
     # and then renders the "compiled_sql" using the node, the
@@ -417,12 +378,6 @@ class Compiler:
 
         compiled_node.compiled = True
 
-        # add ctes for specific test nodes, and also for
-        # possible future use in adapters
-        compiled_node = self._add_ctes(
-            compiled_node, manifest, extra_context
-        )
-
         return compiled_node
 
     def write_graph_file(self, linker: Linker, manifest: Manifest):
@@ -486,18 +441,13 @@ class Compiler:
         logger.debug(f'Writing injected SQL for node "{node.unique_id}"')
 
         if node.compiled_sql:
-            node.build_path = node.write_node(
+            node.compiled_path = node.write_node(
                 self.config.target_path,
                 'compiled',
                 node.compiled_sql
             )
         return node
 
-    # This is the main entry point into this code. It's called by
-    # CompileRunner.compile, GenericRPCRunner.compile, and
-    # RunTask.get_hook_sql. It calls '_compile_node' to convert
-    # the node into a compiled node, and then calls the
-    # recursive method to "prepend" the ctes.
     def compile_node(
         self,
         node: ManifestNode,
@@ -505,6 +455,12 @@ class Compiler:
         extra_context: Optional[Dict[str, Any]] = None,
         write: bool = True,
     ) -> NonSourceCompiledNode:
+        """This is the main entry point into this code. It's called by
+        CompileRunner.compile, GenericRPCRunner.compile, and
+        RunTask.get_hook_sql. It calls '_compile_node' to convert
+        the node into a compiled node, and then calls the
+        recursive method to "prepend" the ctes.
+        """
         node = self._compile_node(node, manifest, extra_context)
 
         node, _ = self._recursively_prepend_ctes(

core/dbt/config/project.py

@@ -349,16 +349,20 @@ class PartialProject(RenderComponents):
         if cfg.quoting is not None:
             quoting = cfg.quoting.to_dict(omit_none=True)
 
+        dispatch: List[Dict[str, Any]]
         models: Dict[str, Any]
         seeds: Dict[str, Any]
         snapshots: Dict[str, Any]
         sources: Dict[str, Any]
+        tests: Dict[str, Any]
         vars_value: VarProvider
 
+        dispatch = cfg.dispatch
         models = cfg.models
         seeds = cfg.seeds
         snapshots = cfg.snapshots
         sources = cfg.sources
+        tests = cfg.tests
         if cfg.vars is None:
             vars_dict: Dict[str, Any] = {}
         else:
@@ -400,6 +404,7 @@ class PartialProject(RenderComponents):
             models=models,
             on_run_start=on_run_start,
             on_run_end=on_run_end,
+            dispatch=dispatch,
             seeds=seeds,
             snapshots=snapshots,
             dbt_version=dbt_version,
@@ -408,6 +413,7 @@ class PartialProject(RenderComponents):
             selectors=selectors,
             query_comment=query_comment,
             sources=sources,
+            tests=tests,
             vars=vars_value,
             config_version=cfg.config_version,
             unrendered=unrendered,
@@ -510,9 +516,11 @@ class Project:
     models: Dict[str, Any]
     on_run_start: List[str]
     on_run_end: List[str]
+    dispatch: List[Dict[str, Any]]
     seeds: Dict[str, Any]
     snapshots: Dict[str, Any]
     sources: Dict[str, Any]
+    tests: Dict[str, Any]
     vars: VarProvider
     dbt_version: List[VersionSpecifier]
     packages: Dict[str, Any]
@@ -568,9 +576,11 @@ class Project:
             'models': self.models,
             'on-run-start': self.on_run_start,
             'on-run-end': self.on_run_end,
+            'dispatch': self.dispatch,
             'seeds': self.seeds,
             'snapshots': self.snapshots,
             'sources': self.sources,
+            'tests': self.tests,
             'vars': self.vars.to_dict(),
             'require-dbt-version': [
                 v.to_version_string() for v in self.dbt_version
@@ -642,3 +652,9 @@ class Project:
                 f'{list(self.selectors)}'
             )
         return self.selectors[name]
+
+    def get_macro_search_order(self, macro_namespace: str):
+        for dispatch_entry in self.dispatch:
+            if dispatch_entry['macro_namespace'] == macro_namespace:
+                return dispatch_entry['search_order']
+        return None

core/dbt/config/renderer.py

@@ -145,9 +145,9 @@ class DbtProjectYamlRenderer(BaseRenderer):
         if first == 'vars':
             return False
 
-        if first in {'seeds', 'models', 'snapshots', 'seeds'}:
+        if first in {'seeds', 'models', 'snapshots', 'tests'}:
             keypath_parts = {
-                (k.lstrip('+') if isinstance(k, str) else k)
+                (k.lstrip('+ ') if isinstance(k, str) else k)
                 for k in keypath
             }
             # model-level hooks

core/dbt/config/runtime.py

@@ -102,6 +102,7 @@ class RuntimeConfig(Project, Profile, AdapterRequiredConfig):
             models=project.models,
             on_run_start=project.on_run_start,
             on_run_end=project.on_run_end,
+            dispatch=project.dispatch,
             seeds=project.seeds,
             snapshots=project.snapshots,
             dbt_version=project.dbt_version,
@@ -110,6 +111,7 @@ class RuntimeConfig(Project, Profile, AdapterRequiredConfig):
             selectors=project.selectors,
             query_comment=project.query_comment,
             sources=project.sources,
+            tests=project.tests,
             vars=project.vars,
             config_version=project.config_version,
             unrendered=project.unrendered,
@@ -272,7 +274,7 @@ class RuntimeConfig(Project, Profile, AdapterRequiredConfig):
         return frozenset(paths)
 
     def get_resource_config_paths(self) -> Dict[str, PathSet]:
-        """Return a dictionary with 'seeds' and 'models' keys whose values are
+        """Return a dictionary with resource type keys whose values are
         lists of lists of strings, where each inner list of strings represents
         a configured path in the resource.
         """
@@ -281,6 +283,7 @@ class RuntimeConfig(Project, Profile, AdapterRequiredConfig):
             'seeds': self._get_config_paths(self.seeds),
             'snapshots': self._get_config_paths(self.snapshots),
             'sources': self._get_config_paths(self.sources),
+            'tests': self._get_config_paths(self.tests),
         }
 
     def get_unused_resource_config_paths(
@@ -326,6 +329,17 @@ class RuntimeConfig(Project, Profile, AdapterRequiredConfig):
         if self.dependencies is None:
             all_projects = {self.project_name: self}
             internal_packages = get_include_paths(self.credentials.type)
+            # raise exception if fewer installed packages than in packages.yml
+            count_packages_specified = len(self.packages.packages)  # type: ignore
+            count_packages_installed = len(tuple(self._get_project_directories()))
+            if count_packages_specified > count_packages_installed:
+                raise_compiler_error(
+                    f'dbt found {count_packages_specified} package(s) '
+                    f'specified in packages.yml, but only '
+                    f'{count_packages_installed} package(s) installed '
+                    f'in {self.modules_path}. Run "dbt deps" to '
+                    f'install package dependencies.'
+                )
             project_paths = itertools.chain(
                 internal_packages,
                 self._get_project_directories()
@@ -480,6 +494,7 @@ class UnsetProfileConfig(RuntimeConfig):
             models=project.models,
             on_run_start=project.on_run_start,
             on_run_end=project.on_run_end,
+            dispatch=project.dispatch,
             seeds=project.seeds,
             snapshots=project.snapshots,
             dbt_version=project.dbt_version,
@@ -488,6 +503,7 @@ class UnsetProfileConfig(RuntimeConfig):
             selectors=project.selectors,
             query_comment=project.query_comment,
             sources=project.sources,
+            tests=project.tests,
             vars=project.vars,
             config_version=project.config_version,
             unrendered=project.unrendered,

core/dbt/context/configured.py

@@ -75,8 +75,26 @@ class SchemaYamlContext(ConfiguredContext):
         )
 
 
+class MacroResolvingContext(ConfiguredContext):
+    def __init__(self, config):
+        super().__init__(config)
+
+    @contextproperty
+    def var(self) -> ConfiguredVar:
+        return ConfiguredVar(
+            self._ctx, self.config, self.config.project_name
+        )
+
+
 def generate_schema_yml(
     config: AdapterRequiredConfig, project_name: str
 ) -> Dict[str, Any]:
     ctx = SchemaYamlContext(config, project_name)
     return ctx.to_dict()
+
+
+def generate_macro_context(
+    config: AdapterRequiredConfig,
+) -> Dict[str, Any]:
+    ctx = MacroResolvingContext(config)
+    return ctx.to_dict()

core/dbt/context/context_config.py

@@ -41,6 +41,8 @@ class UnrenderedConfig(ConfigSource):
             model_configs = unrendered.get('snapshots')
         elif resource_type == NodeType.Source:
             model_configs = unrendered.get('sources')
+        elif resource_type == NodeType.Test:
+            model_configs = unrendered.get('tests')
         else:
             model_configs = unrendered.get('models')
 
@@ -61,6 +63,8 @@ class RenderedConfig(ConfigSource):
             model_configs = self.project.snapshots
         elif resource_type == NodeType.Source:
             model_configs = self.project.sources
+        elif resource_type == NodeType.Test:
+            model_configs = self.project.tests
         else:
             model_configs = self.project.models
         return model_configs
@@ -93,7 +97,7 @@ class BaseContextConfigGenerator(Generic[T]):
             result = {}
             for key, value in level_config.items():
                 if key.startswith('+'):
-                    result[key[1:]] = deepcopy(value)
+                    result[key[1:].strip()] = deepcopy(value)
                 elif not isinstance(value, dict):
                     result[key] = deepcopy(value)
 

core/dbt/context/docs.py

@@ -57,14 +57,19 @@ class DocsRuntimeContext(SchemaYamlContext):
         else:
             doc_invalid_args(self.node, args)
 
+        # ParsedDocumentation
         target_doc = self.manifest.resolve_doc(
             doc_name,
             doc_package_name,
             self._project_name,
             self.node.package_name,
         )
-
-        if target_doc is None:
+        if target_doc:
+            file_id = target_doc.file_id
+            if file_id in self.manifest.files:
+                source_file = self.manifest.files[file_id]
+                source_file.add_node(self.node.unique_id)
+        else:
             doc_target_not_found(self.node, doc_name, doc_package_name)
 
         return target_doc.block_contents

core/dbt/context/macro_resolver.py

@@ -14,8 +14,12 @@ MacroNamespace = Dict[str, ParsedMacro]
 # so that higher precedence macros are found first.
 # This functionality is also provided by the MacroNamespace,
 # but the intention is to eventually replace that class.
-# This enables us to get the macor unique_id without
+# This enables us to get the macro unique_id without
 # processing every macro in the project.
+# Note: the root project macros override everything in the
+# dbt internal projects. External projects (dependencies) will
+# use their own macros first, then pull from the root project
+# followed by dbt internal projects.
 class MacroResolver:
     def __init__(
         self,
@@ -48,18 +52,29 @@ class MacroResolver:
                 self.internal_packages_namespace.update(
                     self.internal_packages[pkg])
 
+    # search order:
+    # local_namespace (package of particular node), not including
+    #    the internal packages or the root package
+    #    This means that within an extra package, it uses its own macros
+    # root package namespace
+    # non-internal packages (that aren't local or root)
+    # dbt internal packages
     def _build_macros_by_name(self):
         macros_by_name = {}
-        # search root package macros
-        for macro in self.root_package_macros.values():
+
+        # all internal packages (already in the right order)
+        for macro in self.internal_packages_namespace.values():
             macros_by_name[macro.name] = macro
-        # search miscellaneous non-internal packages
+
+        # non-internal packages
         for fnamespace in self.packages.values():
             for macro in fnamespace.values():
                 macros_by_name[macro.name] = macro
-        # search all internal packages
-        for macro in self.internal_packages_namespace.values():
+
+        # root package macros
+        for macro in self.root_package_macros.values():
             macros_by_name[macro.name] = macro
+
         self.macros_by_name = macros_by_name
 
     def _add_macro_to(
@@ -97,18 +112,26 @@ class MacroResolver:
         for macro in self.macros.values():
             self.add_macro(macro)
 
-    def get_macro_id(self, local_package, macro_name):
+    def get_macro(self, local_package, macro_name):
         local_package_macros = {}
         if (local_package not in self.internal_package_names and
                 local_package in self.packages):
             local_package_macros = self.packages[local_package]
         # First: search the local packages for this macro
         if macro_name in local_package_macros:
-            return local_package_macros[macro_name].unique_id
+            return local_package_macros[macro_name]
+        # Now look up in the standard search order
         if macro_name in self.macros_by_name:
-            return self.macros_by_name[macro_name].unique_id
+            return self.macros_by_name[macro_name]
         return None
 
+    def get_macro_id(self, local_package, macro_name):
+        macro = self.get_macro(local_package, macro_name)
+        if macro is None:
+            return None
+        else:
+            return macro.unique_id
+
 
 # Currently this is just used by test processing in the schema
 # parser (in connection with the MacroResolver). Future work
@@ -122,16 +145,37 @@ class TestMacroNamespace:
     ):
         self.macro_resolver = macro_resolver
         self.ctx = ctx
-        self.node = node
+        self.node = node  # can be none
         self.thread_ctx = thread_ctx
-        local_namespace = {}
+        self.local_namespace = {}
+        self.project_namespace = {}
         if depends_on_macros:
-            for macro_unique_id in depends_on_macros:
-                macro = self.manifest.macros[macro_unique_id]
-                local_namespace[macro.name] = MacroGenerator(
-                    macro, self.ctx, self.node, self.thread_ctx,
-                )
-        self.local_namespace = local_namespace
+            dep_macros = []
+            self.recursively_get_depends_on_macros(depends_on_macros, dep_macros)
+            for macro_unique_id in dep_macros:
+                if macro_unique_id in self.macro_resolver.macros:
+                    # Split up the macro unique_id to get the project_name
+                    (_, project_name, macro_name) = macro_unique_id.split('.')
+                    # Save the plain macro_name in the local_namespace
+                    macro = self.macro_resolver.macros[macro_unique_id]
+                    macro_gen = MacroGenerator(
+                        macro, self.ctx, self.node, self.thread_ctx,
+                    )
+                    self.local_namespace[macro_name] = macro_gen
+                    # We also need the two part macro name
+                    if project_name not in self.project_namespace:
+                        self.project_namespace[project_name] = {}
+                    self.project_namespace[project_name][macro_name] = macro_gen
+
+    def recursively_get_depends_on_macros(self, depends_on_macros, dep_macros):
+        for macro_unique_id in depends_on_macros:
+            if macro_unique_id in dep_macros:
+                continue
+            dep_macros.append(macro_unique_id)
+            if macro_unique_id in self.macro_resolver.macros:
+                macro = self.macro_resolver.macros[macro_unique_id]
+                if macro.depends_on.macros:
+                    self.recursively_get_depends_on_macros(macro.depends_on.macros, dep_macros)
 
     def get_from_package(
         self, package_name: Optional[str], name: str
@@ -141,12 +185,14 @@ class TestMacroNamespace:
             macro = self.macro_resolver.macros_by_name.get(name)
         elif package_name == GLOBAL_PROJECT_NAME:
             macro = self.macro_resolver.internal_packages_namespace.get(name)
-        elif package_name in self.resolver.packages:
+        elif package_name in self.macro_resolver.packages:
             macro = self.macro_resolver.packages[package_name].get(name)
         else:
             raise_compiler_error(
                 f"Could not find package '{package_name}'"
             )
+        if not macro:
+            return None
         macro_func = MacroGenerator(
             macro, self.ctx, self.node, self.thread_ctx
         )

core/dbt/context/macros.py

@@ -19,13 +19,17 @@ FullNamespace = Dict[str, NamespaceMember]
 # and provide the ability to flatten them into the ManifestContexts
 # that are created for jinja, so that macro calls can be resolved.
 # Creates special iterators and _keys methods to flatten the lists.
+# When this class is created it has a static 'local_namespace' which
+# depends on the package of the node, so it only works for one
+# particular local package at a time for "flattening" into a context.
+# 'get_by_package' should work for any macro.
 class MacroNamespace(Mapping):
     def __init__(
         self,
-        global_namespace: FlatNamespace,
-        local_namespace: FlatNamespace,
-        global_project_namespace: FlatNamespace,
-        packages: Dict[str, FlatNamespace],
+        global_namespace: FlatNamespace,  # root package macros
+        local_namespace: FlatNamespace,   # packages for *this* node
+        global_project_namespace: FlatNamespace,  # internal packages
+        packages: Dict[str, FlatNamespace],  # non-internal packages
     ):
         self.global_namespace: FlatNamespace = global_namespace
         self.local_namespace: FlatNamespace = local_namespace
@@ -33,13 +37,13 @@ class MacroNamespace(Mapping):
         self.global_project_namespace: FlatNamespace = global_project_namespace
 
     def _search_order(self) -> Iterable[Union[FullNamespace, FlatNamespace]]:
-        yield self.local_namespace
-        yield self.global_namespace
-        yield self.packages
+        yield self.local_namespace   # local package
+        yield self.global_namespace  # root package
+        yield self.packages          # non-internal packages
         yield {
-            GLOBAL_PROJECT_NAME: self.global_project_namespace,
+            GLOBAL_PROJECT_NAME: self.global_project_namespace,  # dbt
         }
-        yield self.global_project_namespace
+        yield self.global_project_namespace  # other internal project besides dbt
 
     # provides special keys method for MacroNamespace iterator
     # returns keys from local_namespace, global_namespace, packages,
@@ -98,7 +102,9 @@ class MacroNamespaceBuilder:
         # internal packages comes from get_adapter_package_names
         self.internal_package_names = set(internal_packages)
         self.internal_package_names_order = internal_packages
-        # macro_func is added here if in root package
+        # macro_func is added here if in root package, since
+        # the root package acts as a "global" namespace, overriding
+        # everything else except local external package macro calls
         self.globals: FlatNamespace = {}
         # macro_func is added here if it's the package for this node
         self.locals: FlatNamespace = {}
@@ -169,8 +175,8 @@ class MacroNamespaceBuilder:
                 global_project_namespace.update(self.internal_packages[pkg])
 
         return MacroNamespace(
-            global_namespace=self.globals,
-            local_namespace=self.locals,
-            global_project_namespace=global_project_namespace,
-            packages=self.packages,
+            global_namespace=self.globals,  # root package macros
+            local_namespace=self.locals,  # packages for *this* node
+            global_project_namespace=global_project_namespace,  # internal packages
+            packages=self.packages,  # non internal_packages
         )

core/dbt/context/manifest.py

@@ -2,7 +2,7 @@ from typing import List
 
 from dbt.clients.jinja import MacroStack
 from dbt.contracts.connection import AdapterRequiredConfig
-from dbt.contracts.graph.manifest import Manifest, AnyManifest
+from dbt.contracts.graph.manifest import Manifest
 from dbt.context.macro_resolver import TestMacroNamespace
 
 
@@ -20,7 +20,7 @@ class ManifestContext(ConfiguredContext):
     def __init__(
         self,
         config: AdapterRequiredConfig,
-        manifest: AnyManifest,
+        manifest: Manifest,
         search_package: str,
     ) -> None:
         super().__init__(config)
@@ -62,6 +62,7 @@ class ManifestContext(ConfiguredContext):
         # keys in the manifest dictionary
         if isinstance(self.namespace, TestMacroNamespace):
             dct.update(self.namespace.local_namespace)
+            dct.update(self.namespace.project_namespace)
         else:
             dct.update(self.namespace)
         return dct

core/dbt/context/providers.py

@@ -8,7 +8,9 @@ from typing_extensions import Protocol
 
 from dbt import deprecations
 from dbt.adapters.base.column import Column
-from dbt.adapters.factory import get_adapter, get_adapter_package_names
+from dbt.adapters.factory import (
+    get_adapter, get_adapter_package_names, get_adapter_type_names
+)
 from dbt.clients import agate_helper
 from dbt.clients.jinja import get_rendered, MacroGenerator, MacroStack
 from dbt.config import RuntimeConfig, Project
@@ -20,7 +22,7 @@ from .macros import MacroNamespaceBuilder, MacroNamespace
 from .manifest import ManifestContext
 from dbt.contracts.connection import AdapterResponse
 from dbt.contracts.graph.manifest import (
-    Manifest, AnyManifest, Disabled, MacroManifest
+    Manifest, Disabled
 )
 from dbt.contracts.graph.compiled import (
     CompiledResource,
@@ -107,14 +109,18 @@ class BaseDatabaseWrapper:
         return self._adapter.commit_if_has_connection()
 
     def _get_adapter_macro_prefixes(self) -> List[str]:
-        # a future version of this could have plugins automatically call fall
-        # back to their dependencies' dependencies by using
-        # `get_adapter_type_names` instead of `[self.config.credentials.type]`
-        search_prefixes = [self._adapter.type(), 'default']
+        # order matters for dispatch:
+        #  1. current adapter
+        #  2. any parent adapters (dependencies)
+        #  3. 'default'
+        search_prefixes = get_adapter_type_names(self._adapter.type()) + ['default']
         return search_prefixes
 
     def dispatch(
-        self, macro_name: str, packages: Optional[List[str]] = None
+        self,
+        macro_name: str,
+        macro_namespace: Optional[str] = None,
+        packages: Optional[List[str]] = None,
     ) -> MacroGenerator:
         search_packages: List[Optional[str]]
 
@@ -128,15 +134,25 @@ class BaseDatabaseWrapper:
             )
             raise CompilationException(msg)
 
-        if packages is None:
+        if packages is not None:
+            deprecations.warn('dispatch-packages', macro_name=macro_name)
+
+        namespace = packages if packages else macro_namespace
+
+        if namespace is None:
             search_packages = [None]
-        elif isinstance(packages, str):
-            raise CompilationException(
-                f'In adapter.dispatch, got a string packages argument '
-                f'("{packages}"), but packages should be None or a list.'
-            )
+        elif isinstance(namespace, str):
+            search_packages = self._adapter.config.get_macro_search_order(namespace)
+            if not search_packages and namespace in self._adapter.config.dependencies:
+                search_packages = [namespace]
+            if not search_packages:
+                raise CompilationException(
+                    f'In adapter.dispatch, got a string packages argument '
+                    f'("{packages}"), but packages should be None or a list.'
+                )
         else:
-            search_packages = packages
+            # Not a string and not None so must be a list
+            search_packages = namespace
 
         attempts = []
 
@@ -1179,14 +1195,13 @@ class ProviderContext(ManifestContext):
         """
         deprecations.warn('adapter-macro', macro_name=name)
         original_name = name
-        package_names: Optional[List[str]] = None
+        package_name = None
         if '.' in name:
             package_name, name = name.split('.', 1)
-            package_names = [package_name]
 
         try:
             macro = self.db_wrapper.dispatch(
-                macro_name=name, packages=package_names
+                macro_name=name, macro_namespace=package_name
             )
         except CompilationException as exc:
             raise CompilationException(
@@ -1210,7 +1225,7 @@ class MacroContext(ProviderContext):
         self,
         model: ParsedMacro,
         config: RuntimeConfig,
-        manifest: AnyManifest,
+        manifest: Manifest,
         provider: Provider,
         search_package: Optional[str],
     ) -> None:
@@ -1300,7 +1315,7 @@ class ModelContext(ProviderContext):
 def generate_parser_model(
     model: ManifestNode,
     config: RuntimeConfig,
-    manifest: MacroManifest,
+    manifest: Manifest,
     context_config: ContextConfig,
 ) -> Dict[str, Any]:
     # The __init__ method of ModelContext also initializes
@@ -1317,7 +1332,7 @@ def generate_parser_model(
 def generate_generate_component_name_macro(
     macro: ParsedMacro,
     config: RuntimeConfig,
-    manifest: MacroManifest,
+    manifest: Manifest,
 ) -> Dict[str, Any]:
     ctx = MacroContext(
         macro, config, manifest, GenerateNameProvider(), None
@@ -1370,7 +1385,7 @@ class ExposureSourceResolver(BaseResolver):
 def generate_parse_exposure(
     exposure: ParsedExposure,
     config: RuntimeConfig,
-    manifest: MacroManifest,
+    manifest: Manifest,
     package_name: str,
 ) -> Dict[str, Any]:
     project = config.load_dependencies()[package_name]
@@ -1408,7 +1423,12 @@ class TestContext(ProviderContext):
         self.macro_resolver = macro_resolver
         self.thread_ctx = MacroStack()
         super().__init__(model, config, manifest, provider, context_config)
-        self._build_test_namespace
+        self._build_test_namespace()
+        # We need to rebuild this because it's already been built by
+        # the ProviderContext with the wrong namespace.
+        self.db_wrapper = self.provider.DatabaseWrapper(
+            self.adapter, self.namespace
+        )
 
     def _build_namespace(self):
         return {}
@@ -1421,11 +1441,17 @@ class TestContext(ProviderContext):
         depends_on_macros = []
         if self.model.depends_on and self.model.depends_on.macros:
             depends_on_macros = self.model.depends_on.macros
+        lookup_macros = depends_on_macros.copy()
+        for macro_unique_id in lookup_macros:
+            lookup_macro = self.macro_resolver.macros.get(macro_unique_id)
+            if lookup_macro:
+                depends_on_macros.extend(lookup_macro.depends_on.macros)
+
         macro_namespace = TestMacroNamespace(
-            self.macro_resolver, self.ctx, self.node, self.thread_ctx,
+            self.macro_resolver, self._ctx, self.model, self.thread_ctx,
             depends_on_macros
         )
-        self._namespace = macro_namespace
+        self.namespace = macro_namespace
 
 
 def generate_test_context(

core/dbt/contracts/connection.py

@@ -9,7 +9,7 @@ from dbt.utils import translate_aliases
 from dbt.logger import GLOBAL_LOGGER as logger
 from typing_extensions import Protocol
 from dbt.dataclass_schema import (
-    dbtClassMixin, StrEnum, ExtensibleDbtClassMixin,
+    dbtClassMixin, StrEnum, ExtensibleDbtClassMixin, HyphenatedDbtClassMixin,
     ValidatedStringMixin, register_pattern
 )
 from dbt.contracts.util import Replaceable
@@ -212,9 +212,10 @@ DEFAULT_QUERY_COMMENT = '''
 
 
 @dataclass
-class QueryComment(dbtClassMixin):
+class QueryComment(HyphenatedDbtClassMixin):
     comment: str = DEFAULT_QUERY_COMMENT
     append: bool = False
+    job_label: bool = False
 
 
 class AdapterRequiredConfig(HasCredentials, Protocol):

core/dbt/contracts/files.py

@@ -1,19 +1,43 @@
 import hashlib
 import os
 from dataclasses import dataclass, field
-from typing import List, Optional, Union
+from mashumaro.types import SerializableType
+from typing import List, Optional, Union, Dict, Any
 
-from dbt.dataclass_schema import dbtClassMixin
+from dbt.dataclass_schema import dbtClassMixin, StrEnum
 
-from dbt.exceptions import InternalException
-
-from .util import MacroKey, SourceKey
+from .util import SourceKey
 
 
 MAXIMUM_SEED_SIZE = 1 * 1024 * 1024
 MAXIMUM_SEED_SIZE_NAME = '1MB'
 
 
+class ParseFileType(StrEnum):
+    Macro = 'macro'
+    Model = 'model'
+    Snapshot = 'snapshot'
+    Analysis = 'analysis'
+    Test = 'test'
+    Seed = 'seed'
+    Documentation = 'docs'
+    Schema = 'schema'
+    Hook = 'hook'   # not a real filetype, from dbt_project.yml
+
+
+parse_file_type_to_parser = {
+    ParseFileType.Macro: 'MacroParser',
+    ParseFileType.Model: 'ModelParser',
+    ParseFileType.Snapshot: 'SnapshotParser',
+    ParseFileType.Analysis: 'AnalysisParser',
+    ParseFileType.Test: 'DataTestParser',
+    ParseFileType.Seed: 'SeedParser',
+    ParseFileType.Documentation: 'DocumentationParser',
+    ParseFileType.Schema: 'SchemaParser',
+    ParseFileType.Hook: 'HookParser',
+}
+
+
 @dataclass
 class FilePath(dbtClassMixin):
     searched_path: str
@@ -110,48 +134,57 @@ class RemoteFile(dbtClassMixin):
 
 
 @dataclass
-class SourceFile(dbtClassMixin):
+class BaseSourceFile(dbtClassMixin, SerializableType):
     """Define a source file in dbt"""
     path: Union[FilePath, RemoteFile]  # the path information
     checksum: FileHash
+    # Seems like knowing which project the file came from would be useful
+    project_name: Optional[str] = None
+    # Parse file type: i.e. which parser will process this file
+    parse_file_type: Optional[ParseFileType] = None
     # we don't want to serialize this
-    _contents: Optional[str] = None
+    contents: Optional[str] = None
     # the unique IDs contained in this file
-    nodes: List[str] = field(default_factory=list)
-    docs: List[str] = field(default_factory=list)
-    macros: List[str] = field(default_factory=list)
-    sources: List[str] = field(default_factory=list)
-    exposures: List[str] = field(default_factory=list)
-    # any node patches in this file. The entries are names, not unique ids!
-    patches: List[str] = field(default_factory=list)
-    # any macro patches in this file. The entries are package, name pairs.
-    macro_patches: List[MacroKey] = field(default_factory=list)
-    # any source patches in this file. The entries are package, name pairs
-    source_patches: List[SourceKey] = field(default_factory=list)
 
     @property
-    def search_key(self) -> Optional[str]:
+    def file_id(self):
         if isinstance(self.path, RemoteFile):
             return None
         if self.checksum.name == 'none':
             return None
-        return self.path.search_key
+        return f'{self.project_name}://{self.path.original_file_path}'
 
-    @property
-    def contents(self) -> str:
-        if self._contents is None:
-            raise InternalException('SourceFile has no contents!')
-        return self._contents
-
-    @contents.setter
-    def contents(self, value):
-        self._contents = value
+    def _serialize(self):
+        dct = self.to_dict()
+        return dct
 
     @classmethod
-    def empty(cls, path: FilePath) -> 'SourceFile':
-        self = cls(path=path, checksum=FileHash.empty())
-        self.contents = ''
-        return self
+    def _deserialize(cls, dct: Dict[str, int]):
+        if dct['parse_file_type'] == 'schema':
+            sf = SchemaSourceFile.from_dict(dct)
+        else:
+            sf = SourceFile.from_dict(dct)
+        return sf
+
+    def __post_serialize__(self, dct):
+        dct = super().__post_serialize__(dct)
+        # remove empty lists to save space
+        dct_keys = list(dct.keys())
+        for key in dct_keys:
+            if isinstance(dct[key], list) and not dct[key]:
+                del dct[key]
+        # remove contents. Schema files will still have 'dict_from_yaml'
+        # from the contents
+        if 'contents' in dct:
+            del dct['contents']
+        return dct
+
+
+@dataclass
+class SourceFile(BaseSourceFile):
+    nodes: List[str] = field(default_factory=list)
+    docs: List[str] = field(default_factory=list)
+    macros: List[str] = field(default_factory=list)
 
     @classmethod
     def big_seed(cls, path: FilePath) -> 'SourceFile':
@@ -160,8 +193,106 @@ class SourceFile(dbtClassMixin):
         self.contents = ''
         return self
 
+    def add_node(self, value):
+        if value not in self.nodes:
+            self.nodes.append(value)
+
+    # TODO: do this a different way. This remote file kludge isn't going
+    # to work long term
     @classmethod
-    def remote(cls, contents: str) -> 'SourceFile':
-        self = cls(path=RemoteFile(), checksum=FileHash.empty())
-        self.contents = contents
+    def remote(cls, contents: str, project_name: str) -> 'SourceFile':
+        self = cls(
+            path=RemoteFile(),
+            checksum=FileHash.from_contents(contents),
+            project_name=project_name,
+            contents=contents,
+        )
         return self
+
+
+@dataclass
+class SchemaSourceFile(BaseSourceFile):
+    dfy: Dict[str, Any] = field(default_factory=dict)
+    # these are in the manifest.nodes dictionary
+    tests: Dict[str, Any] = field(default_factory=dict)
+    sources: List[str] = field(default_factory=list)
+    exposures: List[str] = field(default_factory=list)
+    # node patches contain models, seeds, snapshots, analyses
+    ndp: List[str] = field(default_factory=list)
+    # any macro patches in this file by macro unique_id.
+    mcp: List[str] = field(default_factory=list)
+    # any source patches in this file. The entries are package, name pairs
+    # Patches are only against external sources. Sources can be
+    # created too, but those are in 'sources'
+    sop: List[SourceKey] = field(default_factory=list)
+    pp_dict: Optional[Dict[str, Any]] = None
+    pp_test_index: Optional[Dict[str, Any]] = None
+
+    @property
+    def dict_from_yaml(self):
+        return self.dfy
+
+    @property
+    def node_patches(self):
+        return self.ndp
+
+    @property
+    def macro_patches(self):
+        return self.mcp
+
+    @property
+    def source_patches(self):
+        return self.sop
+
+    def __post_serialize__(self, dct):
+        dct = super().__post_serialize__(dct)
+        # Remove partial parsing specific data
+        for key in ('pp_files', 'pp_test_index', 'pp_dict'):
+            if key in dct:
+                del dct[key]
+        return dct
+
+    def append_patch(self, yaml_key, unique_id):
+        self.node_patches.append(unique_id)
+
+    def add_test(self, node_unique_id, test_from):
+        name = test_from['name']
+        key = test_from['key']
+        if key not in self.tests:
+            self.tests[key] = {}
+        if name not in self.tests[key]:
+            self.tests[key][name] = []
+        self.tests[key][name].append(node_unique_id)
+
+    def remove_tests(self, yaml_key, name):
+        if yaml_key in self.tests:
+            if name in self.tests[yaml_key]:
+                del self.tests[yaml_key][name]
+
+    def get_tests(self, yaml_key, name):
+        if yaml_key in self.tests:
+            if name in self.tests[yaml_key]:
+                return self.tests[yaml_key][name]
+        return []
+
+    def get_key_and_name_for_test(self, test_unique_id):
+        yaml_key = None
+        block_name = None
+        for key in self.tests.keys():
+            for name in self.tests[key]:
+                for unique_id in self.tests[key][name]:
+                    if unique_id == test_unique_id:
+                        yaml_key = key
+                        block_name = name
+                        break
+        return (yaml_key, block_name)
+
+    def get_all_test_ids(self):
+        test_ids = []
+        for key in self.tests.keys():
+            for name in self.tests[key]:
+                test_ids.extend(self.tests[key][name])
+        return test_ids
+
+
+AnySourceFile = Union[SchemaSourceFile, SourceFile]

core/dbt/contracts/graph/compiled.py

@@ -43,6 +43,7 @@ class CompiledNode(ParsedNode, CompiledNodeMixin):
     extra_ctes_injected: bool = False
     extra_ctes: List[InjectedCTE] = field(default_factory=list)
     relation_name: Optional[str] = None
+    _pre_injected_sql: Optional[str] = None
 
     def set_cte(self, cte_id: str, sql: str):
         """This is the equivalent of what self.extra_ctes[cte_id] = sql would
@@ -55,6 +56,12 @@ class CompiledNode(ParsedNode, CompiledNodeMixin):
         else:
             self.extra_ctes.append(InjectedCTE(id=cte_id, sql=sql))
 
+    def __post_serialize__(self, dct):
+        dct = super().__post_serialize__(dct)
+        if '_pre_injected_sql' in dct:
+            del dct['_pre_injected_sql']
+        return dct
+
 
 @dataclass
 class CompiledAnalysisNode(CompiledNode):
@@ -112,15 +119,6 @@ class CompiledSchemaTestNode(CompiledNode, HasTestMetadata):
     column_name: Optional[str] = None
     config: TestConfig = field(default_factory=TestConfig)
 
-    def same_config(self, other) -> bool:
-        return (
-            self.unrendered_config.get('severity') ==
-            other.unrendered_config.get('severity')
-        )
-
-    def same_column_name(self, other) -> bool:
-        return self.column_name == other.column_name
-
     def same_contents(self, other) -> bool:
         if other is None:
             return False

core/dbt/contracts/graph/manifest.py

@@ -1,11 +1,11 @@
-import abc
 import enum
 from dataclasses import dataclass, field
 from itertools import chain, islice
+from mashumaro import DataClassMessagePackMixin
 from multiprocessing.synchronize import Lock
 from typing import (
     Dict, List, Optional, Union, Mapping, MutableMapping, Any, Set, Tuple,
-    TypeVar, Callable, Iterable, Generic, cast, AbstractSet
+    TypeVar, Callable, Iterable, Generic, cast, AbstractSet, ClassVar
 )
 from typing_extensions import Protocol
 from uuid import UUID
@@ -15,20 +15,25 @@ from dbt.contracts.graph.compiled import (
 )
 from dbt.contracts.graph.parsed import (
     ParsedMacro, ParsedDocumentation, ParsedNodePatch, ParsedMacroPatch,
-    ParsedSourceDefinition, ParsedExposure
+    ParsedSourceDefinition, ParsedExposure, HasUniqueID,
+    UnpatchedSourceDefinition, ManifestNodes
 )
-from dbt.contracts.files import SourceFile
+from dbt.contracts.graph.unparsed import SourcePatch
+from dbt.contracts.files import SourceFile, SchemaSourceFile, FileHash, AnySourceFile
 from dbt.contracts.util import (
-    BaseArtifactMetadata, MacroKey, SourceKey, ArtifactMixin, schema_version
+    BaseArtifactMetadata, SourceKey, ArtifactMixin, schema_version
 )
+from dbt.dataclass_schema import dbtClassMixin
 from dbt.exceptions import (
+    CompilationException,
     raise_duplicate_resource_name, raise_compiler_error, warn_or_error,
-    raise_invalid_patch,
+    raise_duplicate_patch_name,
+    raise_duplicate_macro_patch_name, raise_duplicate_source_patch_name,
 )
 from dbt.helper_types import PathSet
 from dbt.logger import GLOBAL_LOGGER as logger
 from dbt.node_types import NodeType
-from dbt import deprecations
+from dbt.ui import line_wrap_message
 from dbt import flags
 from dbt import tracking
 import dbt.utils
@@ -40,72 +45,70 @@ RefName = str
 UniqueID = str
 
 
-K_T = TypeVar('K_T')
-V_T = TypeVar('V_T')
-
-
-class PackageAwareCache(Generic[K_T, V_T]):
-    def __init__(self, manifest: 'Manifest'):
-        self.storage: Dict[K_T, Dict[PackageName, UniqueID]] = {}
-        self._manifest = manifest
-        self.populate()
-
-    @abc.abstractmethod
-    def populate(self):
-        pass
-
-    @abc.abstractmethod
-    def perform_lookup(self, unique_id: UniqueID) -> V_T:
-        pass
-
-    def find_cached_value(
-        self, key: K_T, package: Optional[PackageName]
-    ) -> Optional[V_T]:
-        unique_id = self.find_unique_id_for_package(key, package)
-        if unique_id is not None:
-            return self.perform_lookup(unique_id)
+def find_unique_id_for_package(storage, key, package: Optional[PackageName]):
+    if key not in storage:
         return None
 
-    def find_unique_id_for_package(
-        self, key: K_T, package: Optional[PackageName]
-    ) -> Optional[UniqueID]:
-        if key not in self.storage:
+    pkg_dct: Mapping[PackageName, UniqueID] = storage[key]
+
+    if package is None:
+        if not pkg_dct:
             return None
-
-        pkg_dct: Mapping[PackageName, UniqueID] = self.storage[key]
-
-        if package is None:
-            if not pkg_dct:
-                return None
-            else:
-                return next(iter(pkg_dct.values()))
-        elif package in pkg_dct:
-            return pkg_dct[package]
         else:
-            return None
+            return next(iter(pkg_dct.values()))
+    elif package in pkg_dct:
+        return pkg_dct[package]
+    else:
+        return None
 
 
-class DocCache(PackageAwareCache[DocName, ParsedDocumentation]):
+class DocLookup(dbtClassMixin):
+    def __init__(self, manifest: 'Manifest'):
+        self.storage: Dict[str, Dict[PackageName, UniqueID]] = {}
+        self.populate(manifest)
+
+    def get_unique_id(self, key, package: Optional[PackageName]):
+        return find_unique_id_for_package(self.storage, key, package)
+
+    def find(self, key, package: Optional[PackageName], manifest: 'Manifest'):
+        unique_id = self.get_unique_id(key, package)
+        if unique_id is not None:
+            return self.perform_lookup(unique_id, manifest)
+        return None
+
     def add_doc(self, doc: ParsedDocumentation):
         if doc.name not in self.storage:
             self.storage[doc.name] = {}
         self.storage[doc.name][doc.package_name] = doc.unique_id
 
-    def populate(self):
-        for doc in self._manifest.docs.values():
+    def populate(self, manifest):
+        for doc in manifest.docs.values():
             self.add_doc(doc)
 
     def perform_lookup(
-        self, unique_id: UniqueID
+        self, unique_id: UniqueID, manifest
     ) -> ParsedDocumentation:
-        if unique_id not in self._manifest.docs:
+        if unique_id not in manifest.docs:
             raise dbt.exceptions.InternalException(
                 f'Doc {unique_id} found in cache but not found in manifest'
             )
-        return self._manifest.docs[unique_id]
+        return manifest.docs[unique_id]
 
 
-class SourceCache(PackageAwareCache[SourceKey, ParsedSourceDefinition]):
+class SourceLookup(dbtClassMixin):
+    def __init__(self, manifest: 'Manifest'):
+        self.storage: Dict[Tuple[str, str], Dict[PackageName, UniqueID]] = {}
+        self.populate(manifest)
+
+    def get_unique_id(self, key, package: Optional[PackageName]):
+        return find_unique_id_for_package(self.storage, key, package)
+
+    def find(self, key, package: Optional[PackageName], manifest: 'Manifest'):
+        unique_id = self.get_unique_id(key, package)
+        if unique_id is not None:
+            return self.perform_lookup(unique_id, manifest)
+        return None
+
     def add_source(self, source: ParsedSourceDefinition):
         key = (source.source_name, source.name)
         if key not in self.storage:
@@ -113,46 +116,63 @@ class SourceCache(PackageAwareCache[SourceKey, ParsedSourceDefinition]):
 
         self.storage[key][source.package_name] = source.unique_id
 
-    def populate(self):
-        for source in self._manifest.sources.values():
-            self.add_source(source)
+    def populate(self, manifest):
+        for source in manifest.sources.values():
+            if hasattr(source, 'source_name'):
+                self.add_source(source)
 
     def perform_lookup(
-        self, unique_id: UniqueID
+        self, unique_id: UniqueID, manifest: 'Manifest'
     ) -> ParsedSourceDefinition:
-        if unique_id not in self._manifest.sources:
+        if unique_id not in manifest.sources:
             raise dbt.exceptions.InternalException(
                 f'Source {unique_id} found in cache but not found in manifest'
             )
-        return self._manifest.sources[unique_id]
+        return manifest.sources[unique_id]
 
 
-class RefableCache(PackageAwareCache[RefName, ManifestNode]):
+class RefableLookup(dbtClassMixin):
+    # model, seed, snapshot
+    _lookup_types: ClassVar[set] = set(NodeType.refable())
+
     # refables are actually unique, so the Dict[PackageName, UniqueID] will
     # only ever have exactly one value, but doing 3 dict lookups instead of 1
     # is not a big deal at all and retains consistency
     def __init__(self, manifest: 'Manifest'):
-        self._cached_types = set(NodeType.refable())
-        super().__init__(manifest)
+        self.storage: Dict[str, Dict[PackageName, UniqueID]] = {}
+        self.populate(manifest)
+
+    def get_unique_id(self, key, package: Optional[PackageName]):
+        return find_unique_id_for_package(self.storage, key, package)
+
+    def find(self, key, package: Optional[PackageName], manifest: 'Manifest'):
+        unique_id = self.get_unique_id(key, package)
+        if unique_id is not None:
+            return self.perform_lookup(unique_id, manifest)
+        return None
 
     def add_node(self, node: ManifestNode):
-        if node.resource_type in self._cached_types:
+        if node.resource_type in self._lookup_types:
             if node.name not in self.storage:
                 self.storage[node.name] = {}
             self.storage[node.name][node.package_name] = node.unique_id
 
-    def populate(self):
-        for node in self._manifest.nodes.values():
+    def populate(self, manifest):
+        for node in manifest.nodes.values():
             self.add_node(node)
 
     def perform_lookup(
-        self, unique_id: UniqueID
+        self, unique_id: UniqueID, manifest
     ) -> ManifestNode:
-        if unique_id not in self._manifest.nodes:
+        if unique_id not in manifest.nodes:
             raise dbt.exceptions.InternalException(
                 f'Node {unique_id} found in cache but not found in manifest'
             )
-        return self._manifest.nodes[unique_id]
+        return manifest.nodes[unique_id]
+
+
+class AnalysisLookup(RefableLookup):
+    _lookup_types: ClassVar[set] = set(NodeType.Analysis)
 
 
 def _search_packages(
@@ -223,7 +243,7 @@ def _sort_values(dct):
     return {k: sorted(v) for k, v in dct.items()}
 
 
-def build_edges(nodes: List[ManifestNode]):
+def build_node_edges(nodes: List[ManifestNode]):
     """Build the forward and backward edges on the given list of ParsedNodes
     and return them as two separate dictionaries, each mapping unique IDs to
     lists of edges.
@@ -239,6 +259,18 @@ def build_edges(nodes: List[ManifestNode]):
     return _sort_values(forward_edges), _sort_values(backward_edges)
 
 
+# Build a map of children of macros
+def build_macro_edges(nodes: List[Any]):
+    forward_edges: Dict[str, List[str]] = {
+        n.unique_id: [] for n in nodes if n.unique_id.startswith('macro') or n.depends_on.macros
+    }
+    for node in nodes:
+        for unique_id in node.depends_on.macros:
+            if unique_id in forward_edges.keys():
+                forward_edges[unique_id].append(node.unique_id)
+    return _sort_values(forward_edges)
+
+
 def _deepcopy(value):
     return value.from_dict(value.to_dict(omit_none=True))
 
@@ -506,26 +538,71 @@ class MacroMethods:
 
 
 @dataclass
-class Manifest(MacroMethods):
+class ParsingInfo:
+    static_analysis_parsed_path_count: int = 0
+    static_analysis_path_count: int = 0
+
+
+@dataclass
+class ManifestStateCheck(dbtClassMixin):
+    vars_hash: FileHash = field(default_factory=FileHash.empty)
+    profile_hash: FileHash = field(default_factory=FileHash.empty)
+    project_hashes: MutableMapping[str, FileHash] = field(default_factory=dict)
+
+
+@dataclass
+class Manifest(MacroMethods, DataClassMessagePackMixin, dbtClassMixin):
     """The manifest for the full graph, after parsing and during compilation.
     """
     # These attributes are both positional and by keyword. If an attribute
     # is added it must all be added in the __reduce_ex__ method in the
     # args tuple in the right position.
-    nodes: MutableMapping[str, ManifestNode]
-    sources: MutableMapping[str, ParsedSourceDefinition]
-    macros: MutableMapping[str, ParsedMacro]
-    docs: MutableMapping[str, ParsedDocumentation]
-    exposures: MutableMapping[str, ParsedExposure]
-    selectors: MutableMapping[str, Any]
-    disabled: List[CompileResultNode]
-    files: MutableMapping[str, SourceFile]
+    nodes: MutableMapping[str, ManifestNode] = field(default_factory=dict)
+    sources: MutableMapping[str, ParsedSourceDefinition] = field(default_factory=dict)
+    macros: MutableMapping[str, ParsedMacro] = field(default_factory=dict)
+    docs: MutableMapping[str, ParsedDocumentation] = field(default_factory=dict)
+    exposures: MutableMapping[str, ParsedExposure] = field(default_factory=dict)
+    selectors: MutableMapping[str, Any] = field(default_factory=dict)
+    disabled: List[CompileResultNode] = field(default_factory=list)
+    files: MutableMapping[str, AnySourceFile] = field(default_factory=dict)
     metadata: ManifestMetadata = field(default_factory=ManifestMetadata)
     flat_graph: Dict[str, Any] = field(default_factory=dict)
-    _docs_cache: Optional[DocCache] = None
-    _sources_cache: Optional[SourceCache] = None
-    _refs_cache: Optional[RefableCache] = None
-    _lock: Lock = field(default_factory=flags.MP_CONTEXT.Lock)
+    state_check: ManifestStateCheck = field(default_factory=ManifestStateCheck)
+    # Moved from the ParseResult object
+    source_patches: MutableMapping[SourceKey, SourcePatch] = field(default_factory=dict)
+    # following is from ParseResult
+    _disabled: MutableMapping[str, List[CompileResultNode]] = field(default_factory=dict)
+    _doc_lookup: Optional[DocLookup] = field(
+        default=None, metadata={'serialize': lambda x: None, 'deserialize': lambda x: None}
+    )
+    _source_lookup: Optional[SourceLookup] = field(
+        default=None, metadata={'serialize': lambda x: None, 'deserialize': lambda x: None}
+    )
+    _ref_lookup: Optional[RefableLookup] = field(
+        default=None, metadata={'serialize': lambda x: None, 'deserialize': lambda x: None}
+    )
+    _analysis_lookup: Optional[AnalysisLookup] = field(
+        default=None, metadata={'serialize': lambda x: None, 'deserialize': lambda x: None}
+    )
+    _parsing_info: ParsingInfo = field(
+        default_factory=ParsingInfo,
+        metadata={'serialize': lambda x: None, 'deserialize': lambda x: None}
+    )
+    _lock: Lock = field(
+        default_factory=flags.MP_CONTEXT.Lock,
+        metadata={'serialize': lambda x: None, 'deserialize': lambda x: None}
+    )
+
+    def __pre_serialize__(self):
+        # serialization won't work with anything except an empty source_patches because
+        # tuple keys are not supported, so ensure it's empty
+        self.source_patches = {}
+        return self
+
+    @classmethod
+    def __post_deserialize__(cls, obj):
+        obj._lock = flags.MP_CONTEXT.Lock()
+        return obj
 
     def sync_update_node(
         self, new_node: NonSourceCompiledNode
@@ -563,6 +640,10 @@ class Manifest(MacroMethods):
         manifest!
         """
         self.flat_graph = {
+            'exposures': {
+                k: v.to_dict(omit_none=False)
+                for k, v in self.exposures.items()
+            },
             'nodes': {
                 k: v.to_dict(omit_none=False)
                 for k, v in self.nodes.items()
@@ -629,7 +710,7 @@ class Manifest(MacroMethods):
 
     def get_resource_fqns(self) -> Mapping[str, PathSet]:
         resource_fqns: Dict[str, Set[Tuple[str, ...]]] = {}
-        all_resources = chain(self.nodes.values(), self.sources.values())
+        all_resources = chain(self.exposures.values(), self.nodes.values(), self.sources.values())
         for resource in all_resources:
             resource_type_plural = resource.resource_type.pluralize()
             if resource_type_plural not in resource_fqns:
@@ -637,74 +718,59 @@ class Manifest(MacroMethods):
             resource_fqns[resource_type_plural].add(tuple(resource.fqn))
         return resource_fqns
 
-    def add_nodes(self, new_nodes: Mapping[str, ManifestNode]):
-        """Add the given dict of new nodes to the manifest."""
-        for unique_id, node in new_nodes.items():
-            if unique_id in self.nodes:
-                raise_duplicate_resource_name(node, self.nodes[unique_id])
-            self.nodes[unique_id] = node
-            # fixup the cache if it exists.
-            if self._refs_cache is not None:
-                if node.resource_type in NodeType.refable():
-                    self._refs_cache.add_node(node)
-
-    def patch_macros(
-        self, patches: MutableMapping[MacroKey, ParsedMacroPatch]
+    # This is called by 'parse_patch' in the NodePatchParser
+    def add_patch(
+        self, source_file: SchemaSourceFile, patch: ParsedNodePatch,
     ) -> None:
-        for macro in self.macros.values():
-            key = (macro.package_name, macro.name)
-            patch = patches.pop(key, None)
-            if not patch:
-                continue
-            macro.patch(patch)
-
-        if patches:
-            for patch in patches.values():
-                warn_or_error(
-                    f'WARNING: Found documentation for macro "{patch.name}" '
-                    f'which was not found'
-                )
-
-    def patch_nodes(
-        self, patches: MutableMapping[str, ParsedNodePatch]
-    ) -> None:
-        """Patch nodes with the given dict of patches. Note that this consumes
-        the input!
-        This relies on the fact that all nodes have unique _name_ fields, not
-        just unique unique_id fields.
-        """
-        # because we don't have any mapping from node _names_ to nodes, and we
-        # only have the node name in the patch, we have to iterate over all the
-        # nodes looking for matching names. We could use a NameSearcher if we
-        # were ok with doing an O(n*m) search (one nodes scan per patch)
-        for node in self.nodes.values():
-            patch = patches.pop(node.name, None)
-            if not patch:
-                continue
-
-            expected_key = node.resource_type.pluralize()
-            if expected_key != patch.yaml_key:
-                if patch.yaml_key == 'models':
-                    deprecations.warn(
-                        'models-key-mismatch',
-                        patch=patch, node=node, expected_key=expected_key
-                    )
-                else:
-                    raise_invalid_patch(
-                        node, patch.yaml_key, patch.original_file_path
-                    )
+        if patch.yaml_key in ['models', 'seeds', 'snapshots']:
+            unique_id = self.ref_lookup.get_unique_id(patch.name, None)
+        elif patch.yaml_key == 'analyses':
+            unique_id = self.analysis_lookup.get_unique_id(patch.name, None)
+        else:
+            raise dbt.exceptions.InternalException(
+                f'Unexpected yaml_key {patch.yaml_key} for patch in '
+                f'file {source_file.path.original_file_path}'
+            )
+        if unique_id is None:
+            # This will usually happen when a node is disabled
+            return
 
+        # patches can't be overwritten
+        node = self.nodes.get(unique_id)
+        if node:
+            if node.patch_path:
+                package_name, existing_file_path = node.patch_path.split('://')
+                raise_duplicate_patch_name(patch, existing_file_path)
+            source_file.append_patch(patch.yaml_key, unique_id)
             node.patch(patch)
 
-        # log debug-level warning about nodes we couldn't find
-        if patches:
-            for patch in patches.values():
-                # since patches aren't nodes, we can't use the existing
-                # target_not_found warning
-                logger.debug((
-                    'WARNING: Found documentation for resource "{}" which was '
-                    'not found or is disabled').format(patch.name)
-                )
+    def add_macro_patch(
+        self, source_file: SchemaSourceFile, patch: ParsedMacroPatch,
+    ) -> None:
+        # macros are fully namespaced
+        unique_id = f'macro.{patch.package_name}.{patch.name}'
+        macro = self.macros.get(unique_id)
+        if not macro:
+            warn_or_error(
+                f'WARNING: Found documentation for macro "{patch.name}" '
+                f'which was not found'
+            )
+            return
+        if macro.patch_path:
+            package_name, existing_file_path = macro.patch_path.split('://')
+            raise_duplicate_macro_patch_name(patch, existing_file_path)
+        source_file.macro_patches.append(unique_id)
+        macro.patch(patch)
+
+    def add_source_patch(
+        self, source_file: SchemaSourceFile, patch: SourcePatch,
+    ) -> None:
+        # source patches must be unique
+        key = (patch.overrides, patch.name)
+        if key in self.source_patches:
+            raise_duplicate_source_patch_name(patch, self.source_patches[key])
+        self.source_patches[key] = patch
+        source_file.source_patches.append(key)
 
     def get_used_schemas(self, resource_types=None):
         return frozenset({
@@ -719,6 +785,7 @@ class Manifest(MacroMethods):
             chain(self.nodes.values(), self.sources.values())
         )
 
+    # This is used in dbt.task.rpc.sql_commands 'add_new_refs'
     def deepcopy(self):
         return Manifest(
             nodes={k: _deepcopy(v) for k, v in self.nodes.items()},
@@ -726,20 +793,33 @@ class Manifest(MacroMethods):
             macros={k: _deepcopy(v) for k, v in self.macros.items()},
             docs={k: _deepcopy(v) for k, v in self.docs.items()},
             exposures={k: _deepcopy(v) for k, v in self.exposures.items()},
-            selectors=self.root_project.manifest_selectors,
+            selectors={k: _deepcopy(v) for k, v in self.selectors.items()},
             metadata=self.metadata,
             disabled=[_deepcopy(n) for n in self.disabled],
             files={k: _deepcopy(v) for k, v in self.files.items()},
+            state_check=_deepcopy(self.state_check),
         )
 
-    def writable_manifest(self):
+    def build_parent_and_child_maps(self):
         edge_members = list(chain(
             self.nodes.values(),
             self.sources.values(),
             self.exposures.values(),
         ))
-        forward_edges, backward_edges = build_edges(edge_members)
+        forward_edges, backward_edges = build_node_edges(edge_members)
+        self.child_map = forward_edges
+        self.parent_map = backward_edges
 
+    def build_macro_child_map(self):
+        edge_members = list(chain(
+            self.nodes.values(),
+            self.macros.values(),
+        ))
+        forward_edges = build_macro_edges(edge_members)
+        return forward_edges
+
+    def writable_manifest(self):
+        self.build_parent_and_child_maps()
         return WritableManifest(
             nodes=self.nodes,
             sources=self.sources,
@@ -749,18 +829,15 @@ class Manifest(MacroMethods):
             selectors=self.selectors,
             metadata=self.metadata,
             disabled=self.disabled,
-            child_map=forward_edges,
-            parent_map=backward_edges,
+            child_map=self.child_map,
+            parent_map=self.parent_map,
         )
 
-    # When 'to_dict' is called on the Manifest, it substitues a
-    # WritableManifest
-    def __pre_serialize__(self):
-        return self.writable_manifest()
-
     def write(self, path):
         self.writable_manifest().write(path)
 
+    # Called in dbt.compilation.Linker.write_graph and
+    # dbt.graph.queue.get and ._include_in_cost
     def expect(self, unique_id: str) -> GraphMemberNode:
         if unique_id in self.nodes:
             return self.nodes[unique_id]
@@ -775,29 +852,40 @@ class Manifest(MacroMethods):
             )
 
     @property
-    def docs_cache(self) -> DocCache:
-        if self._docs_cache is not None:
-            return self._docs_cache
-        cache = DocCache(self)
-        self._docs_cache = cache
-        return cache
+    def doc_lookup(self) -> DocLookup:
+        if self._doc_lookup is None:
+            self._doc_lookup = DocLookup(self)
+        return self._doc_lookup
+
+    def rebuild_doc_lookup(self):
+        self._doc_lookup = DocLookup(self)
 
     @property
-    def source_cache(self) -> SourceCache:
-        if self._sources_cache is not None:
-            return self._sources_cache
-        cache = SourceCache(self)
-        self._sources_cache = cache
-        return cache
+    def source_lookup(self) -> SourceLookup:
+        if self._source_lookup is None:
+            self._source_lookup = SourceLookup(self)
+        return self._source_lookup
+
+    def rebuild_source_lookup(self):
+        self._source_lookup = SourceLookup(self)
 
     @property
-    def refs_cache(self) -> RefableCache:
-        if self._refs_cache is not None:
-            return self._refs_cache
-        cache = RefableCache(self)
-        self._refs_cache = cache
-        return cache
+    def ref_lookup(self) -> RefableLookup:
+        if self._ref_lookup is None:
+            self._ref_lookup = RefableLookup(self)
+        return self._ref_lookup
 
+    def rebuild_ref_lookup(self):
+        self._ref_lookup = RefableLookup(self)
+
+    @property
+    def analysis_lookup(self) -> AnalysisLookup:
+        if self._analysis_lookup is None:
+            self._analysis_lookup = AnalysisLookup(self)
+        return self._analysis_lookup
+
+    # Called by dbt.parser.manifest._resolve_refs_for_exposure
+    # and dbt.parser.manifest._process_refs_for_node
     def resolve_ref(
         self,
         target_model_name: str,
@@ -813,7 +901,7 @@ class Manifest(MacroMethods):
             current_project, node_package, target_model_package
         )
         for pkg in candidates:
-            node = self.refs_cache.find_cached_value(target_model_name, pkg)
+            node = self.ref_lookup.find(target_model_name, pkg, self)
 
             if node is not None and node.config.enabled:
                 return node
@@ -828,6 +916,8 @@ class Manifest(MacroMethods):
             return Disabled(disabled)
         return None
 
+    # Called by dbt.parser.manifest._resolve_sources_for_exposure
+    # and dbt.parser.manifest._process_source_for_node
     def resolve_source(
         self,
         target_source_name: str,
@@ -842,7 +932,7 @@ class Manifest(MacroMethods):
         disabled: Optional[ParsedSourceDefinition] = None
 
         for pkg in candidates:
-            source = self.source_cache.find_cached_value(key, pkg)
+            source = self.source_lookup.find(key, pkg, self)
             if source is not None and source.config.enabled:
                 return source
 
@@ -855,6 +945,7 @@ class Manifest(MacroMethods):
             return Disabled(disabled)
         return None
 
+    # Called by DocsRuntimeContext.doc
     def resolve_doc(
         self,
         name: str,
@@ -871,11 +962,12 @@ class Manifest(MacroMethods):
         )
 
         for pkg in candidates:
-            result = self.docs_cache.find_cached_value(name, pkg)
+            result = self.doc_lookup.find(name, pkg, self)
             if result is not None:
                 return result
         return None
 
+    # Called by RunTask.defer_to_manifest
     def merge_from_artifact(
         self,
         adapter,
@@ -908,6 +1000,91 @@ class Manifest(MacroMethods):
             f'Merged {len(merged)} items from state (sample: {sample})'
         )
 
+    # Methods that were formerly in ParseResult
+
+    def add_macro(self, source_file: SourceFile, macro: ParsedMacro):
+        if macro.unique_id in self.macros:
+            # detect that the macro exists and emit an error
+            other_path = self.macros[macro.unique_id].original_file_path
+            # subtract 2 for the "Compilation Error" indent
+            # note that the line wrap eats newlines, so if you want newlines,
+            # this is the result :(
+            msg = line_wrap_message(
+                f'''\
+                dbt found two macros named "{macro.name}" in the project
+                "{macro.package_name}".
+
+
+                To fix this error, rename or remove one of the following
+                macros:
+
+                    - {macro.original_file_path}
+
+                    - {other_path}
+                ''',
+                subtract=2
+            )
+            raise_compiler_error(msg)
+
+        self.macros[macro.unique_id] = macro
+        source_file.macros.append(macro.unique_id)
+
+    def has_file(self, source_file: SourceFile) -> bool:
+        key = source_file.file_id
+        if key is None:
+            return False
+        if key not in self.files:
+            return False
+        my_checksum = self.files[key].checksum
+        return my_checksum == source_file.checksum
+
+    def add_source(
+        self, source_file: SchemaSourceFile, source: UnpatchedSourceDefinition
+    ):
+        # sources can't be overwritten!
+        _check_duplicates(source, self.sources)
+        self.sources[source.unique_id] = source  # type: ignore
+        source_file.sources.append(source.unique_id)
+
+    def add_node_nofile(self, node: ManifestNodes):
+        # nodes can't be overwritten!
+        _check_duplicates(node, self.nodes)
+        self.nodes[node.unique_id] = node
+
+    def add_node(self, source_file: AnySourceFile, node: ManifestNodes, test_from=None):
+        self.add_node_nofile(node)
+        if isinstance(source_file, SchemaSourceFile):
+            assert test_from
+            source_file.add_test(node.unique_id, test_from)
+        else:
+            source_file.nodes.append(node.unique_id)
+
+    def add_exposure(self, source_file: SchemaSourceFile, exposure: ParsedExposure):
+        _check_duplicates(exposure, self.exposures)
+        self.exposures[exposure.unique_id] = exposure
+        source_file.exposures.append(exposure.unique_id)
+
+    def add_disabled_nofile(self, node: CompileResultNode):
+        if node.unique_id in self._disabled:
+            self._disabled[node.unique_id].append(node)
+        else:
+            self._disabled[node.unique_id] = [node]
+
+    def add_disabled(self, source_file: AnySourceFile, node: CompileResultNode, test_from=None):
+        self.add_disabled_nofile(node)
+        if isinstance(source_file, SchemaSourceFile):
+            assert test_from
+            source_file.add_test(node.unique_id, test_from)
+        else:
+            source_file.nodes.append(node.unique_id)
+
+    def add_doc(self, source_file: SourceFile, doc: ParsedDocumentation):
+        _check_duplicates(doc, self.docs)
+        self.docs[doc.unique_id] = doc
+        source_file.docs.append(doc.unique_id)
+
+    # end of methods formerly in ParseResult
+
     # Provide support for copy.deepcopy() - we just need to avoid the lock!
     # pickle and deepcopy use this. It returns a callable object used to
     # create the initial version of the object and a tuple of arguments
@@ -927,17 +1104,19 @@ class Manifest(MacroMethods):
             self.files,
             self.metadata,
             self.flat_graph,
-            self._docs_cache,
-            self._sources_cache,
-            self._refs_cache,
+            self.state_check,
+            self.source_patches,
+            self._disabled,
+            self._doc_lookup,
+            self._source_lookup,
+            self._ref_lookup,
         )
         return self.__class__, args
 
 
 class MacroManifest(MacroMethods):
-    def __init__(self, macros, files):
+    def __init__(self, macros):
         self.macros = macros
-        self.files = files
         self.metadata = ManifestMetadata()
         # This is returned by the 'graph' context property
         # in the ProviderContext class.
@@ -948,7 +1127,7 @@ AnyManifest = Union[Manifest, MacroManifest]
 
 
 @dataclass
-@schema_version('manifest', 1)
+@schema_version('manifest', 2)
 class WritableManifest(ArtifactMixin):
     nodes: Mapping[UniqueID, ManifestNode] = field(
         metadata=dict(description=(
@@ -992,3 +1171,26 @@ class WritableManifest(ArtifactMixin):
     metadata: ManifestMetadata = field(metadata=dict(
         description='Metadata about the manifest',
     ))
+
+
+def _check_duplicates(
+    value: HasUniqueID, src: Mapping[str, HasUniqueID]
+):
+    if value.unique_id in src:
+        raise_duplicate_resource_name(value, src[value.unique_id])
+
+
+K_T = TypeVar('K_T')
+V_T = TypeVar('V_T')
+
+
+def _expect_value(
+    key: K_T, src: Mapping[K_T, V_T], old_file: SourceFile, name: str
+) -> V_T:
+    if key not in src:
+        raise CompilationException(
+            'Expected to find "{}" in cached "result.{}" based '
+            'on cached file information: {}!'
+            .format(key, name, old_file)
+        )
+    return src[key]

core/dbt/contracts/graph/model_config.py

@@ -2,14 +2,13 @@ from dataclasses import field, Field, dataclass
 from enum import Enum
 from itertools import chain
 from typing import (
-    Any, List, Optional, Dict, MutableMapping, Union, Type,
-    TypeVar, Callable,
+    Any, List, Optional, Dict, Union, Type, TypeVar
 )
 from dbt.dataclass_schema import (
     dbtClassMixin, ValidationError, register_pattern,
 )
 from dbt.contracts.graph.unparsed import AdditionalPropertiesAllowed
-from dbt.exceptions import CompilationException, InternalException
+from dbt.exceptions import InternalException
 from dbt.contracts.util import Replaceable, list_str
 from dbt import hooks
 from dbt.node_types import NodeType
@@ -182,53 +181,29 @@ T = TypeVar('T', bound='BaseConfig')
 
 @dataclass
 class BaseConfig(
-    AdditionalPropertiesAllowed, Replaceable, MutableMapping[str, Any]
+    AdditionalPropertiesAllowed, Replaceable
 ):
-    # Implement MutableMapping so this config will behave as some macros expect
-    # during parsing (notably, syntax like `{{ node.config['schema'] }}`)
+
+    # enable syntax like: config['key']
     def __getitem__(self, key):
-        """Handle parse-time use of `config` as a dictionary, making the extra
-        values available during parsing.
-        """
+        return self.get(key)
+
+    # like doing 'get' on a dictionary
+    def get(self, key, default=None):
         if hasattr(self, key):
             return getattr(self, key)
-        else:
+        elif key in self._extra:
             return self._extra[key]
+        else:
+            return default
 
+    # enable syntax like: config['key'] = value
     def __setitem__(self, key, value):
         if hasattr(self, key):
             setattr(self, key, value)
         else:
             self._extra[key] = value
 
-    def __delitem__(self, key):
-        if hasattr(self, key):
-            msg = (
-                'Error, tried to delete config key "{}": Cannot delete '
-                'built-in keys'
-            ).format(key)
-            raise CompilationException(msg)
-        else:
-            del self._extra[key]
-
-    def _content_iterator(self, include_condition: Callable[[Field], bool]):
-        seen = set()
-        for fld, _ in self._get_fields():
-            seen.add(fld.name)
-            if include_condition(fld):
-                yield fld.name
-
-        for key in self._extra:
-            if key not in seen:
-                seen.add(key)
-                yield key
-
-    def __iter__(self):
-        yield from self._content_iterator(include_condition=lambda f: True)
-
-    def __len__(self):
-        return len(self._get_fields()) + len(self._extra)
-
     @staticmethod
     def compare_key(
         unrendered: Dict[str, Any],
@@ -436,8 +411,42 @@ class SeedConfig(NodeConfig):
 
 @dataclass
 class TestConfig(NodeConfig):
+    schema: Optional[str] = field(
+        default='dbt_test__audit',
+        metadata=CompareBehavior.Exclude.meta(),
+    )
     materialized: str = 'test'
     severity: Severity = Severity('ERROR')
+    store_failures: Optional[bool] = None
+    where: Optional[str] = None
+    limit: Optional[int] = None
+    fail_calc: str = 'count(*)'
+    warn_if: str = '!= 0'
+    error_if: str = '!= 0'
+
+    @classmethod
+    def same_contents(
+        cls, unrendered: Dict[str, Any], other: Dict[str, Any]
+    ) -> bool:
+        """This is like __eq__, except it explicitly checks certain fields."""
+        modifiers = [
+            'severity',
+            'where',
+            'limit',
+            'fail_calc',
+            'warn_if',
+            'error_if',
+            'store_failures'
+        ]
+
+        seen = set()
+        for _, target_name in cls._get_fields():
+            key = target_name
+            seen.add(key)
+            if key in modifiers:
+                if not cls.compare_key(unrendered, other, key):
+                    return False
+        return True
 
 
 @dataclass
@@ -457,6 +466,11 @@ class SnapshotConfig(EmptySnapshotConfig):
     @classmethod
     def validate(cls, data):
         super().validate(data)
+        if not data.get('strategy') or not data.get('unique_key') or not \
+                data.get('target_schema'):
+            raise ValidationError(
+                "Snapshots must be configured with a 'strategy', 'unique_key', "
+                "and 'target_schema'.")
         if data.get('strategy') == 'check':
             if not data.get('check_cols'):
                 raise ValidationError(

core/dbt/contracts/graph/parsed.py

@@ -1,5 +1,7 @@
 import os
+import time
 from dataclasses import dataclass, field
+from mashumaro.types import SerializableType
 from pathlib import Path
 from typing import (
     Optional,
@@ -115,6 +117,21 @@ class ParsedNodeMixins(dbtClassMixin):
     def is_refable(self):
         return self.resource_type in NodeType.refable()
 
+    @property
+    def should_store_failures(self):
+        return self.resource_type == NodeType.Test and (
+            self.config.store_failures if self.config.store_failures is not None
+            else flags.STORE_FAILURES
+        )
+
+    # will this node map to an object in the database?
+    @property
+    def is_relational(self):
+        return (
+            self.resource_type in NodeType.refable() or
+            self.should_store_failures
+        )
+
     @property
     def is_ephemeral(self):
         return self.config.materialized == 'ephemeral'
@@ -131,7 +148,9 @@ class ParsedNodeMixins(dbtClassMixin):
         """Given a ParsedNodePatch, add the new information to the node."""
         # explicitly pick out the parts to update so we don't inadvertently
         # step on the model name or anything
-        self.patch_path: Optional[str] = patch.original_file_path
+        self.patch_path: Optional[str] = patch.file_id
+        # update created_at so process_docs will run in partial parsing
+        self.created_at = int(time.time())
         self.description = patch.description
         self.columns = patch.columns
         self.meta = patch.meta
@@ -179,9 +198,11 @@ class ParsedNodeDefaults(ParsedNodeMandatory):
     meta: Dict[str, Any] = field(default_factory=dict)
     docs: Docs = field(default_factory=Docs)
     patch_path: Optional[str] = None
+    compiled_path: Optional[str] = None
     build_path: Optional[str] = None
     deferred: bool = False
     unrendered_config: Dict[str, Any] = field(default_factory=dict)
+    created_at: int = field(default_factory=lambda: int(time.time()))
 
     def write_node(self, target_path: str, subdirectory: str, payload: str):
         if (os.path.basename(self.path) ==
@@ -203,7 +224,39 @@ T = TypeVar('T', bound='ParsedNode')
 
 
 @dataclass
-class ParsedNode(ParsedNodeDefaults, ParsedNodeMixins):
+class ParsedNode(ParsedNodeDefaults, ParsedNodeMixins, SerializableType):
+
+    def _serialize(self):
+        return self.to_dict()
+
+    @classmethod
+    def _deserialize(cls, dct: Dict[str, int]):
+        # The serialized ParsedNodes do not differ from each other
+        # in fields that would allow 'from_dict' to distinguis
+        # between them.
+        resource_type = dct['resource_type']
+        if resource_type == 'model':
+            return ParsedModelNode.from_dict(dct)
+        elif resource_type == 'analysis':
+            return ParsedAnalysisNode.from_dict(dct)
+        elif resource_type == 'seed':
+            return ParsedSeedNode.from_dict(dct)
+        elif resource_type == 'rpc':
+            return ParsedRPCNode.from_dict(dct)
+        elif resource_type == 'test':
+            if 'test_metadata' in dct:
+                return ParsedSchemaTestNode.from_dict(dct)
+            else:
+                return ParsedDataTestNode.from_dict(dct)
+        elif resource_type == 'operation':
+            return ParsedHookNode.from_dict(dct)
+        elif resource_type == 'seed':
+            return ParsedSeedNode.from_dict(dct)
+        elif resource_type == 'snapshot':
+            return ParsedSnapshotNode.from_dict(dct)
+        else:
+            return cls.from_dict(dct)
+
     def _persist_column_docs(self) -> bool:
         return bool(self.config.persist_docs.get('columns'))
 
@@ -368,15 +421,6 @@ class ParsedSchemaTestNode(ParsedNode, HasTestMetadata):
     column_name: Optional[str] = None
     config: TestConfig = field(default_factory=TestConfig)
 
-    def same_config(self, other) -> bool:
-        return (
-            self.unrendered_config.get('severity') ==
-            other.unrendered_config.get('severity')
-        )
-
-    def same_column_name(self, other) -> bool:
-        return self.column_name == other.column_name
-
     def same_contents(self, other) -> bool:
         if other is None:
             return False
@@ -441,13 +485,15 @@ class ParsedMacro(UnparsedBaseNode, HasUniqueID):
     docs: Docs = field(default_factory=Docs)
     patch_path: Optional[str] = None
     arguments: List[MacroArgument] = field(default_factory=list)
+    created_at: int = field(default_factory=lambda: int(time.time()))
 
     def local_vars(self):
         return {}
 
     def patch(self, patch: ParsedMacroPatch):
-        self.patch_path: Optional[str] = patch.original_file_path
+        self.patch_path: Optional[str] = patch.file_id
         self.description = patch.description
+        self.created_at = int(time.time())
         self.meta = patch.meta
         self.docs = patch.docs
         self.arguments = patch.arguments
@@ -546,7 +592,8 @@ class ParsedSourceDefinition(
     UnparsedBaseNode,
     HasUniqueID,
     HasRelationMetadata,
-    HasFqn
+    HasFqn,
+
 ):
     name: str
     source_name: str
@@ -567,6 +614,7 @@ class ParsedSourceDefinition(
     patch_path: Optional[Path] = None
     unrendered_config: Dict[str, Any] = field(default_factory=dict)
     relation_name: Optional[str] = None
+    created_at: int = field(default_factory=lambda: int(time.time()))
 
     def same_database_representation(
         self, other: 'ParsedSourceDefinition'
@@ -642,6 +690,10 @@ class ParsedSourceDefinition(
     def depends_on_nodes(self):
         return []
 
+    @property
+    def depends_on(self):
+        return {'nodes': []}
+
     @property
     def refs(self):
         return []
@@ -667,10 +719,13 @@ class ParsedExposure(UnparsedBaseNode, HasUniqueID, HasFqn):
     resource_type: NodeType = NodeType.Exposure
     description: str = ''
     maturity: Optional[MaturityType] = None
+    meta: Dict[str, Any] = field(default_factory=dict)
+    tags: List[str] = field(default_factory=list)
     url: Optional[str] = None
     depends_on: DependsOn = field(default_factory=DependsOn)
     refs: List[List[str]] = field(default_factory=list)
     sources: List[List[str]] = field(default_factory=list)
+    created_at: int = field(default_factory=lambda: int(time.time()))
 
     @property
     def depends_on_nodes(self):
@@ -680,11 +735,6 @@ class ParsedExposure(UnparsedBaseNode, HasUniqueID, HasFqn):
     def search_name(self):
         return self.name
 
-    # no tags for now, but we could definitely add them
-    @property
-    def tags(self):
-        return []
-
     def same_depends_on(self, old: 'ParsedExposure') -> bool:
         return set(self.depends_on.nodes) == set(old.depends_on.nodes)
 
@@ -705,6 +755,7 @@ class ParsedExposure(UnparsedBaseNode, HasUniqueID, HasFqn):
 
     def same_contents(self, old: Optional['ParsedExposure']) -> bool:
         # existing when it didn't before is a change!
+        # metadata/tags changes are not "changes"
         if old is None:
             return True
 
@@ -720,6 +771,18 @@ class ParsedExposure(UnparsedBaseNode, HasUniqueID, HasFqn):
         )
 
 
+ManifestNodes = Union[
+    ParsedAnalysisNode,
+    ParsedDataTestNode,
+    ParsedHookNode,
+    ParsedModelNode,
+    ParsedRPCNode,
+    ParsedSchemaTestNode,
+    ParsedSeedNode,
+    ParsedSnapshotNode,
+]
+
+
 ParsedResource = Union[
     ParsedDocumentation,
     ParsedMacro,

core/dbt/contracts/graph/unparsed.py

@@ -25,6 +25,10 @@ class UnparsedBaseNode(dbtClassMixin, Replaceable):
     path: str
     original_file_path: str
 
+    @property
+    def file_id(self):
+        return f'{self.package_name}://{self.original_file_path}'
+
 
 @dataclass
 class HasSQL:
@@ -116,6 +120,10 @@ class HasYamlMetadata(dbtClassMixin):
     yaml_key: str
     package_name: str
 
+    @property
+    def file_id(self):
+        return f'{self.package_name}://{self.original_file_path}'
+
 
 @dataclass
 class UnparsedAnalysisUpdate(HasColumnDocs, HasDocs, HasYamlMetadata):
@@ -347,6 +355,10 @@ class UnparsedDocumentation(dbtClassMixin, Replaceable):
     path: str
     original_file_path: str
 
+    @property
+    def file_id(self):
+        return f'{self.package_name}://{self.original_file_path}'
+
     @property
     def resource_type(self):
         return NodeType.Documentation
@@ -413,5 +425,7 @@ class UnparsedExposure(dbtClassMixin, Replaceable):
     owner: ExposureOwner
     description: str = ''
     maturity: Optional[MaturityType] = None
+    meta: Dict[str, Any] = field(default_factory=dict)
+    tags: List[str] = field(default_factory=list)
     url: Optional[str] = None
     depends_on: List[str] = field(default_factory=list)

core/dbt/contracts/project.py

@@ -70,6 +70,7 @@ class GitPackage(Package):
     git: str
     revision: Optional[RawVersion] = None
     warn_unpinned: Optional[bool] = None
+    subdirectory: Optional[str] = None
 
     def get_revisions(self) -> List[str]:
         if self.revision is None:
@@ -190,11 +191,13 @@ class Project(HyphenatedDbtClassMixin, Replaceable):
     on_run_start: Optional[List[str]] = field(default_factory=list_str)
     on_run_end: Optional[List[str]] = field(default_factory=list_str)
     require_dbt_version: Optional[Union[List[str], str]] = None
+    dispatch: List[Dict[str, Any]] = field(default_factory=list)
     models: Dict[str, Any] = field(default_factory=dict)
     seeds: Dict[str, Any] = field(default_factory=dict)
     snapshots: Dict[str, Any] = field(default_factory=dict)
     analyses: Dict[str, Any] = field(default_factory=dict)
     sources: Dict[str, Any] = field(default_factory=dict)
+    tests: Dict[str, Any] = field(default_factory=dict)
     vars: Optional[Dict[str, Any]] = field(
         default=None,
         metadata=dict(
@@ -211,6 +214,13 @@ class Project(HyphenatedDbtClassMixin, Replaceable):
             raise ValidationError(
                 f"Invalid project name: {data['name']} is a reserved word"
             )
+        # validate dispatch config
+        if 'dispatch' in data and data['dispatch']:
+            entries = data['dispatch']
+            for entry in entries:
+                if ('macro_namespace' not in entry or 'search_order' not in entry or
+                        not isinstance(entry['search_order'], list)):
+                    raise ValidationError(f"Invalid project dispatch config: {entry}")
 
 
 @dataclass

core/dbt/contracts/results.py

@@ -78,6 +78,7 @@ class TestStatus(StrEnum):
     Error = NodeStatus.Error
     Fail = NodeStatus.Fail
     Warn = NodeStatus.Warn
+    Skipped = NodeStatus.Skipped
 
 
 class FreshnessStatus(StrEnum):
@@ -94,13 +95,16 @@ class BaseResult(dbtClassMixin):
     thread_id: str
     execution_time: float
     adapter_response: Dict[str, Any]
-    message: Optional[Union[str, int]]
+    message: Optional[str]
+    failures: Optional[int]
 
     @classmethod
     def __pre_deserialize__(cls, data):
         data = super().__pre_deserialize__(data)
         if 'message' not in data:
             data['message'] = None
+        if 'failures' not in data:
+            data['failures'] = None
         return data
 
 
@@ -157,7 +161,8 @@ def process_run_result(result: RunResult) -> RunResultOutput:
         thread_id=result.thread_id,
         execution_time=result.execution_time,
         message=result.message,
-        adapter_response=result.adapter_response
+        adapter_response=result.adapter_response,
+        failures=result.failures
     )
 
 
@@ -180,7 +185,7 @@ class RunExecutionResult(
 
 
 @dataclass
-@schema_version('run-results', 1)
+@schema_version('run-results', 2)
 class RunResultsArtifact(ExecutionResult, ArtifactMixin):
     results: Sequence[RunResultOutput]
     args: Dict[str, Any] = field(default_factory=dict)
@@ -378,6 +383,7 @@ class FreshnessExecutionResultArtifact(
 
 
 Primitive = Union[bool, str, float, None]
+PrimitiveDict = Dict[str, Primitive]
 
 CatalogKey = NamedTuple(
     'CatalogKey',

core/dbt/contracts/rpc.py

@@ -63,6 +63,16 @@ class RPCCompileParameters(RPCParameters):
     state: Optional[str] = None
 
 
+@dataclass
+class RPCListParameters(RPCParameters):
+    resource_types: Optional[List[str]] = None
+    models: Union[None, str, List[str]] = None
+    exclude: Union[None, str, List[str]] = None
+    select: Union[None, str, List[str]] = None
+    selector: Optional[str] = None
+    output: Optional[str] = 'json'
+
+
 @dataclass
 class RPCRunParameters(RPCParameters):
     threads: Optional[int] = None
@@ -190,6 +200,13 @@ class RemoteResult(VersionedSchema):
     logs: List[LogMessage]
 
 
+@dataclass
+@schema_version('remote-list-results', 1)
+class RemoteListResults(RemoteResult):
+    output: List[Any]
+    generated_at: datetime = field(default_factory=datetime.utcnow)
+
+
 @dataclass
 @schema_version('remote-deps-result', 1)
 class RemoteDepsResult(RemoteResult):

core/dbt/contracts/util.py

@@ -14,7 +14,6 @@ from dbt.version import __version__
 from dbt.tracking import get_invocation_id
 from dbt.dataclass_schema import dbtClassMixin
 
-MacroKey = Tuple[str, str]
 SourceKey = Tuple[str, str]
 
 

core/dbt/dataclass_schema.py

@@ -73,7 +73,9 @@ class dbtClassMixin(DataClassDictMixin, JsonSchemaMixin):
     # performing the conversion to a dict
     @classmethod
     def __pre_deserialize__(cls, data):
-        if cls._hyphenated:
+        # `data` might not be a dict, e.g. for `query_comment`, which accepts
+        # a dict or a string; only snake-case for dict values.
+        if cls._hyphenated and isinstance(data, dict):
             new_dict = {}
             for key in data:
                 if '-' in key:

core/dbt/deprecations.py

@@ -43,6 +43,20 @@ class DBTDeprecation:
             active_deprecations.add(self.name)
 
 
+class DispatchPackagesDeprecation(DBTDeprecation):
+    _name = 'dispatch-packages'
+    _description = '''\
+    The "packages" argument of adapter.dispatch() has been deprecated.
+    Use the "macro_namespace" argument instead.
+
+    Raised during dispatch for: {macro_name}
+
+    For more information, see:
+
+    https://docs.getdbt.com/reference/dbt-jinja-functions/dispatch
+    '''
+
+
 class MaterializationReturnDeprecation(DBTDeprecation):
     _name = 'materialization-return'
 
@@ -155,6 +169,7 @@ def warn(name, *args, **kwargs):
 active_deprecations: Set[str] = set()
 
 deprecations_list: List[DBTDeprecation] = [
+    DispatchPackagesDeprecation(),
     MaterializationReturnDeprecation(),
     NotADictionaryDeprecation(),
     ColumnQuotingDeprecation(),

core/dbt/deps/base.py

@@ -93,6 +93,9 @@ class PinnedPackage(BasePackage):
         dest_dirname = self.get_project_name(project, renderer)
         return os.path.join(project.modules_path, dest_dirname)
 
+    def get_subdirectory(self):
+        return None
+
 
 SomePinned = TypeVar('SomePinned', bound=PinnedPackage)
 SomeUnpinned = TypeVar('SomeUnpinned', bound='UnpinnedPackage')

core/dbt/deps/git.py

@@ -1,6 +1,6 @@
 import os
 import hashlib
-from typing import List
+from typing import List, Optional
 
 from dbt.clients import git, system
 from dbt.config import Project
@@ -37,19 +37,27 @@ class GitPackageMixin:
 
 class GitPinnedPackage(GitPackageMixin, PinnedPackage):
     def __init__(
-        self, git: str, revision: str, warn_unpinned: bool = True
+        self,
+        git: str,
+        revision: str,
+        warn_unpinned: bool = True,
+        subdirectory: Optional[str] = None,
     ) -> None:
         super().__init__(git)
         self.revision = revision
         self.warn_unpinned = warn_unpinned
+        self.subdirectory = subdirectory
         self._checkout_name = md5sum(self.git)
 
     def get_version(self):
         return self.revision
 
+    def get_subdirectory(self):
+        return self.subdirectory
+
     def nice_version_name(self):
         if self.revision == 'HEAD':
-            return 'HEAD (default branch)'
+            return 'HEAD (default revision)'
         else:
             return 'revision {}'.format(self.revision)
 
@@ -68,8 +76,8 @@ class GitPinnedPackage(GitPackageMixin, PinnedPackage):
         the path to the checked out directory."""
         try:
             dir_ = git.clone_and_checkout(
-                self.git, get_downloads_path(), branch=self.revision,
-                dirname=self._checkout_name
+                self.git, get_downloads_path(), revision=self.revision,
+                dirname=self._checkout_name, subdirectory=self.subdirectory
             )
         except ExecutableError as exc:
             if exc.cmd and exc.cmd[0] == 'git':
@@ -107,11 +115,16 @@ class GitPinnedPackage(GitPackageMixin, PinnedPackage):
 
 class GitUnpinnedPackage(GitPackageMixin, UnpinnedPackage[GitPinnedPackage]):
     def __init__(
-        self, git: str, revisions: List[str], warn_unpinned: bool = True
+        self,
+        git: str,
+        revisions: List[str],
+        warn_unpinned: bool = True,
+        subdirectory: Optional[str] = None,
     ) -> None:
         super().__init__(git)
         self.revisions = revisions
         self.warn_unpinned = warn_unpinned
+        self.subdirectory = subdirectory
 
     @classmethod
     def from_contract(
@@ -122,7 +135,7 @@ class GitUnpinnedPackage(GitPackageMixin, UnpinnedPackage[GitPinnedPackage]):
         # we want to map None -> True
         warn_unpinned = contract.warn_unpinned is not False
         return cls(git=contract.git, revisions=revisions,
-                   warn_unpinned=warn_unpinned)
+                   warn_unpinned=warn_unpinned, subdirectory=contract.subdirectory)
 
     def all_names(self) -> List[str]:
         if self.git.endswith('.git'):
@@ -140,6 +153,7 @@ class GitUnpinnedPackage(GitPackageMixin, UnpinnedPackage[GitPinnedPackage]):
             git=self.git,
             revisions=self.revisions + other.revisions,
             warn_unpinned=warn_unpinned,
+            subdirectory=self.subdirectory,
         )
 
     def resolved(self) -> GitPinnedPackage:
@@ -153,5 +167,5 @@ class GitUnpinnedPackage(GitPackageMixin, UnpinnedPackage[GitPinnedPackage]):
 
         return GitPinnedPackage(
             git=self.git, revision=requested.pop(),
-            warn_unpinned=self.warn_unpinned
+            warn_unpinned=self.warn_unpinned, subdirectory=self.subdirectory
         )

core/dbt/exceptions.py

@@ -289,6 +289,15 @@ class JinjaRenderingException(CompilationException):
     pass
 
 
+class UndefinedMacroException(CompilationException):
+
+    def __str__(self, prefix='! ') -> str:
+        msg = super().__str__(prefix)
+        return f'{msg}. This can happen when calling a macro that does ' \
+            'not exist. Check for typos and/or install package dependencies ' \
+            'with "dbt deps".'
+
+
 class UnknownAsyncIDException(Exception):
     CODE = 10012
     MESSAGE = 'RPC server got an unknown async ID'
@@ -845,11 +854,11 @@ def _fix_dupe_msg(path_1: str, path_2: str, name: str, type_name: str) -> str:
         )
 
 
-def raise_duplicate_patch_name(patch_1, patch_2):
+def raise_duplicate_patch_name(patch_1, existing_patch_path):
     name = patch_1.name
     fix = _fix_dupe_msg(
         patch_1.original_file_path,
-        patch_2.original_file_path,
+        existing_patch_path,
         name,
         'resource',
     )
@@ -860,12 +869,12 @@ def raise_duplicate_patch_name(patch_1, patch_2):
     )
 
 
-def raise_duplicate_macro_patch_name(patch_1, patch_2):
+def raise_duplicate_macro_patch_name(patch_1, existing_patch_path):
     package_name = patch_1.package_name
     name = patch_1.name
     fix = _fix_dupe_msg(
         patch_1.original_file_path,
-        patch_2.original_file_path,
+        existing_patch_path,
         name,
         'macros'
     )

core/dbt/flags.py

@@ -13,9 +13,11 @@ FULL_REFRESH = None
 USE_CACHE = None
 WARN_ERROR = None
 TEST_NEW_PARSER = None
+USE_EXPERIMENTAL_PARSER = None
 WRITE_JSON = None
 PARTIAL_PARSE = None
 USE_COLORS = None
+STORE_FAILURES = None
 
 
 def env_set_truthy(key: str) -> Optional[str]:
@@ -53,22 +55,26 @@ MP_CONTEXT = _get_context()
 
 def reset():
     global STRICT_MODE, FULL_REFRESH, USE_CACHE, WARN_ERROR, TEST_NEW_PARSER, \
-        WRITE_JSON, PARTIAL_PARSE, MP_CONTEXT, USE_COLORS
+        USE_EXPERIMENTAL_PARSER, WRITE_JSON, PARTIAL_PARSE, MP_CONTEXT, USE_COLORS, \
+        STORE_FAILURES
 
     STRICT_MODE = False
     FULL_REFRESH = False
     USE_CACHE = True
     WARN_ERROR = False
     TEST_NEW_PARSER = False
+    USE_EXPERIMENTAL_PARSER = False
     WRITE_JSON = True
     PARTIAL_PARSE = False
     MP_CONTEXT = _get_context()
     USE_COLORS = True
+    STORE_FAILURES = False
 
 
 def set_from_args(args):
     global STRICT_MODE, FULL_REFRESH, USE_CACHE, WARN_ERROR, TEST_NEW_PARSER, \
-        WRITE_JSON, PARTIAL_PARSE, MP_CONTEXT, USE_COLORS
+        USE_EXPERIMENTAL_PARSER, WRITE_JSON, PARTIAL_PARSE, MP_CONTEXT, USE_COLORS, \
+        STORE_FAILURES
 
     USE_CACHE = getattr(args, 'use_cache', USE_CACHE)
 
@@ -80,6 +86,7 @@ def set_from_args(args):
     )
 
     TEST_NEW_PARSER = getattr(args, 'test_new_parser', TEST_NEW_PARSER)
+    USE_EXPERIMENTAL_PARSER = getattr(args, 'use_experimental_parser', USE_EXPERIMENTAL_PARSER)
     WRITE_JSON = getattr(args, 'write_json', WRITE_JSON)
     PARTIAL_PARSE = getattr(args, 'partial_parse', None)
     MP_CONTEXT = _get_context()
@@ -91,6 +98,8 @@ def set_from_args(args):
     if use_colors_override is not None:
         USE_COLORS = use_colors_override
 
+    STORE_FAILURES = getattr(args, 'store_failures', STORE_FAILURES)
+
 
 # initialize everything to the defaults on module load
 reset()

core/dbt/graph/cli.py

@@ -26,7 +26,7 @@ SCHEMA_TEST_SELECTOR: str = 'test_type:schema'
 
 
 def parse_union(
-    components: List[str], expect_exists: bool
+    components: List[str], expect_exists: bool, greedy: bool = False
 ) -> SelectionUnion:
     # turn ['a b', 'c'] -> ['a', 'b', 'c']
     raw_specs = itertools.chain.from_iterable(
@@ -37,7 +37,7 @@ def parse_union(
     # ['a', 'b', 'c,d'] -> union('a', 'b', intersection('c', 'd'))
     for raw_spec in raw_specs:
         intersection_components: List[SelectionSpec] = [
-            SelectionCriteria.from_single_spec(part)
+            SelectionCriteria.from_single_spec(part, greedy=greedy)
             for part in raw_spec.split(INTERSECTION_DELIMITER)
         ]
         union_components.append(SelectionIntersection(
@@ -45,7 +45,6 @@ def parse_union(
             expect_exists=expect_exists,
             raw=raw_spec,
         ))
-
     return SelectionUnion(
         components=union_components,
         expect_exists=False,
@@ -54,21 +53,21 @@ def parse_union(
 
 
 def parse_union_from_default(
-    raw: Optional[List[str]], default: List[str]
+    raw: Optional[List[str]], default: List[str], greedy: bool = False
 ) -> SelectionUnion:
     components: List[str]
     expect_exists: bool
     if raw is None:
-        return parse_union(components=default, expect_exists=False)
+        return parse_union(components=default, expect_exists=False, greedy=greedy)
     else:
-        return parse_union(components=raw, expect_exists=True)
+        return parse_union(components=raw, expect_exists=True, greedy=greedy)
 
 
 def parse_difference(
     include: Optional[List[str]], exclude: Optional[List[str]]
 ) -> SelectionDifference:
     included = parse_union_from_default(include, DEFAULT_INCLUDES)
-    excluded = parse_union_from_default(exclude, DEFAULT_EXCLUDES)
+    excluded = parse_union_from_default(exclude, DEFAULT_EXCLUDES, greedy=True)
     return SelectionDifference(components=[included, excluded])
 
 

core/dbt/graph/queue.py

@@ -1,10 +1,8 @@
-import threading
-from queue import PriorityQueue
-from typing import (
-    Dict, Set, Optional
-)
-
 import networkx as nx  # type: ignore
+import threading
+
+from queue import PriorityQueue
+from typing import Dict, Set, List, Generator, Optional
 
 from .graph import UniqueId
 from dbt.contracts.graph.parsed import ParsedSourceDefinition, ParsedExposure
@@ -21,9 +19,8 @@ class GraphQueue:
     that separate threads do not call `.empty()` or `__len__()` and `.get()` at
     the same time, as there is an unlocked race!
     """
-    def __init__(
-        self, graph: nx.DiGraph, manifest: Manifest, selected: Set[UniqueId]
-    ):
+
+    def __init__(self, graph: nx.DiGraph, manifest: Manifest, selected: Set[UniqueId]):
         self.graph = graph
         self.manifest = manifest
         self._selected = selected
@@ -37,7 +34,7 @@ class GraphQueue:
         # this lock controls most things
         self.lock = threading.Lock()
         # store the 'score' of each node as a number. Lower is higher priority.
-        self._scores = self._calculate_scores()
+        self._scores = self._get_scores(self.graph)
         # populate the initial queue
         self._find_new_additions()
         # awaits after task end
@@ -56,30 +53,59 @@ class GraphQueue:
             return False
         return True
 
-    def _calculate_scores(self) -> Dict[UniqueId, int]:
-        """Calculate the 'value' of each node in the graph based on how many
-        blocking descendants it has. We use this score for the internal
-        priority queue's ordering, so the quality of this metric is important.
+    @staticmethod
+    def _grouped_topological_sort(
+        graph: nx.DiGraph,
+    ) -> Generator[List[str], None, None]:
+        """Topological sort of given graph that groups ties.
 
-        The score is stored as a negative number because the internal
-        PriorityQueue picks lowest values first.
+        Adapted from `nx.topological_sort`, this function returns a topo sort of a graph however
+        instead of arbitrarily ordering ties in the sort order, ties are grouped together in
+        lists.
 
-        We could do this in one pass over the graph instead of len(self.graph)
-        passes but this is easy. For large graphs this may hurt performance.
+        Args:
+            graph: The graph to be sorted.
 
-        This operates on the graph, so it would require a lock if called from
-        outside __init__.
-
-        :return Dict[str, int]: The score dict, mapping unique IDs to integer
-            scores. Lower scores are higher priority.
+        Returns:
+            A generator that yields lists of nodes, one list per graph depth level.
         """
+        indegree_map = {v: d for v, d in graph.in_degree() if d > 0}
+        zero_indegree = [v for v, d in graph.in_degree() if d == 0]
+
+        while zero_indegree:
+            yield zero_indegree
+            new_zero_indegree = []
+            for v in zero_indegree:
+                for _, child in graph.edges(v):
+                    indegree_map[child] -= 1
+                    if not indegree_map[child]:
+                        new_zero_indegree.append(child)
+            zero_indegree = new_zero_indegree
+
+    def _get_scores(self, graph: nx.DiGraph) -> Dict[str, int]:
+        """Scoring nodes for processing order.
+
+        Scores are calculated by the graph depth level. Lowest score (0) should be processed first.
+
+        Args:
+            graph: The graph to be scored.
+
+        Returns:
+            A dictionary consisting of `node name`:`score` pairs.
+        """
+        # split graph by connected subgraphs
+        subgraphs = (
+            graph.subgraph(x) for x in nx.connected_components(nx.Graph(graph))
+        )
+
+        # score all nodes in all subgraphs
         scores = {}
-        for node in self.graph.nodes():
-            score = -1 * len([
-                d for d in nx.descendants(self.graph, node)
-                if self._include_in_cost(d)
-            ])
-            scores[node] = score
+        for subgraph in subgraphs:
+            grouped_nodes = self._grouped_topological_sort(subgraph)
+            for level, group in enumerate(grouped_nodes):
+                for node in group:
+                    scores[node] = level
+
         return scores
 
     def get(
@@ -133,8 +159,6 @@ class GraphQueue:
     def _find_new_additions(self) -> None:
         """Find any nodes in the graph that need to be added to the internal
         queue and add them.
-
-        Callers must hold the lock.
         """
         for node, in_degree in self.graph.in_degree():
             if not self._already_known(node) and in_degree == 0:

core/dbt/graph/selector.py

@@ -1,5 +1,5 @@
 
-from typing import Set, List, Optional
+from typing import Set, List, Optional, Tuple
 
 from .graph import Graph, UniqueId
 from .queue import GraphQueue
@@ -30,6 +30,18 @@ def alert_non_existence(raw_spec, nodes):
         )
 
 
+def can_select_indirectly(node):
+    """If a node is not selected itself, but its parent(s) are, it may qualify
+    for indirect selection.
+    Today, only Test nodes can be indirectly selected. In the future,
+    other node types or invocation flags might qualify.
+    """
+    if node.resource_type == NodeType.Test:
+        return True
+    else:
+        return False
+
+
 class NodeSelector(MethodManager):
     """The node selector is aware of the graph and manifest,
     """
@@ -61,8 +73,8 @@ class NodeSelector(MethodManager):
 
     def get_nodes_from_criteria(
         self,
-        spec: SelectionCriteria,
-    ) -> Set[UniqueId]:
+        spec: SelectionCriteria
+    ) -> Tuple[Set[UniqueId], Set[UniqueId]]:
         """Get all nodes specified by the single selection criteria.
 
         - collect the directly included nodes
@@ -79,11 +91,14 @@ class NodeSelector(MethodManager):
                 f"The '{spec.method}' selector specified in {spec.raw} is "
                 f"invalid. Must be one of [{valid_selectors}]"
             )
-            return set()
+            return set(), set()
 
-        extras = self.collect_specified_neighbors(spec, collected)
-        result = self.expand_selection(collected | extras)
-        return result
+        neighbors = self.collect_specified_neighbors(spec, collected)
+        direct_nodes, indirect_nodes = self.expand_selection(
+            selected=(collected | neighbors),
+            greedy=spec.greedy
+        )
+        return direct_nodes, indirect_nodes
 
     def collect_specified_neighbors(
         self, spec: SelectionCriteria, selected: Set[UniqueId]
@@ -106,24 +121,46 @@ class NodeSelector(MethodManager):
             additional.update(self.graph.select_children(selected, depth))
         return additional
 
-    def select_nodes(self, spec: SelectionSpec) -> Set[UniqueId]:
-        """Select the nodes in the graph according to the spec.
-
-        If the spec is a composite spec (a union, difference, or intersection),
+    def select_nodes_recursively(self, spec: SelectionSpec) -> Tuple[Set[UniqueId], Set[UniqueId]]:
+        """If the spec is a composite spec (a union, difference, or intersection),
         recurse into its selections and combine them. If the spec is a concrete
         selection criteria, resolve that using the given graph.
         """
         if isinstance(spec, SelectionCriteria):
-            result = self.get_nodes_from_criteria(spec)
+            direct_nodes, indirect_nodes = self.get_nodes_from_criteria(spec)
         else:
-            node_selections = [
-                self.select_nodes(component)
+            bundles = [
+                self.select_nodes_recursively(component)
                 for component in spec
             ]
-            result = spec.combined(node_selections)
+
+            direct_sets = []
+            indirect_sets = []
+
+            for direct, indirect in bundles:
+                direct_sets.append(direct)
+                indirect_sets.append(direct | indirect)
+
+            initial_direct = spec.combined(direct_sets)
+            indirect_nodes = spec.combined(indirect_sets)
+
+            direct_nodes = self.incorporate_indirect_nodes(initial_direct, indirect_nodes)
+
             if spec.expect_exists:
-                alert_non_existence(spec.raw, result)
-        return result
+                alert_non_existence(spec.raw, direct_nodes)
+
+        return direct_nodes, indirect_nodes
+
+    def select_nodes(self, spec: SelectionSpec) -> Set[UniqueId]:
+        """Select the nodes in the graph according to the spec.
+
+        This is the main point of entry for turning a spec into a set of nodes:
+        - Recurse through spec, select by criteria, combine by set operation
+        - Return final (unfiltered) selection set
+        """
+
+        direct_nodes, indirect_nodes = self.select_nodes_recursively(spec)
+        return direct_nodes
 
     def _is_graph_member(self, unique_id: UniqueId) -> bool:
         if unique_id in self.manifest.sources:
@@ -162,12 +199,55 @@ class NodeSelector(MethodManager):
             unique_id for unique_id in selected if self._is_match(unique_id)
         }
 
-    def expand_selection(self, selected: Set[UniqueId]) -> Set[UniqueId]:
-        """Perform selector-specific expansion."""
+    def expand_selection(
+        self, selected: Set[UniqueId], greedy: bool = False
+    ) -> Tuple[Set[UniqueId], Set[UniqueId]]:
+        # Test selection can expand to include an implicitly/indirectly selected test.
+        # In this way, `dbt test -m model_a` also includes tests that directly depend on `model_a`.
+        # Expansion has two modes, GREEDY and NOT GREEDY.
+        #
+        # GREEDY mode: If ANY parent is selected, select the test. We use this for EXCLUSION.
+        #
+        # NOT GREEDY mode:
+        #  - If ALL parents are selected, select the test.
+        #  - If ANY parent is missing, return it separately. We'll keep it around
+        #    for later and see if its other parents show up.
+        # We use this for INCLUSION.
+
+        direct_nodes = set(selected)
+        indirect_nodes = set()
+
+        for unique_id in self.graph.select_successors(selected):
+            if unique_id in self.manifest.nodes:
+                node = self.manifest.nodes[unique_id]
+                if can_select_indirectly(node):
+                    # should we add it in directly?
+                    if greedy or set(node.depends_on.nodes) <= set(selected):
+                        direct_nodes.add(unique_id)
+                    # if not:
+                    else:
+                        indirect_nodes.add(unique_id)
+
+        return direct_nodes, indirect_nodes
+
+    def incorporate_indirect_nodes(
+        self, direct_nodes: Set[UniqueId], indirect_nodes: Set[UniqueId] = set()
+    ) -> Set[UniqueId]:
+        # Check tests previously selected indirectly to see if ALL their
+        # parents are now present.
+
+        selected = set(direct_nodes)
+
+        for unique_id in indirect_nodes:
+            if unique_id in self.manifest.nodes:
+                node = self.manifest.nodes[unique_id]
+                if set(node.depends_on.nodes) <= set(selected):
+                    selected.add(unique_id)
+
         return selected
 
     def get_selected(self, spec: SelectionSpec) -> Set[UniqueId]:
-        """get_selected runs trhough the node selection process:
+        """get_selected runs through the node selection process:
 
             - node selection. Based on the include/exclude sets, the set
                 of matched unique IDs is returned

core/dbt/graph/selector_methods.py

@@ -49,25 +49,23 @@ class MethodName(StrEnum):
     Exposure = 'exposure'
 
 
-def is_selected_node(real_node, node_selector):
-    for i, selector_part in enumerate(node_selector):
+def is_selected_node(fqn: List[str], node_selector: str):
 
-        is_last = (i == len(node_selector) - 1)
+    # If qualified_name exactly matches model name (fqn's leaf), return True
+    if fqn[-1] == node_selector:
+        return True
+    # Flatten node parts. Dots in model names act as namespace separators
+    flat_fqn = [item for segment in fqn for item in segment.split('.')]
+    # Selector components cannot be more than fqn's
+    if len(flat_fqn) < len(node_selector.split('.')):
+        return False
 
+    for i, selector_part in enumerate(node_selector.split('.')):
         # if we hit a GLOB, then this node is selected
         if selector_part == SELECTOR_GLOB:
             return True
-
-        # match package.node_name or package.dir.node_name
-        elif is_last and selector_part == real_node[-1]:
-            return True
-
-        elif len(real_node) <= i:
-            return False
-
-        elif real_node[i] == selector_part:
+        elif flat_fqn[i] == selector_part:
             continue
-
         else:
             return False
 
@@ -154,31 +152,20 @@ class SelectorMethod(metaclass=abc.ABCMeta):
 
 
 class QualifiedNameSelectorMethod(SelectorMethod):
-    def node_is_match(
-        self,
-        qualified_name: List[str],
-        package_names: Set[str],
-        fqn: List[str],
-    ) -> bool:
-        """Determine if a qualfied name matches an fqn, given the set of package
+    def node_is_match(self, qualified_name: str, fqn: List[str]) -> bool:
+        """Determine if a qualified name matches an fqn for all package
         names in the graph.
 
-        :param List[str] qualified_name: The components of the selector or node
-            name, split on '.'.
-        :param Set[str] package_names: The set of pacakge names in the graph.
+        :param str qualified_name: The qualified name to match the nodes with
         :param List[str] fqn: The node's fully qualified name in the graph.
         """
-        if len(qualified_name) == 1 and fqn[-1] == qualified_name[0]:
+        unscoped_fqn = fqn[1:]
+
+        if is_selected_node(fqn, qualified_name):
+            return True
+        # Match nodes across different packages
+        elif is_selected_node(unscoped_fqn, qualified_name):
             return True
-
-        if qualified_name[0] in package_names:
-            if is_selected_node(fqn, qualified_name):
-                return True
-
-        for package_name in package_names:
-            local_qualified_node_name = [package_name] + qualified_name
-            if is_selected_node(fqn, local_qualified_node_name):
-                return True
 
         return False
 
@@ -189,15 +176,9 @@ class QualifiedNameSelectorMethod(SelectorMethod):
 
         :param str selector: The selector or node name
         """
-        qualified_name = selector.split(".")
         parsed_nodes = list(self.parsed_nodes(included_nodes))
-        package_names = {n.package_name for _, n in parsed_nodes}
         for node, real_node in parsed_nodes:
-            if self.node_is_match(
-                qualified_name,
-                package_names,
-                real_node.fqn,
-            ):
+            if self.node_is_match(selector, real_node.fqn):
                 yield node
 
 

core/dbt/graph/selector_spec.py

@@ -66,6 +66,7 @@ class SelectionCriteria:
     parents_depth: Optional[int]
     children: bool
     children_depth: Optional[int]
+    greedy: bool = False
 
     def __post_init__(self):
         if self.children and self.childrens_parents:
@@ -103,7 +104,7 @@ class SelectionCriteria:
 
     @classmethod
     def selection_criteria_from_dict(
-        cls, raw: Any, dct: Dict[str, Any]
+        cls, raw: Any, dct: Dict[str, Any], greedy: bool = False
     ) -> 'SelectionCriteria':
         if 'value' not in dct:
             raise RuntimeException(
@@ -123,10 +124,11 @@ class SelectionCriteria:
             parents_depth=parents_depth,
             children=bool(dct.get('children')),
             children_depth=children_depth,
+            greedy=greedy
         )
 
     @classmethod
-    def dict_from_single_spec(cls, raw: str):
+    def dict_from_single_spec(cls, raw: str, greedy: bool = False):
         result = RAW_SELECTOR_PATTERN.match(raw)
         if result is None:
             return {'error': 'Invalid selector spec'}
@@ -146,13 +148,13 @@ class SelectionCriteria:
         return dct
 
     @classmethod
-    def from_single_spec(cls, raw: str) -> 'SelectionCriteria':
+    def from_single_spec(cls, raw: str, greedy: bool = False) -> 'SelectionCriteria':
         result = RAW_SELECTOR_PATTERN.match(raw)
         if result is None:
             # bad spec!
             raise RuntimeException(f'Invalid selector spec "{raw}"')
 
-        return cls.selection_criteria_from_dict(raw, result.groupdict())
+        return cls.selection_criteria_from_dict(raw, result.groupdict(), greedy=greedy)
 
 
 class BaseSelectionGroup(Iterable[SelectionSpec], metaclass=ABCMeta):

core/dbt/include/global_project/macros/adapters/common.sql

@@ -51,6 +51,29 @@
 
 {% endmacro %}
 
+{% macro get_create_index_sql(relation, index_dict) -%}
+  {{ return(adapter.dispatch('get_create_index_sql')(relation, index_dict)) }}
+{% endmacro %}
+
+{% macro default__get_create_index_sql(relation, index_dict) -%}
+  {% do return(None) %}
+{% endmacro %}
+
+{% macro create_indexes(relation) -%}
+  {{ adapter.dispatch('create_indexes')(relation) }}
+{%- endmacro %}
+
+{% macro default__create_indexes(relation) -%}
+  {%- set _indexes = config.get('indexes', default=[]) -%}
+
+  {% for _index_dict in _indexes %}
+    {% set create_index_sql = get_create_index_sql(relation, _index_dict) %}
+    {% if create_index_sql %}
+      {% do run_query(create_index_sql) %}
+    {% endif %}
+  {% endfor %}
+{% endmacro %}
+
 {% macro create_view_as(relation, sql) -%}
   {{ adapter.dispatch('create_view_as')(relation, sql) }}
 {%- endmacro %}

core/dbt/include/global_project/macros/materializations/common/merge.sql

@@ -18,6 +18,7 @@
 {% macro default__get_merge_sql(target, source, unique_key, dest_columns, predicates) -%}
     {%- set predicates = [] if predicates is none else [] + predicates -%}
     {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute="name")) -%}
+    {%- set update_columns = config.get('merge_update_columns', default = dest_columns | map(attribute="quoted") | list) -%}
     {%- set sql_header = config.get('sql_header', none) -%}
 
     {% if unique_key %}
@@ -37,8 +38,8 @@
 
     {% if unique_key %}
     when matched then update set
-        {% for column in dest_columns -%}
-            {{ adapter.quote(column.name) }} = DBT_INTERNAL_SOURCE.{{ adapter.quote(column.name) }}
+        {% for column_name in update_columns -%}
+            {{ column_name }} = DBT_INTERNAL_SOURCE.{{ column_name }}
             {%- if not loop.last %}, {%- endif %}
         {%- endfor %}
     {% endif %}

core/dbt/include/global_project/macros/materializations/helpers.sql

@@ -72,3 +72,12 @@
   {% endif %}
   {% do return(config_full_refresh) %}
 {% endmacro %}
+
+
+{% macro should_store_failures() %}
+  {% set config_store_failures = config.get('store_failures') %}
+  {% if config_store_failures is none %}
+    {% set config_store_failures = flags.STORE_FAILURES %}
+  {% endif %}
+  {% do return(config_store_failures) %}
+{% endmacro %}

core/dbt/include/global_project/macros/materializations/incremental/incremental.sql

@@ -5,7 +5,22 @@
 
   {% set target_relation = this.incorporate(type='table') %}
   {% set existing_relation = load_relation(this) %}
-  {% set tmp_relation = make_temp_relation(this) %}
+
+  {% set tmp_identifier = model['name'] + '__dbt_tmp' %}
+  {% set backup_identifier = model['name'] + "__dbt_backup" %}
+
+  -- the intermediate_ and backup_ relations should not already exist in the database; get_relation
+  -- will return None in that case. Otherwise, we get a relation that we can drop
+  -- later, before we try to use this name for the current operation. This has to happen before
+  -- BEGIN, in a separate transaction
+  {% set preexisting_intermediate_relation = adapter.get_relation(identifier=tmp_identifier, 
+                                                                  schema=schema,
+                                                                  database=database) %}                                               
+  {% set preexisting_backup_relation = adapter.get_relation(identifier=backup_identifier,
+                                                            schema=schema,
+                                                            database=database) %}
+  {{ drop_relation_if_exists(preexisting_intermediate_relation) }}
+  {{ drop_relation_if_exists(preexisting_backup_relation) }}
 
   {{ run_hooks(pre_hooks, inside_transaction=False) }}
 
@@ -16,13 +31,11 @@
   {% if existing_relation is none %}
       {% set build_sql = create_table_as(False, target_relation, sql) %}
   {% elif existing_relation.is_view or should_full_refresh() %}
-      {#-- Make sure the backup doesn't exist so we don't encounter issues with the rename below #}
-      {% set backup_identifier = existing_relation.identifier ~ "__dbt_backup" %}
+      {% set intermediate_relation = existing_relation.incorporate(path={"identifier": tmp_identifier}) %}
       {% set backup_relation = existing_relation.incorporate(path={"identifier": backup_identifier}) %}
-      {% do adapter.drop_relation(backup_relation) %}
 
-      {% do adapter.rename_relation(target_relation, backup_relation) %}
-      {% set build_sql = create_table_as(False, target_relation, sql) %}
+      {% set build_sql = create_table_as(False, intermediate_relation, sql) %}
+      {% set need_swap = true %}
       {% do to_drop.append(backup_relation) %}
   {% else %}
       {% set tmp_relation = make_temp_relation(target_relation) %}
@@ -37,8 +50,17 @@
       {{ build_sql }}
   {% endcall %}
 
+  {% if need_swap %} 
+      {% do adapter.rename_relation(target_relation, backup_relation) %} 
+      {% do adapter.rename_relation(intermediate_relation, target_relation) %} 
+  {% endif %}
+
   {% do persist_docs(target_relation, model) %}
 
+  {% if existing_relation is none or existing_relation.is_view or should_full_refresh() %}
+    {% do create_indexes(target_relation) %}
+  {% endif %}
+
   {{ run_hooks(post_hooks, inside_transaction=True) }}
 
   -- `COMMIT` happens here

core/dbt/include/global_project/macros/materializations/seed/seed.sql

@@ -142,6 +142,10 @@
   {% set target_relation = this.incorporate(type='table') %}
   {% do persist_docs(target_relation, model) %}
 
+  {% if full_refresh_mode or not exists_as_table %}
+    {% do create_indexes(target_relation) %}
+  {% endif %}
+
   {{ run_hooks(post_hooks, inside_transaction=True) }}
 
   -- `COMMIT` happens here

core/dbt/include/global_project/macros/materializations/snapshot/snapshot.sql

@@ -263,6 +263,10 @@
 
   {% do persist_docs(target_relation, model) %}
 
+  {% if not target_relation_exists %}
+    {% do create_indexes(target_relation) %}
+  {% endif %}
+
   {{ run_hooks(post_hooks, inside_transaction=True) }}
 
   {{ adapter.commit() }}

core/dbt/include/global_project/macros/materializations/snapshot/strategies.sql

@@ -144,7 +144,7 @@
     {% if now is none or now is undefined -%}
         {%- do exceptions.raise_compiler_error('Could not get a snapshot start time from the database') -%}
     {%- endif %}
-    {% set updated_at = snapshot_string_as_time(now) %}
+    {% set updated_at = config.get('updated_at', snapshot_string_as_time(now)) %}
 
     {% set column_added = false %}
 

core/dbt/include/global_project/macros/materializations/table/table.sql

@@ -12,7 +12,12 @@
                                                       schema=schema,
                                                       database=database,
                                                       type='table') -%}
-
+  -- the intermediate_relation should not already exist in the database; get_relation
+  -- will return None in that case. Otherwise, we get a relation that we can drop
+  -- later, before we try to use this name for the current operation
+  {%- set preexisting_intermediate_relation = adapter.get_relation(identifier=tmp_identifier, 
+                                                                   schema=schema,
+                                                                   database=database) -%}
   /*
       See ../view/view.sql for more information about this relation.
   */
@@ -21,14 +26,15 @@
                                                 schema=schema,
                                                 database=database,
                                                 type=backup_relation_type) -%}
-
-  {%- set exists_as_table = (old_relation is not none and old_relation.is_table) -%}
-  {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%}
+  -- as above, the backup_relation should not already exist
+  {%- set preexisting_backup_relation = adapter.get_relation(identifier=backup_identifier,
+                                                             schema=schema,
+                                                             database=database) -%}
 
 
-  -- drop the temp relations if they exists for some reason
-  {{ adapter.drop_relation(intermediate_relation) }}
-  {{ adapter.drop_relation(backup_relation) }}
+  -- drop the temp relations if they exist already in the database
+  {{ drop_relation_if_exists(preexisting_intermediate_relation) }}
+  {{ drop_relation_if_exists(preexisting_backup_relation) }}
 
   {{ run_hooks(pre_hooks, inside_transaction=False) }}
 
@@ -47,6 +53,8 @@
 
   {{ adapter.rename_relation(intermediate_relation, target_relation) }}
 
+  {% do create_indexes(target_relation) %}
+
   {{ run_hooks(post_hooks, inside_transaction=True) }}
 
   {% do persist_docs(target_relation, model) %}

core/dbt/include/global_project/macros/materializations/test.sql

@@ -0,0 +1,64 @@
+{% macro get_test_sql(main_sql, fail_calc, warn_if, error_if, limit) -%}
+  {{ adapter.dispatch('get_test_sql')(main_sql, fail_calc, warn_if, error_if, limit) }}
+{%- endmacro %}
+
+
+{% macro default__get_test_sql(main_sql, fail_calc, warn_if, error_if, limit) -%}
+    select
+      {{ fail_calc }} as failures,
+      {{ fail_calc }} {{ warn_if }} as should_warn,
+      {{ fail_calc }} {{ error_if }} as should_error
+    from (
+      {{ main_sql }}
+      {{ "limit " ~ limit if limit != none }}
+    ) dbt_internal_test
+{%- endmacro %}
+
+{%- materialization test, default -%}
+
+  {% set relations = [] %}
+
+  {% if should_store_failures() %}
+
+    {% set identifier = model['alias'] %}
+    {% set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) %}
+    {% set target_relation = api.Relation.create(
+        identifier=identifier, schema=schema, database=database, type='table') -%} %}
+    
+    {% if old_relation %}
+        {% do adapter.drop_relation(old_relation) %}
+    {% endif %}
+    
+    {% call statement(auto_begin=True) %}
+        {{ create_table_as(False, target_relation, sql) }}
+    {% endcall %}
+    
+    {% do relations.append(target_relation) %}
+  
+    {% set main_sql %}
+        select *
+        from {{ target_relation }}
+    {% endset %}
+    
+    {{ adapter.commit() }}
+  
+  {% else %}
+
+      {% set main_sql = sql %}
+  
+  {% endif %}
+
+  {% set limit = config.get('limit') %}
+  {% set fail_calc = config.get('fail_calc') %}
+  {% set warn_if = config.get('warn_if') %}
+  {% set error_if = config.get('error_if') %}
+
+  {% call statement('main', fetch_result=True) -%}
+
+    {{ get_test_sql(main_sql, fail_calc, warn_if, error_if, limit)}}
+
+  {%- endcall %}
+  
+  {{ return({'relations': relations}) }}
+
+{%- endmaterialization -%}

core/dbt/include/global_project/macros/materializations/view/create_or_replace_view.sql

@@ -1,6 +1,6 @@
 
 {% macro handle_existing_table(full_refresh, old_relation) %}
-    {{ adapter.dispatch("handle_existing_table", packages=['dbt'])(full_refresh, old_relation) }}
+    {{ adapter.dispatch('handle_existing_table', macro_namespace = 'dbt')(full_refresh, old_relation) }}
 {% endmacro %}
 
 {% macro default__handle_existing_table(full_refresh, old_relation) %}

core/dbt/include/global_project/macros/materializations/view/view.sql

@@ -9,7 +9,12 @@
                                                 type='view') -%}
   {%- set intermediate_relation = api.Relation.create(identifier=tmp_identifier,
                                                       schema=schema, database=database, type='view') -%}
-
+  -- the intermediate_relation should not already exist in the database; get_relation
+  -- will return None in that case. Otherwise, we get a relation that we can drop
+  -- later, before we try to use this name for the current operation
+  {%- set preexisting_intermediate_relation = adapter.get_relation(identifier=tmp_identifier, 
+                                                                   schema=schema,
+                                                                   database=database) -%}
   /*
      This relation (probably) doesn't exist yet. If it does exist, it's a leftover from
      a previous run, and we're going to try to drop it immediately. At the end of this
@@ -27,14 +32,16 @@
   {%- set backup_relation = api.Relation.create(identifier=backup_identifier,
                                                 schema=schema, database=database,
                                                 type=backup_relation_type) -%}
-
-  {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%}
+  -- as above, the backup_relation should not already exist
+  {%- set preexisting_backup_relation = adapter.get_relation(identifier=backup_identifier,
+                                                             schema=schema,
+                                                             database=database) -%}
 
   {{ run_hooks(pre_hooks, inside_transaction=False) }}
 
-  -- drop the temp relations if they exists for some reason
-  {{ adapter.drop_relation(intermediate_relation) }}
-  {{ adapter.drop_relation(backup_relation) }}
+  -- drop the temp relations if they exist already in the database
+  {{ drop_relation_if_exists(preexisting_intermediate_relation) }}
+  {{ drop_relation_if_exists(preexisting_backup_relation) }}
 
   -- `BEGIN` happens here:
   {{ run_hooks(pre_hooks, inside_transaction=True) }}

core/dbt/include/global_project/macros/schema_tests/accepted_values.sql

@@ -1,43 +1,33 @@
-
-{% macro default__test_accepted_values(model, values) %}
-
-{% set column_name = kwargs.get('column_name', kwargs.get('field')) %}
-{% set quote_values = kwargs.get('quote', True) %}
+{% macro default__test_accepted_values(model, column_name, values, quote=True) %}
 
 with all_values as (
 
-    select distinct
-        {{ column_name }} as value_field
+    select
+        {{ column_name }} as value_field,
+        count(*) as n_records
 
     from {{ model }}
+    group by 1
 
-),
-
-validation_errors as (
-
-    select
-        value_field
-
-    from all_values
-    where value_field not in (
-        {% for value in values -%}
-            {% if quote_values -%}
-            '{{ value }}'
-            {%- else -%}
-            {{ value }}
-            {%- endif -%}
-            {%- if not loop.last -%},{%- endif %}
-        {%- endfor %}
-    )
 )
 
-select count(*) as validation_errors
-from validation_errors
+select *
+from all_values
+where value_field not in (
+    {% for value in values -%}
+        {% if quote -%}
+        '{{ value }}'
+        {%- else -%}
+        {{ value }}
+        {%- endif -%}
+        {%- if not loop.last -%},{%- endif %}
+    {%- endfor %}
+)
 
 {% endmacro %}
 
 
-{% macro test_accepted_values(model, values) %}
+{% test accepted_values(model, column_name, values, quote=True) %}
     {% set macro = adapter.dispatch('test_accepted_values') %}
-    {{ macro(model, values, **kwargs) }}
-{% endmacro %}
+    {{ macro(model, column_name, values, quote) }}
+{% endtest %}

core/dbt/include/global_project/macros/schema_tests/not_null.sql

@@ -1,17 +1,13 @@
+{% macro default__test_not_null(model, column_name) %}
 
-{% macro default__test_not_null(model) %}
-
-{% set column_name = kwargs.get('column_name', kwargs.get('arg')) %}
-
-select count(*) as validation_errors
+select *
 from {{ model }}
 where {{ column_name }} is null
 
 {% endmacro %}
 
 
-
-{% macro test_not_null(model) %}
+{% test not_null(model, column_name) %}
     {% set macro = adapter.dispatch('test_not_null') %}
-    {{ macro(model, **kwargs) }}
-{% endmacro %}
+    {{ macro(model, column_name) }}
+{% endtest %}

core/dbt/include/global_project/macros/schema_tests/relationships.sql

@@ -1,24 +1,21 @@
 
-{% macro default__test_relationships(model, to, field) %}
+{% macro default__test_relationships(model, column_name, to, field) %}
 
-{% set column_name = kwargs.get('column_name', kwargs.get('from')) %}
+select
+    child.{{ column_name }}
 
+from {{ model }} as child
 
-select count(*) as validation_errors
-from (
-    select {{ column_name }} as id from {{ model }}
-) as child
-left join (
-    select {{ field }} as id from {{ to }}
-) as parent on parent.id = child.id
-where child.id is not null
-  and parent.id is null
+left join {{ to }} as parent
+    on child.{{ column_name }} = parent.{{ field }}
+
+where child.{{ column_name }} is not null
+  and parent.{{ field }} is null
 
 {% endmacro %}
 
 
-
-{% macro test_relationships(model, to, field) %}
+{% test relationships(model, column_name, to, field) %}
     {% set macro = adapter.dispatch('test_relationships') %}
-    {{ macro(model, to, field, **kwargs) }}
-{% endmacro %}
+    {{ macro(model, column_name, to, field) }}
+{% endtest %}

core/dbt/include/global_project/macros/schema_tests/unique.sql

@@ -1,25 +1,18 @@
+{% macro default__test_unique(model, column_name) %}
 
-{% macro default__test_unique(model) %}
+select
+    {{ column_name }},
+    count(*) as n_records
 
-{% set column_name = kwargs.get('column_name', kwargs.get('arg')) %}
-
-select count(*) as validation_errors
-from (
-
-    select
-        {{ column_name }}
-
-    from {{ model }}
-    where {{ column_name }} is not null
-    group by {{ column_name }}
-    having count(*) > 1
-
-) validation_errors
+from {{ model }}
+where {{ column_name }} is not null
+group by {{ column_name }}
+having count(*) > 1
 
 {% endmacro %}
 
 
-{% macro test_unique(model) %}
+{% test unique(model, column_name) %}
     {% set macro = adapter.dispatch('test_unique') %}
-    {{ macro(model, **kwargs) }}
-{% endmacro %}
+    {{ macro(model, column_name) }}
+{% endtest %}

core/dbt/include/starter_project/.gitignore

@@ -0,0 +1,4 @@
+
+target/
+dbt_modules/
+logs/

core/dbt/include/starter_project/README.md

@@ -0,0 +1,15 @@
+Welcome to your new dbt project!
+
+### Using the starter project
+
+Try running the following commands:
+- dbt run
+- dbt test
+
+
+### Resources:
+- Learn more about dbt [in the docs](https://docs.getdbt.com/docs/introduction)
+- Check out [Discourse](https://discourse.getdbt.com/) for commonly asked questions and answers
+- Join the [chat](https://community.getdbt.com/) on Slack for live discussions and support
+- Find [dbt events](https://events.getdbt.com) near you
+- Check out [the blog](https://blog.getdbt.com/) for the latest news on dbt's development and best practices

core/dbt/include/starter_project/__init__.py

@@ -0,0 +1,3 @@
+import os
+
+PACKAGE_PATH = os.path.dirname(__file__)

core/dbt/include/starter_project/dbt_project.yml

@@ -0,0 +1,38 @@
+
+# Name your project! Project names should contain only lowercase characters
+# and underscores. A good package name should reflect your organization's
+# name or the intended use of these models
+name: 'my_new_project'
+version: '1.0.0'
+config-version: 2
+
+# This setting configures which "profile" dbt uses for this project.
+profile: 'default'
+
+# These configurations specify where dbt should look for different types of files.
+# The `source-paths` config, for example, states that models in this project can be
+# found in the "models/" directory. You probably won't need to change these!
+source-paths: ["models"]
+analysis-paths: ["analysis"]
+test-paths: ["tests"]
+data-paths: ["data"]
+macro-paths: ["macros"]
+snapshot-paths: ["snapshots"]
+
+target-path: "target"  # directory which will store compiled SQL files
+clean-targets:         # directories to be removed by `dbt clean`
+  - "target"
+  - "dbt_modules"
+
+
+# Configuring models
+# Full documentation: https://docs.getdbt.com/docs/configuring-models
+
+# In this example config, we tell dbt to build all models in the example/ directory
+# as tables. These settings can be overridden in the individual model files
+# using the `{{ config(...) }}` macro.
+models:
+  my_new_project:
+    # Config indicated by + and applies to all files under models/example/
+    example:
+      +materialized: view

core/dbt/include/starter_project/models/example/my_first_dbt_model.sql

@@ -0,0 +1,27 @@
+
+/*
+    Welcome to your first dbt model!
+    Did you know that you can also configure models directly within SQL files?
+    This will override configurations stated in dbt_project.yml
+
+    Try changing "table" to "view" below
+*/
+
+{{ config(materialized='table') }}
+
+with source_data as (
+
+    select 1 as id
+    union all
+    select null as id
+
+)
+
+select *
+from source_data
+
+/*
+    Uncomment the line below to remove records with null `id` values
+*/
+
+-- where id is not null

core/dbt/include/starter_project/models/example/my_second_dbt_model.sql

@@ -0,0 +1,6 @@
+
+-- Use the `ref` function to select from other models
+
+select *
+from {{ ref('my_first_dbt_model') }}
+where id = 1

core/dbt/include/starter_project/models/example/schema.yml

@@ -0,0 +1,21 @@
+
+version: 2
+
+models:
+  - name: my_first_dbt_model
+    description: "A starter dbt model"
+    columns:
+      - name: id
+        description: "The primary key for this table"
+        tests:
+          - unique
+          - not_null
+
+  - name: my_second_dbt_model
+    description: "A starter dbt model"
+    columns:
+      - name: id
+        description: "The primary key for this table"
+        tests:
+          - unique
+          - not_null

core/dbt/main.py

@@ -11,6 +11,7 @@ from pathlib import Path
 import dbt.version
 import dbt.flags as flags
 import dbt.task.run as run_task
+import dbt.task.build as build_task
 import dbt.task.compile as compile_task
 import dbt.task.debug as debug_task
 import dbt.task.clean as clean_task
@@ -367,7 +368,6 @@ def _build_init_subparser(subparsers, base_subparser):
     )
     sub.add_argument(
         '--adapter',
-        default='redshift',
         type=str,
         help='''
         Write sample profiles.yml for which adapter
@@ -377,6 +377,30 @@ def _build_init_subparser(subparsers, base_subparser):
     return sub
 
 
+def _build_build_subparser(subparsers, base_subparser):
+    sub = subparsers.add_parser(
+        'build',
+        parents=[base_subparser],
+        help='''
+        Run all Seeds, Models, Snapshots, and tests in DAG order
+        '''
+    )
+    sub.set_defaults(
+        cls=build_task.BuildTask,
+        which='build',
+        rpc_method='build'
+    )
+    sub.add_argument(
+        '-x',
+        '--fail-fast',
+        action='store_true',
+        help='''
+        Stop execution upon a first failure.
+        '''
+    )
+    return sub
+
+
 def _build_clean_subparser(subparsers, base_subparser):
     sub = subparsers.add_parser(
         'clean',
@@ -719,6 +743,13 @@ def _build_test_subparser(subparsers, base_subparser):
         Stop execution upon a first test failure.
         '''
     )
+    sub.add_argument(
+        '--store-failures',
+        action='store_true',
+        help='''
+        Store test results (failing rows) in the database
+        '''
+    )
 
     sub.set_defaults(cls=test_task.TestTask, which='test', rpc_method='test')
     return sub
@@ -998,12 +1029,23 @@ def parse_args(args, cls=DBTArgumentParser):
     # if set, extract all models and blocks with the jinja block extractor, and
     # verify that we don't fail anywhere the actual jinja parser passes. The
     # reverse (passing files that ends up failing jinja) is fine.
+    # TODO remove?
     p.add_argument(
         '--test-new-parser',
         action='store_true',
         help=argparse.SUPPRESS
     )
 
+    # if set, will use the tree-sitter-jinja2 parser and extractor instead of
+    # jinja rendering when possible.
+    p.add_argument(
+        '--use-experimental-parser',
+        action='store_true',
+        help='''
+        Uses an experimental parser to extract jinja values.
+        '''
+    )
+
     subs = p.add_subparsers(title="Available sub-commands")
 
     base_subparser = _build_base_subparser()
@@ -1020,6 +1062,7 @@ def parse_args(args, cls=DBTArgumentParser):
     _build_deps_subparser(subs, base_subparser)
     _build_list_subparser(subs, base_subparser)
 
+    build_sub = _build_build_subparser(subs, base_subparser)
     snapshot_sub = _build_snapshot_subparser(subs, base_subparser)
     rpc_sub = _build_rpc_subparser(subs, base_subparser)
     run_sub = _build_run_subparser(subs, base_subparser)
@@ -1033,7 +1076,7 @@ def parse_args(args, cls=DBTArgumentParser):
                           rpc_sub, seed_sub, parse_sub)
     # --models, --exclude
     # list_sub sets up its own arguments.
-    _add_selection_arguments(run_sub, compile_sub, generate_sub, test_sub)
+    _add_selection_arguments(build_sub, run_sub, compile_sub, generate_sub, test_sub)
     _add_selection_arguments(snapshot_sub, seed_sub, models_name='select')
     # --defer
     _add_defer_argument(run_sub, test_sub)
@@ -1051,7 +1094,7 @@ def parse_args(args, cls=DBTArgumentParser):
     parsed = p.parse_args(args)
 
     if hasattr(parsed, 'profiles_dir'):
-        parsed.profiles_dir = os.path.expanduser(parsed.profiles_dir)
+        parsed.profiles_dir = os.path.abspath(parsed.profiles_dir)
 
     if getattr(parsed, 'project_dir', None) is not None:
         expanded_user = os.path.expanduser(parsed.project_dir)

core/dbt/parser/__init__.py

@@ -5,12 +5,11 @@ from .docs import DocumentationParser  # noqa
 from .hooks import HookParser  # noqa
 from .macros import MacroParser  # noqa
 from .models import ModelParser  # noqa
-from .results import ParseResult  # noqa
 from .schemas import SchemaParser  # noqa
 from .seeds import SeedParser  # noqa
 from .snapshots import SnapshotParser  # noqa
 
 from . import (  # noqa
-    analysis, base, data_test, docs, hooks, macros, models, results, schemas,
+    analysis, base, data_test, docs, hooks, macros, models, schemas,
     snapshots
 )

core/dbt/parser/analysis.py

@@ -3,15 +3,10 @@ import os
 from dbt.contracts.graph.parsed import ParsedAnalysisNode
 from dbt.node_types import NodeType
 from dbt.parser.base import SimpleSQLParser
-from dbt.parser.search import FilesystemSearcher, FileBlock
+from dbt.parser.search import FileBlock
 
 
 class AnalysisParser(SimpleSQLParser[ParsedAnalysisNode]):
-    def get_paths(self):
-        return FilesystemSearcher(
-            self.project, self.project.analysis_paths, '.sql'
-        )
-
     def parse_from_dict(self, dct, validate=True) -> ParsedAnalysisNode:
         if validate:
             ParsedAnalysisNode.validate(dct)

core/dbt/parser/base.py

@@ -2,14 +2,13 @@ import abc
 import itertools
 import os
 from typing import (
-    List, Dict, Any, Iterable, Generic, TypeVar
+    List, Dict, Any, Generic, Optional, TypeVar
 )
 
 from dbt.dataclass_schema import ValidationError
 
 from dbt import utils
 from dbt.clients.jinja import MacroGenerator
-from dbt.clients.system import load_file_contents
 from dbt.context.providers import (
     generate_parser_model,
     generate_generate_component_name_macro,
@@ -20,18 +19,14 @@ from dbt.config import Project, RuntimeConfig
 from dbt.context.context_config import (
     ContextConfig
 )
-from dbt.contracts.files import (
-    SourceFile, FilePath, FileHash
-)
-from dbt.contracts.graph.manifest import MacroManifest
-from dbt.contracts.graph.parsed import HasUniqueID
+from dbt.contracts.graph.manifest import Manifest
+from dbt.contracts.graph.parsed import HasUniqueID, ManifestNodes
 from dbt.contracts.graph.unparsed import UnparsedNode
 from dbt.exceptions import (
     CompilationException, validator_error_message, InternalException
 )
 from dbt import hooks
 from dbt.node_types import NodeType
-from dbt.parser.results import ParseResult, ManifestNodes
 from dbt.parser.search import FileBlock
 
 # internally, the parser may store a less-restrictive type that will be
@@ -48,20 +43,9 @@ ConfiguredBlockType = TypeVar('ConfiguredBlockType', bound=FileBlock)
 
 
 class BaseParser(Generic[FinalValue]):
-    def __init__(self, results: ParseResult, project: Project) -> None:
-        self.results = results
+    def __init__(self, project: Project, manifest: Manifest) -> None:
         self.project = project
-        # this should be a superset of [x.path for x in self.results.files]
-        # because we fill it via search()
-        self.searched: List[FilePath] = []
-
-    @abc.abstractmethod
-    def get_paths(self) -> Iterable[FilePath]:
-        pass
-
-    def search(self) -> List[FilePath]:
-        self.searched = list(self.get_paths())
-        return self.searched
+        self.manifest = manifest
 
     @abc.abstractmethod
     def parse_file(self, block: FileBlock) -> None:
@@ -71,47 +55,44 @@ class BaseParser(Generic[FinalValue]):
     def resource_type(self) -> NodeType:
         pass
 
-    def generate_unique_id(self, resource_name: str) -> str:
-        """Returns a unique identifier for a resource"""
-        return "{}.{}.{}".format(self.resource_type,
-                                 self.project.project_name,
-                                 resource_name)
-
-    def load_file(
+    def generate_unique_id(
         self,
-        path: FilePath,
-        *,
-        set_contents: bool = True,
-    ) -> SourceFile:
-        file_contents = load_file_contents(path.absolute_path, strip=False)
-        checksum = FileHash.from_contents(file_contents)
-        source_file = SourceFile(path=path, checksum=checksum)
-        if set_contents:
-            source_file.contents = file_contents.strip()
-        else:
-            source_file.contents = ''
-        return source_file
+        resource_name: str,
+        hash: Optional[str] = None
+    ) -> str:
+        """Returns a unique identifier for a resource
+        An optional hash may be passed in to ensure uniqueness for edge cases"""
+
+        return '.'.join(
+            filter(
+                None,
+                [
+                    self.resource_type,
+                    self.project.project_name,
+                    resource_name,
+                    hash
+                ]
+            )
+        )
 
 
 class Parser(BaseParser[FinalValue], Generic[FinalValue]):
     def __init__(
         self,
-        results: ParseResult,
         project: Project,
+        manifest: Manifest,
         root_project: RuntimeConfig,
-        macro_manifest: MacroManifest,
     ) -> None:
-        super().__init__(results, project)
+        super().__init__(project, manifest)
         self.root_project = root_project
-        self.macro_manifest = macro_manifest
 
 
 class RelationUpdate:
     def __init__(
-        self, config: RuntimeConfig, macro_manifest: MacroManifest,
+        self, config: RuntimeConfig, manifest: Manifest,
         component: str
     ) -> None:
-        macro = macro_manifest.find_generate_macro_by_name(
+        macro = manifest.find_generate_macro_by_name(
             component=component,
             root_project_name=config.project_name,
         )
@@ -121,7 +102,7 @@ class RelationUpdate:
             )
 
         root_context = generate_generate_component_name_macro(
-            macro, config, macro_manifest
+            macro, config, manifest
         )
         self.updater = MacroGenerator(macro, root_context)
         self.component = component
@@ -142,23 +123,22 @@ class ConfiguredParser(
 ):
     def __init__(
         self,
-        results: ParseResult,
         project: Project,
+        manifest: Manifest,
         root_project: RuntimeConfig,
-        macro_manifest: MacroManifest,
     ) -> None:
-        super().__init__(results, project, root_project, macro_manifest)
+        super().__init__(project, manifest, root_project)
 
         self._update_node_database = RelationUpdate(
-            macro_manifest=macro_manifest, config=root_project,
+            manifest=manifest, config=root_project,
             component='database'
         )
         self._update_node_schema = RelationUpdate(
-            macro_manifest=macro_manifest, config=root_project,
+            manifest=manifest, config=root_project,
             component='schema'
         )
         self._update_node_alias = RelationUpdate(
-            macro_manifest=macro_manifest, config=root_project,
+            manifest=manifest, config=root_project,
             component='alias'
         )
 
@@ -273,7 +253,7 @@ class ConfiguredParser(
         self, parsed_node: IntermediateNode, config: ContextConfig
     ) -> Dict[str, Any]:
         return generate_parser_model(
-            parsed_node, self.root_project, self.macro_manifest, config
+            parsed_node, self.root_project, self.manifest, config
         )
 
     def render_with_context(
@@ -302,7 +282,7 @@ class ConfiguredParser(
     ) -> None:
         # Overwrite node config
         final_config_dict = parsed_node.config.to_dict(omit_none=True)
-        final_config_dict.update(config_dict)
+        final_config_dict.update({k.strip(): v for (k, v) in config_dict.items()})
         # re-mangle hooks, in case we got new ones
         self._mangle_hooks(final_config_dict)
         parsed_node.config = parsed_node.config.from_dict(final_config_dict)
@@ -386,9 +366,9 @@ class ConfiguredParser(
 
     def add_result_node(self, block: FileBlock, node: ManifestNodes):
         if node.config.enabled:
-            self.results.add_node(block.file, node)
+            self.manifest.add_node(block.file, node)
         else:
-            self.results.add_disabled(block.file, node)
+            self.manifest.add_disabled(block.file, node)
 
     def parse_node(self, block: ConfiguredBlockType) -> FinalNode:
         compiled_path: str = self.get_compiled_path(block)

core/dbt/parser/data_test.py

@@ -1,16 +1,11 @@
 from dbt.contracts.graph.parsed import ParsedDataTestNode
 from dbt.node_types import NodeType
 from dbt.parser.base import SimpleSQLParser
-from dbt.parser.search import FilesystemSearcher, FileBlock
+from dbt.parser.search import FileBlock
 from dbt.utils import get_pseudo_test_path
 
 
 class DataTestParser(SimpleSQLParser[ParsedDataTestNode]):
-    def get_paths(self):
-        return FilesystemSearcher(
-            self.project, self.project.test_paths, '.sql'
-        )
-
     def parse_from_dict(self, dct, validate=True) -> ParsedDataTestNode:
         if validate:
             ParsedDataTestNode.validate(dct)

core/dbt/parser/docs.py

@@ -1,13 +1,14 @@
-from typing import Iterable
+from typing import Iterable, Optional
 
 import re
 
 from dbt.clients.jinja import get_rendered
+from dbt.contracts.files import SourceFile
 from dbt.contracts.graph.parsed import ParsedDocumentation
 from dbt.node_types import NodeType
 from dbt.parser.base import Parser
 from dbt.parser.search import (
-    BlockContents, FileBlock, FilesystemSearcher, BlockSearcher
+    BlockContents, FileBlock, BlockSearcher
 )
 
 
@@ -15,13 +16,6 @@ SHOULD_PARSE_RE = re.compile(r'{[{%]')
 
 
 class DocumentationParser(Parser[ParsedDocumentation]):
-    def get_paths(self):
-        return FilesystemSearcher(
-            project=self.project,
-            relative_dirs=self.project.docs_paths,
-            extension='.md',
-        )
-
     @property
     def resource_type(self) -> NodeType:
         return NodeType.Documentation
@@ -30,7 +24,7 @@ class DocumentationParser(Parser[ParsedDocumentation]):
     def get_compiled_path(cls, block: FileBlock):
         return block.path.relative_path
 
-    def generate_unique_id(self, resource_name: str) -> str:
+    def generate_unique_id(self, resource_name: str, _: Optional[str] = None) -> str:
         # because docs are in their own graph namespace, node type doesn't
         # need to be part of the unique ID.
         return '{}.{}'.format(self.project.project_name, resource_name)
@@ -53,6 +47,7 @@ class DocumentationParser(Parser[ParsedDocumentation]):
         return [doc]
 
     def parse_file(self, file_block: FileBlock):
+        assert isinstance(file_block.file, SourceFile)
         searcher: Iterable[BlockContents] = BlockSearcher(
             source=[file_block],
             allowed_blocks={'docs'},
@@ -60,6 +55,4 @@ class DocumentationParser(Parser[ParsedDocumentation]):
         )
         for block in searcher:
             for parsed in self.parse_block(block):
-                self.results.add_doc(file_block.file, parsed)
-        # mark the file as seen, even if there are no macros in it
-        self.results.get_file(file_block.file)
+                self.manifest.add_doc(file_block.file, parsed)

core/dbt/parser/hooks.py

@@ -70,13 +70,14 @@ class HookParser(SimpleParser[HookBlock, ParsedHookNode]):
     def transform(self, node):
         return node
 
-    def get_paths(self) -> List[FilePath]:
+    # Hooks are only in the dbt_project.yml file for the project
+    def get_path(self) -> FilePath:
         path = FilePath(
             project_root=self.project.project_root,
             searched_path='.',
             relative_path='dbt_project.yml',
         )
-        return [path]
+        return path
 
     def parse_from_dict(self, dct, validate=True) -> ParsedHookNode:
         if validate:

core/dbt/parser/macros.py

@@ -5,6 +5,7 @@ import jinja2
 from dbt.clients import jinja
 from dbt.contracts.graph.unparsed import UnparsedMacro
 from dbt.contracts.graph.parsed import ParsedMacro
+from dbt.contracts.files import FilePath, SourceFile
 from dbt.exceptions import CompilationException
 from dbt.logger import GLOBAL_LOGGER as logger
 from dbt.node_types import NodeType
@@ -14,12 +15,14 @@ from dbt.utils import MACRO_PREFIX
 
 
 class MacroParser(BaseParser[ParsedMacro]):
-    def get_paths(self):
-        return FilesystemSearcher(
+    # This is only used when creating a MacroManifest separate
+    # from the normal parsing flow.
+    def get_paths(self) -> List[FilePath]:
+        return list(FilesystemSearcher(
             project=self.project,
             relative_dirs=self.project.macro_paths,
             extension='.sql',
-        )
+        ))
 
     @property
     def resource_type(self) -> NodeType:
@@ -53,7 +56,7 @@ class MacroParser(BaseParser[ParsedMacro]):
                 t for t in
                 jinja.extract_toplevel_blocks(
                     base_node.raw_sql,
-                    allowed_blocks={'macro', 'materialization'},
+                    allowed_blocks={'macro', 'materialization', 'test'},
                     collect_raw_data=False,
                 )
                 if isinstance(t, jinja.BlockTag)
@@ -89,12 +92,10 @@ class MacroParser(BaseParser[ParsedMacro]):
             yield node
 
     def parse_file(self, block: FileBlock):
-        # mark the file as seen, even if there are no macros in it
-        self.results.get_file(block.file)
+        assert isinstance(block.file, SourceFile)
         source_file = block.file
-
+        assert isinstance(source_file.contents, str)
         original_file_path = source_file.path.original_file_path
-
         logger.debug("Parsing {}".format(original_file_path))
 
         # this is really only used for error messages
@@ -108,4 +109,4 @@ class MacroParser(BaseParser[ParsedMacro]):
         )
 
         for node in self.parse_unparsed_macros(base_node):
-            self.results.add_macro(block.file, node)
+            self.manifest.add_macro(block.file, node)

core/dbt/parser/models.py

@@ -1,15 +1,18 @@
+from dbt.context.context_config import ContextConfig
 from dbt.contracts.graph.parsed import ParsedModelNode
+import dbt.flags as flags
 from dbt.node_types import NodeType
 from dbt.parser.base import SimpleSQLParser
-from dbt.parser.search import FilesystemSearcher, FileBlock
+from dbt.parser.search import FileBlock
+import dbt.tracking as tracking
+from dbt import utils
+from dbt_extractor import ExtractionError, py_extract_from_source  # type: ignore
+import itertools
+import random
+from typing import Any, Dict, List, Tuple
 
 
 class ModelParser(SimpleSQLParser[ParsedModelNode]):
-    def get_paths(self):
-        return FilesystemSearcher(
-            self.project, self.project.source_paths, '.sql'
-        )
-
     def parse_from_dict(self, dct, validate=True) -> ParsedModelNode:
         if validate:
             ParsedModelNode.validate(dct)
@@ -22,3 +25,128 @@ class ModelParser(SimpleSQLParser[ParsedModelNode]):
     @classmethod
     def get_compiled_path(cls, block: FileBlock):
         return block.path.relative_path
+
+    def render_update(
+        self, node: ParsedModelNode, config: ContextConfig
+    ) -> None:
+        self.manifest._parsing_info.static_analysis_path_count += 1
+
+        # `True` roughly 1/100 times this function is called
+        sample: bool = random.randint(1, 101) == 100
+
+        # run the experimental parser if the flag is on or if we're sampling
+        if flags.USE_EXPERIMENTAL_PARSER or sample:
+            try:
+                experimentally_parsed: Dict[str, List[Any]] = py_extract_from_source(node.raw_sql)
+
+                # second config format
+                config_calls: List[Dict[str, str]] = []
+                for c in experimentally_parsed['configs']:
+                    config_calls.append({c[0]: c[1]})
+
+                # format sources TODO change extractor to match this type
+                source_calls: List[List[str]] = []
+                for s in experimentally_parsed['sources']:
+                    source_calls.append([s[0], s[1]])
+                experimentally_parsed['sources'] = source_calls
+
+            except ExtractionError as e:
+                experimentally_parsed = e
+
+        # normal dbt run
+        if not flags.USE_EXPERIMENTAL_PARSER:
+            # normal rendering
+            super().render_update(node, config)
+            # if we're sampling, compare for correctness
+            if sample:
+                result: List[str] = []
+                # experimental parser couldn't parse
+                if isinstance(experimentally_parsed, Exception):
+                    result += ["01_experimental_parser_cannot_parse"]
+                else:
+                    # rearrange existing configs to match:
+                    real_configs: List[Tuple[str, Any]] = list(
+                        itertools.chain.from_iterable(
+                            map(lambda x: x.items(), config._config_calls)
+                        )
+                    )
+
+                    # look for false positive configs
+                    for c in experimentally_parsed['configs']:
+                        if c not in real_configs:
+                            result += ["02_false_positive_config_value"]
+                            break
+
+                    # look for missed configs
+                    for c in real_configs:
+                        if c not in experimentally_parsed['configs']:
+                            result += ["03_missed_config_value"]
+                            break
+
+                    # look for false positive sources
+                    for s in experimentally_parsed['sources']:
+                        if s not in node.sources:
+                            result += ["04_false_positive_source_value"]
+                            break
+
+                    # look for missed sources
+                    for s in node.sources:
+                        if s not in experimentally_parsed['sources']:
+                            result += ["05_missed_source_value"]
+                            break
+
+                    # look for false positive refs
+                    for r in experimentally_parsed['refs']:
+                        if r not in node.refs:
+                            result += ["06_false_positive_ref_value"]
+                            break
+
+                    # look for missed refs
+                    for r in node.refs:
+                        if r not in experimentally_parsed['refs']:
+                            result += ["07_missed_ref_value"]
+                            break
+
+                    # if there are no errors, return a success value
+                    if not result:
+                        result = ["00_exact_match"]
+
+                # fire a tracking event. this fires one event for every sample
+                # so that we have data on a per file basis. Not only can we expect
+                # no false positives or misses, we can expect the number model
+                # files parseable by the experimental parser to match our internal
+                # testing.
+                tracking.track_experimental_parser_sample({
+                    "project_id": self.root_project.hashed_name(),
+                    "file_id": utils.get_hash(node),
+                    "status": result
+                })
+
+        # if the --use-experimental-parser flag was set, and the experimental parser succeeded
+        elif not isinstance(experimentally_parsed, Exception):
+            # since it doesn't need python jinja, fit the refs, sources, and configs
+            # into the node. Down the line the rest of the node will be updated with
+            # this information. (e.g. depends_on etc.)
+            config._config_calls = config_calls
+
+            # this uses the updated config to set all the right things in the node.
+            # if there are hooks present, it WILL render jinja. Will need to change
+            # when the experimental parser supports hooks
+            self.update_parsed_node(node, config)
+
+            # update the unrendered config with values from the file.
+            # values from yaml files are in there already
+            node.unrendered_config.update(dict(experimentally_parsed['configs']))
+
+            # set refs, sources, and configs on the node object
+            node.refs += experimentally_parsed['refs']
+            node.sources += experimentally_parsed['sources']
+            for configv in experimentally_parsed['configs']:
+                node.config[configv[0]] = configv[1]
+
+            self.manifest._parsing_info.static_analysis_parsed_path_count += 1
+
+        # the experimental parser tried and failed on this model.
+        # fall back to python jinja rendering.
+        else:
+            super().render_update(node, config)

core/dbt/parser/partial.py

@@ -0,0 +1,695 @@
+from typing import MutableMapping, Dict, List
+from dbt.contracts.graph.manifest import Manifest
+from dbt.contracts.files import (
+    AnySourceFile, ParseFileType, parse_file_type_to_parser,
+)
+from dbt.logger import GLOBAL_LOGGER as logger
+from dbt.node_types import NodeType
+
+
+mssat_files = (
+    ParseFileType.Model,
+    ParseFileType.Seed,
+    ParseFileType.Snapshot,
+    ParseFileType.Analysis,
+    ParseFileType.Test,
+)
+
+
+key_to_prefix = {
+    'models': 'model',
+    'seeds': 'seed',
+    'snapshots': 'snapshot',
+    'analyses': 'analysis',
+}
+
+
+parse_file_type_to_key = {
+    ParseFileType.Model: 'models',
+    ParseFileType.Seed: 'seeds',
+    ParseFileType.Snapshot: 'snapshots',
+    ParseFileType.Analysis: 'analyses',
+}
+
+
+# Partial parsing. Create a diff of files from saved manifest and current
+# files and produce a project_parser_file dictionary to drive parsing of
+# only the necessary changes.
+# Will produce a 'skip_parsing' method, and a project_parser_file dictionary
+class PartialParsing:
+    def __init__(self, saved_manifest: Manifest, new_files: MutableMapping[str, AnySourceFile]):
+        self.saved_manifest = saved_manifest
+        self.new_files = new_files
+        self.project_parser_files: Dict = {}
+        self.saved_files = self.saved_manifest.files
+        self.project_parser_files = {}
+        self.deleted_manifest = Manifest()
+        self.macro_child_map: Dict[str, List[str]] = {}
+        self.build_file_diff()
+
+    def skip_parsing(self):
+        return (
+            not self.file_diff['deleted'] and
+            not self.file_diff['added'] and
+            not self.file_diff['changed'] and
+            not self.file_diff['changed_schema_files'] and
+            not self.file_diff['deleted_schema_files']
+        )
+
+    # Compare the previously saved manifest files and the just-loaded manifest
+    # files to see if anything changed
+    def build_file_diff(self):
+        saved_file_ids = set(self.saved_files.keys())
+        new_file_ids = set(self.new_files.keys())
+        deleted_all_files = saved_file_ids.difference(new_file_ids)
+        added = new_file_ids.difference(saved_file_ids)
+        common = saved_file_ids.intersection(new_file_ids)
+        changed_or_deleted_macro_file = False
+
+        # separate out deleted schema files
+        deleted_schema_files = []
+        deleted = []
+        for file_id in deleted_all_files:
+            if self.saved_files[file_id].parse_file_type == ParseFileType.Schema:
+                deleted_schema_files.append(file_id)
+            else:
+                if self.saved_files[file_id].parse_file_type == ParseFileType.Macro:
+                    changed_or_deleted_macro_file = True
+                deleted.append(file_id)
+
+        changed = []
+        changed_schema_files = []
+        unchanged = []
+        for file_id in common:
+            if self.saved_files[file_id].checksum == self.new_files[file_id].checksum:
+                unchanged.append(file_id)
+            else:
+                # separate out changed schema files
+                if self.saved_files[file_id].parse_file_type == ParseFileType.Schema:
+                    sf = self.saved_files[file_id]
+                    if type(sf).__name__ != 'SchemaSourceFile':
+                        raise Exception(f"Serialization failure for {file_id}")
+                    changed_schema_files.append(file_id)
+                else:
+                    if self.saved_files[file_id].parse_file_type == ParseFileType.Macro:
+                        changed_or_deleted_macro_file = True
+                    changed.append(file_id)
+        file_diff = {
+            "deleted": deleted,
+            "deleted_schema_files": deleted_schema_files,
+            "added": added,
+            "changed": changed,
+            "changed_schema_files": changed_schema_files,
+            "unchanged": unchanged,
+        }
+        if changed_or_deleted_macro_file:
+            self.macro_child_map = self.saved_manifest.build_macro_child_map()
+        logger.info(f"Partial parsing enabled: "
+                    f"{len(deleted) + len(deleted_schema_files)} files deleted, "
+                    f"{len(added)} files added, "
+                    f"{len(changed) + len(changed_schema_files)} files changed.")
+        self.file_diff = file_diff
+
+    # generate the list of files that need parsing
+    # uses self.manifest.files generated by 'read_files'
+    def get_parsing_files(self):
+        if self.skip_parsing():
+            return {}
+        # Need to add new files first, because changes in schema files
+        # might refer to them
+        for file_id in self.file_diff['added']:
+            self.add_to_saved(file_id)
+        # Need to process schema files next, because the dictionaries
+        # need to be in place for handling SQL file changes
+        for file_id in self.file_diff['changed_schema_files']:
+            self.change_schema_file(file_id)
+        for file_id in self.file_diff['deleted_schema_files']:
+            self.delete_schema_file(file_id)
+        for file_id in self.file_diff['deleted']:
+            self.delete_from_saved(file_id)
+        for file_id in self.file_diff['changed']:
+            self.update_in_saved(file_id)
+        return self.project_parser_files
+
+    # Add the file to the project parser dictionaries to schedule parsing
+    def add_to_pp_files(self, source_file):
+        file_id = source_file.file_id
+        parser_name = parse_file_type_to_parser[source_file.parse_file_type]
+        project_name = source_file.project_name
+        if not parser_name or not project_name:
+            raise Exception(f"Did not find parse_file_type or project_name "
+                            f"in SourceFile for {source_file.file_id}")
+        if project_name not in self.project_parser_files:
+            self.project_parser_files[project_name] = {}
+        if parser_name not in self.project_parser_files[project_name]:
+            self.project_parser_files[project_name][parser_name] = []
+        if (file_id not in self.project_parser_files[project_name][parser_name] and
+                file_id not in self.file_diff['deleted']):
+            self.project_parser_files[project_name][parser_name].append(file_id)
+
+    # Add new files, including schema files
+    def add_to_saved(self, file_id):
+        # add file object to saved manifest.files
+        source_file = self.new_files[file_id]
+        if source_file.parse_file_type == ParseFileType.Schema:
+            self.handle_added_schema_file(source_file)
+        self.saved_files[file_id] = source_file
+        # update pp_files to parse
+        self.add_to_pp_files(source_file)
+        logger.debug(f"Partial parsing: added file: {file_id}")
+
+    def handle_added_schema_file(self, source_file):
+        source_file.pp_dict = source_file.dict_from_yaml.copy()
+        if 'sources' in source_file.pp_dict:
+            for source in source_file.pp_dict['sources']:
+                # We need to remove the original source, so it can
+                # be properly patched
+                if 'overrides' in source:
+                    self.remove_source_override_target(source)
+
+    # Deletes for all non-schema files
+    def delete_from_saved(self, file_id):
+        # Look at all things touched by file, remove those
+        # nodes, and update pp_files to parse unless the
+        # file creating those nodes has also been deleted
+        saved_source_file = self.saved_files[file_id]
+
+        # SQL file: models, seeds, snapshots, analyses, tests: SQL files, except
+        # macros/tests
+        if saved_source_file.parse_file_type in mssat_files:
+            self.remove_mssat_file(saved_source_file)
+            self.deleted_manifest.files[file_id] = self.saved_manifest.files.pop(file_id)
+
+        # macros
+        if saved_source_file.parse_file_type == ParseFileType.Macro:
+            self.delete_macro_file(saved_source_file, follow_references=True)
+
+        # docs
+        if saved_source_file.parse_file_type == ParseFileType.Documentation:
+            self.delete_doc_node(saved_source_file)
+
+        logger.debug(f"Partial parsing: deleted file: {file_id}")
+
+    # Updates for non-schema files
+    def update_in_saved(self, file_id):
+        new_source_file = self.new_files[file_id]
+        old_source_file = self.saved_files[file_id]
+
+        if new_source_file.parse_file_type in mssat_files:
+            self.update_mssat_in_saved(new_source_file, old_source_file)
+        elif new_source_file.parse_file_type == ParseFileType.Macro:
+            self.update_macro_in_saved(new_source_file, old_source_file)
+        elif new_source_file.parse_file_type == ParseFileType.Documentation:
+            self.update_doc_in_saved(new_source_file, old_source_file)
+        else:
+            raise Exception(f"Invalid parse_file_type in source_file {file_id}")
+        logger.debug(f"Partial parsing: updated file: {file_id}")
+
+    # Models, seeds, snapshots: patches and tests
+    # analyses: patches, no tests
+    # tests: not touched by schema files (no patches, no tests)
+    # Updated schema files should have been processed already.
+    def update_mssat_in_saved(self, new_source_file, old_source_file):
+
+        # These files only have one node.
+        unique_id = old_source_file.nodes[0]
+
+        # replace source_file in saved and add to parsing list
+        file_id = new_source_file.file_id
+        self.deleted_manifest.files[file_id] = old_source_file
+        self.saved_files[file_id] = new_source_file
+        self.add_to_pp_files(new_source_file)
+        self.remove_node_in_saved(new_source_file, unique_id)
+
+    def remove_node_in_saved(self, source_file, unique_id):
+        # Has already been deleted by another action
+        if unique_id not in self.saved_manifest.nodes:
+            return
+
+        # delete node in saved
+        node = self.saved_manifest.nodes.pop(unique_id)
+        self.deleted_manifest.nodes[unique_id] = node
+
+        # look at patch_path in model node to see if we need
+        # to reapply a patch from a schema_file.
+        if node.patch_path:
+            file_id = node.patch_path
+            # it might be changed...  then what?
+            if file_id not in self.file_diff['deleted']:
+                # schema_files should already be updated
+                schema_file = self.saved_files[file_id]
+                dict_key = parse_file_type_to_key[source_file.parse_file_type]
+                # look for a matching list dictionary
+                for elem in schema_file.dict_from_yaml[dict_key]:
+                    if elem['name'] == node.name:
+                        elem_patch = elem
+                        break
+                if elem_patch:
+                    self.delete_schema_mssa_links(schema_file, dict_key, elem_patch)
+                    self.merge_patch(schema_file, dict_key, elem_patch)
+                    if unique_id in schema_file.node_patches:
+                        schema_file.node_patches.remove(unique_id)
+
+    def update_macro_in_saved(self, new_source_file, old_source_file):
+        self.handle_macro_file_links(old_source_file, follow_references=True)
+        file_id = new_source_file.file_id
+        self.saved_files[file_id] = new_source_file
+        self.add_to_pp_files(new_source_file)
+
+    def update_doc_in_saved(self, new_source_file, old_source_file):
+        self.delete_doc_node(old_source_file)
+        self.saved_files[new_source_file.file_id] = new_source_file
+        self.add_to_pp_files(new_source_file)
+
+    def remove_mssat_file(self, source_file):
+        # nodes [unique_ids] -- SQL files
+        # There should always be a node for a SQL file
+        if not source_file.nodes:
+            raise Exception(f"No nodes found for source file {source_file.file_id}")
+        # There is generally only 1 node for SQL files, except for macros
+        for unique_id in source_file.nodes:
+            self.remove_node_in_saved(source_file, unique_id)
+            self.schedule_referencing_nodes_for_parsing(unique_id)
+
+    # We need to re-parse nodes that reference another removed node
+    def schedule_referencing_nodes_for_parsing(self, unique_id):
+        # Look at "children", i.e. nodes that reference this node
+        self.schedule_nodes_for_parsing(self.saved_manifest.child_map[unique_id])
+
+    def schedule_nodes_for_parsing(self, unique_ids):
+        for unique_id in unique_ids:
+            if unique_id in self.saved_manifest.nodes:
+                node = self.saved_manifest.nodes[unique_id]
+                if node.resource_type == NodeType.Test:
+                    # test nodes are handled separately. Must be removed from schema file
+                    continue
+                file_id = node.file_id
+                if file_id in self.saved_files and file_id not in self.file_diff['deleted']:
+                    source_file = self.saved_files[file_id]
+                    self.remove_mssat_file(source_file)
+                    # content of non-schema files is only in new files
+                    self.saved_files[file_id] = self.new_files[file_id]
+                    self.add_to_pp_files(self.saved_files[file_id])
+            elif unique_id in self.saved_manifest.sources:
+                source = self.saved_manifest.sources[unique_id]
+                file_id = source.file_id
+                if file_id in self.saved_files and file_id not in self.file_diff['deleted']:
+                    schema_file = self.saved_files[file_id]
+                    sources = []
+                    if 'sources' in schema_file.dict_from_yaml:
+                        sources = schema_file.dict_from_yaml['sources']
+                    source_element = self.get_schema_element(sources, source.source_name)
+                    if source_element:
+                        self.delete_schema_source(schema_file, source_element)
+                        self.remove_tests(schema_file, 'sources', source_element['name'])
+                        self.merge_patch(schema_file, 'sources', source_element)
+            elif unique_id in self.saved_manifest.exposures:
+                exposure = self.saved_manifest.exposures[unique_id]
+                file_id = exposure.file_id
+                if file_id in self.saved_files and file_id not in self.file_diff['deleted']:
+                    schema_file = self.saved_files[file_id]
+                    exposures = []
+                    if 'exposures' in schema_file.dict_from_yaml:
+                        exposures = schema_file.dict_from_yaml['exposures']
+                    exposure_element = self.get_schema_element(exposures, exposure.name)
+                    if exposure_element:
+                        self.delete_schema_exposure(schema_file, exposure_element)
+                        self.merge_patch(schema_file, 'exposures', exposure_element)
+            elif unique_id in self.saved_manifest.macros:
+                macro = self.saved_manifest.macros[unique_id]
+                file_id = macro.file_id
+                if file_id in self.saved_files and file_id not in self.file_diff['deleted']:
+                    source_file = self.saved_files[file_id]
+                    self.delete_macro_file(source_file)
+                    self.saved_files[file_id] = self.new_files[file_id]
+                    self.add_to_pp_files(self.saved_files[file_id])
+
+    def delete_macro_file(self, source_file, follow_references=False):
+        self.handle_macro_file_links(source_file, follow_references)
+        file_id = source_file.file_id
+        self.deleted_manifest.files[file_id] = self.saved_files.pop(file_id)
+
+    def recursively_gather_macro_references(self, macro_unique_id, referencing_nodes):
+        for unique_id in self.macro_child_map[macro_unique_id]:
+            if unique_id in referencing_nodes:
+                continue
+            referencing_nodes.append(unique_id)
+            if unique_id.startswith('macro.'):
+                self.recursively_gather_macro_references(unique_id, referencing_nodes)
+
+    def handle_macro_file_links(self, source_file, follow_references=False):
+        # remove the macros in the 'macros' dictionary
+        macros = source_file.macros.copy()
+        for unique_id in macros:
+            if unique_id not in self.saved_manifest.macros:
+                # This happens when a macro has already been removed
+                source_file.macros.remove(unique_id)
+                continue
+
+            base_macro = self.saved_manifest.macros.pop(unique_id)
+            self.deleted_manifest.macros[unique_id] = base_macro
+
+            # Recursively check children of this macro
+            # The macro_child_map might not exist if a macro is removed by
+            # schedule_nodes_for parsing. We only want to follow
+            # references if the macro file itself has been updated or
+            # deleted, not if we're just updating referenced nodes.
+            if self.macro_child_map and follow_references:
+                referencing_nodes = []
+                self.recursively_gather_macro_references(unique_id, referencing_nodes)
+                self.schedule_macro_nodes_for_parsing(referencing_nodes)
+
+            if base_macro.patch_path:
+                file_id = base_macro.patch_path
+                if file_id in self.saved_files:
+                    schema_file = self.saved_files[file_id]
+                    macro_patches = []
+                    if 'macros' in schema_file.dict_from_yaml:
+                        macro_patches = schema_file.dict_from_yaml['macros']
+                    macro_patch = self.get_schema_element(macro_patches, base_macro.name)
+                    self.delete_schema_macro_patch(schema_file, macro_patch)
+                    self.merge_patch(schema_file, 'macros', macro_patch)
+            source_file.macros.remove(unique_id)
+
+    # similar to schedule_nodes_for_parsing but doesn't do sources and exposures
+    # and handles schema tests
+    def schedule_macro_nodes_for_parsing(self, unique_ids):
+        for unique_id in unique_ids:
+            if unique_id in self.saved_manifest.nodes:
+                node = self.saved_manifest.nodes[unique_id]
+                if node.resource_type == NodeType.Test:
+                    schema_file_id = node.file_id
+                    schema_file = self.saved_manifest.files[schema_file_id]
+                    (key, name) = schema_file.get_key_and_name_for_test(node.unique_id)
+                    if key and name:
+                        patch_list = []
+                        if key in schema_file.dict_from_yaml:
+                            patch_list = schema_file.dict_from_yaml[key]
+                        node_patch = self.get_schema_element(patch_list, name)
+                        if node_patch:
+                            self.delete_schema_mssa_links(schema_file, key, node_patch)
+                            self.merge_patch(schema_file, key, node_patch)
+                            if unique_id in schema_file.node_patches:
+                                schema_file.node_patches.remove(unique_id)
+                else:
+                    file_id = node.file_id
+                    if file_id in self.saved_files and file_id not in self.file_diff['deleted']:
+                        source_file = self.saved_files[file_id]
+                        self.remove_mssat_file(source_file)
+                        # content of non-schema files is only in new files
+                        self.saved_files[file_id] = self.new_files[file_id]
+                        self.add_to_pp_files(self.saved_files[file_id])
+            elif unique_id in self.saved_manifest.macros:
+                macro = self.saved_manifest.macros[unique_id]
+                file_id = macro.file_id
+                if file_id in self.saved_files and file_id not in self.file_diff['deleted']:
+                    source_file = self.saved_files[file_id]
+                    self.delete_macro_file(source_file)
+                    self.saved_files[file_id] = self.new_files[file_id]
+                    self.add_to_pp_files(self.saved_files[file_id])
+
+    def delete_doc_node(self, source_file):
+        # remove the nodes in the 'docs' dictionary
+        docs = source_file.docs.copy()
+        for unique_id in docs:
+            self.deleted_manifest.docs[unique_id] = self.saved_manifest.docs.pop(unique_id)
+            source_file.docs.remove(unique_id)
+        # The unique_id of objects that contain a doc call are stored in the
+        # doc source_file.nodes
+        self.schedule_nodes_for_parsing(source_file.nodes)
+        source_file.nodes = []
+
+    # Schema files -----------------------
+    # Changed schema files
+    def change_schema_file(self, file_id):
+        saved_schema_file = self.saved_files[file_id]
+        new_schema_file = self.new_files[file_id]
+        saved_yaml_dict = saved_schema_file.dict_from_yaml
+        new_yaml_dict = new_schema_file.dict_from_yaml
+        saved_schema_file.pp_dict = {"version": saved_yaml_dict['version']}
+        self.handle_schema_file_changes(saved_schema_file, saved_yaml_dict, new_yaml_dict)
+
+        # copy from new schema_file to saved_schema_file to preserve references
+        # that weren't removed
+        saved_schema_file.contents = new_schema_file.contents
+        saved_schema_file.checksum = new_schema_file.checksum
+        saved_schema_file.dfy = new_schema_file.dfy
+        # schedule parsing
+        self.add_to_pp_files(saved_schema_file)
+        # schema_file pp_dict should have been generated already
+        logger.debug(f"Partial parsing: update schema file: {file_id}")
+
+    # Delete schema files -- a variation on change_schema_file
+    def delete_schema_file(self, file_id):
+        saved_schema_file = self.saved_files[file_id]
+        saved_yaml_dict = saved_schema_file.dict_from_yaml
+        new_yaml_dict = {}
+        self.handle_schema_file_changes(saved_schema_file, saved_yaml_dict, new_yaml_dict)
+        self.deleted_manifest.files[file_id] = self.saved_manifest.files.pop(file_id)
+
+    # For each key in a schema file dictionary, process the changed, deleted, and added
+    # elemnts for the key lists
+    def handle_schema_file_changes(self, schema_file, saved_yaml_dict, new_yaml_dict):
+        # loop through comparing previous dict_from_yaml with current dict_from_yaml
+        # Need to do the deleted/added/changed thing, just like the files lists
+
+        # models, seeds, snapshots, analyses
+        for dict_key in ['models', 'seeds', 'snapshots', 'analyses']:
+            key_diff = self.get_diff_for(dict_key, saved_yaml_dict, new_yaml_dict)
+            if key_diff['changed']:
+                for elem in key_diff['changed']:
+                    self.delete_schema_mssa_links(schema_file, dict_key, elem)
+                    self.merge_patch(schema_file, dict_key, elem)
+            if key_diff['deleted']:
+                for elem in key_diff['deleted']:
+                    self.delete_schema_mssa_links(schema_file, dict_key, elem)
+            if key_diff['added']:
+                for elem in key_diff['added']:
+                    self.merge_patch(schema_file, dict_key, elem)
+
+        # sources
+        source_diff = self.get_diff_for('sources', saved_yaml_dict, new_yaml_dict)
+        if source_diff['changed']:
+            for source in source_diff['changed']:
+                if 'overrides' in source:  # This is a source patch; need to re-parse orig source
+                    self.remove_source_override_target(source)
+                self.delete_schema_source(schema_file, source)
+                self.remove_tests(schema_file, 'sources', source['name'])
+                self.merge_patch(schema_file, 'sources', source)
+        if source_diff['deleted']:
+            for source in source_diff['deleted']:
+                if 'overrides' in source:  # This is a source patch; need to re-parse orig source
+                    self.remove_source_override_target(source)
+                self.delete_schema_source(schema_file, source)
+                self.remove_tests(schema_file, 'sources', source['name'])
+        if source_diff['added']:
+            for source in source_diff['added']:
+                if 'overrides' in source:  # This is a source patch; need to re-parse orig source
+                    self.remove_source_override_target(source)
+                self.merge_patch(schema_file, 'sources', source)
+
+        # macros
+        macro_diff = self.get_diff_for('macros', saved_yaml_dict, new_yaml_dict)
+        if macro_diff['changed']:
+            for macro in macro_diff['changed']:
+                self.delete_schema_macro_patch(schema_file, macro)
+                self.merge_patch(schema_file, 'macros', macro)
+        if macro_diff['deleted']:
+            for macro in macro_diff['deleted']:
+                self.delete_schema_macro_patch(schema_file, macro)
+        if macro_diff['added']:
+            for macro in macro_diff['added']:
+                self.merge_patch(schema_file, 'macros', macro)
+
+        # exposures
+        exposure_diff = self.get_diff_for('exposures', saved_yaml_dict, new_yaml_dict)
+        if exposure_diff['changed']:
+            for exposure in exposure_diff['changed']:
+                self.delete_schema_exposure(schema_file, exposure)
+                self.merge_patch(schema_file, 'exposures', exposure)
+        if exposure_diff['deleted']:
+            for exposure in exposure_diff['deleted']:
+                self.delete_schema_exposure(schema_file, exposure)
+        if exposure_diff['added']:
+            for exposure in exposure_diff['added']:
+                self.merge_patch(schema_file, 'exposures', exposure)
+
+    # Take a "section" of the schema file yaml dictionary from saved and new schema files
+    # and determine which parts have changed
+    def get_diff_for(self, key, saved_yaml_dict, new_yaml_dict):
+        if key in saved_yaml_dict or key in new_yaml_dict:
+            saved_elements = saved_yaml_dict[key] if key in saved_yaml_dict else []
+            new_elements = new_yaml_dict[key] if key in new_yaml_dict else []
+        else:
+            return {'deleted': [], 'added': [], 'changed': []}
+        # for each set of keys, need to create a dictionary of names pointing to entry
+        saved_elements_by_name = {}
+        new_elements_by_name = {}
+        # sources have two part names?
+        for element in saved_elements:
+            saved_elements_by_name[element['name']] = element
+        for element in new_elements:
+            new_elements_by_name[element['name']] = element
+
+        # now determine which elements, by name, are added, deleted or changed
+        saved_element_names = set(saved_elements_by_name.keys())
+        new_element_names = set(new_elements_by_name.keys())
+        deleted = saved_element_names.difference(new_element_names)
+        added = new_element_names.difference(saved_element_names)
+        common = saved_element_names.intersection(new_element_names)
+        changed = []
+        for element_name in common:
+            if saved_elements_by_name[element_name] != new_elements_by_name[element_name]:
+                changed.append(element_name)
+
+        # make lists of yaml elements to return as diffs
+        deleted_elements = [saved_elements_by_name[name].copy() for name in deleted]
+        added_elements = [new_elements_by_name[name].copy() for name in added]
+        changed_elements = [new_elements_by_name[name].copy() for name in changed]
+
+        diff = {
+            "deleted": deleted_elements,
+            "added": added_elements,
+            "changed": changed_elements,
+        }
+        return diff
+
+    # Merge a patch file into the pp_dict in a schema file
+    def merge_patch(self, schema_file, key, patch):
+        if not schema_file.pp_dict:
+            schema_file.pp_dict = {"version": schema_file.dict_from_yaml['version']}
+        pp_dict = schema_file.pp_dict
+        if key not in pp_dict:
+            pp_dict[key] = [patch]
+        else:
+            # check that this patch hasn't already been saved
+            found = False
+            for elem in pp_dict[key]:
+                if elem['name'] == patch['name']:
+                    found = True
+            if not found:
+                pp_dict[key].append(patch)
+        self.add_to_pp_files(schema_file)
+
+    # For model, seed, snapshot, analysis schema dictionary keys,
+    # delete the patches and tests from the patch
+    def delete_schema_mssa_links(self, schema_file, dict_key, elem):
+        # find elem node unique_id in node_patches
+        prefix = key_to_prefix[dict_key]
+        elem_unique_id = ''
+        for unique_id in schema_file.node_patches:
+            if not unique_id.startswith(prefix):
+                continue
+            parts = unique_id.split('.')
+            elem_name = parts[-1]
+            if elem_name == elem['name']:
+                elem_unique_id = unique_id
+                break
+
+        # remove elem node and remove unique_id from node_patches
+        if elem_unique_id:
+            # might have been already removed
+            if elem_unique_id in self.saved_manifest.nodes:
+                node = self.saved_manifest.nodes.pop(elem_unique_id)
+                self.deleted_manifest.nodes[elem_unique_id] = node
+                # need to add the node source_file to pp_files
+                file_id = node.file_id
+                # need to copy new file to saved files in order to get content
+                if file_id in self.new_files:
+                    self.saved_files[file_id] = self.new_files[file_id]
+                if self.saved_files[file_id]:
+                    source_file = self.saved_files[file_id]
+                    self.add_to_pp_files(source_file)
+            # remove from patches
+            schema_file.node_patches.remove(elem_unique_id)
+
+        # for models, seeds, snapshots (not analyses)
+        if dict_key in ['models', 'seeds', 'snapshots']:
+            # find related tests and remove them
+            self.remove_tests(schema_file, dict_key, elem['name'])
+
+    def remove_tests(self, schema_file, dict_key, name):
+        tests = schema_file.get_tests(dict_key, name)
+        for test_unique_id in tests:
+            node = self.saved_manifest.nodes.pop(test_unique_id)
+            self.deleted_manifest.nodes[test_unique_id] = node
+        schema_file.remove_tests(dict_key, name)
+
+    def delete_schema_source(self, schema_file, source_dict):
+        # both patches, tests, and source nodes
+        source_name = source_dict['name']
+        # There may be multiple sources for each source dict, since
+        # there will be a separate source node for each table.
+        # ParsedSourceDefinition name = table name, dict name is source_name
+        sources = schema_file.sources.copy()
+        for unique_id in sources:
+            if unique_id in self.saved_manifest.sources:
+                source = self.saved_manifest.sources[unique_id]
+                if source.source_name == source_name:
+                    source = self.saved_manifest.sources.pop(unique_id)
+                    self.deleted_manifest.sources[unique_id] = source
+                    schema_file.sources.remove(unique_id)
+                    self.schedule_referencing_nodes_for_parsing(unique_id)
+                    logger.debug(f"Partial parsing: deleted source {unique_id}")
+
+    def delete_schema_macro_patch(self, schema_file, macro):
+        # This is just macro patches that need to be reapplied
+        for unique_id in schema_file.macro_patches:
+            parts = unique_id.split('.')
+            macro_name = parts[-1]
+            if macro_name == macro['name']:
+                macro_unique_id = unique_id
+                break
+        if macro_unique_id and macro_unique_id in self.saved_manifest.macros:
+            macro = self.saved_manifest.macros.pop(macro_unique_id)
+            self.deleted_manifest.macros[macro_unique_id] = macro
+            macro_file_id = macro.file_id
+            self.add_to_pp_files(self.saved_files[macro_file_id])
+        if macro_unique_id in schema_file.macro_patches:
+            schema_file.macro_patches.remove(macro_unique_id)
+
+    # exposures are created only from schema files, so just delete
+    # the exposure.
+    def delete_schema_exposure(self, schema_file, exposure_dict):
+        exposure_name = exposure_dict['name']
+        exposures = schema_file.exposures.copy()
+        for unique_id in exposures:
+            exposure = self.saved_manifest.exposures[unique_id]
+            if unique_id in self.saved_manifest.exposures:
+                if exposure.name == exposure_name:
+                    self.deleted_manifest.exposures[unique_id] = \
+                        self.saved_manifest.exposures.pop(unique_id)
+                    schema_file.exposures.remove(unique_id)
+                    logger.debug(f"Partial parsing: deleted exposure {unique_id}")
+
+    def get_schema_element(self, elem_list, elem_name):
+        for element in elem_list:
+            if 'name' in element and element['name'] == elem_name:
+                return element
+        return None
+
+    def get_schema_file_for_source(self, package_name, source_name):
+        schema_file = None
+        for source in self.saved_manifest.sources.values():
+            if source.package_name == package_name and source.source_name == source_name:
+                file_id = source.file_id
+                if file_id in self.saved_files:
+                    schema_file = self.saved_files[file_id]
+                break
+        return schema_file
+
+    def get_source_override_file_and_dict(self, source):
+        package = source['overrides']
+        source_name = source['name']
+        orig_source_schema_file = self.get_schema_file_for_source(package, source_name)
+        orig_sources = orig_source_schema_file.dict_from_yaml['sources']
+        orig_source = self.get_schema_element(orig_sources, source_name)
+        return (orig_source_schema_file, orig_source)
+
+    def remove_source_override_target(self, source_dict):
+        (orig_file, orig_source) = self.get_source_override_file_and_dict(source_dict)
+        if orig_source:
+            self.delete_schema_source(orig_file, orig_source)
+            self.remove_tests(orig_file, 'sources', orig_source['name'])
+            self.merge_patch(orig_file, 'sources', orig_source)
+            self.add_to_pp_files(orig_file)

core/dbt/parser/read_files.py

@@ -0,0 +1,143 @@
+from dbt.clients.system import load_file_contents
+from dbt.contracts.files import (
+    FilePath, ParseFileType, SourceFile, FileHash, AnySourceFile, SchemaSourceFile
+)
+
+from dbt.parser.schemas import yaml_from_file, schema_file_keys, check_format_version
+from dbt.exceptions import CompilationException
+from dbt.parser.search import FilesystemSearcher
+
+
+# This loads the files contents and creates the SourceFile object
+def load_source_file(
+        path: FilePath, parse_file_type: ParseFileType,
+        project_name: str) -> AnySourceFile:
+    file_contents = load_file_contents(path.absolute_path, strip=False)
+    checksum = FileHash.from_contents(file_contents)
+    sf_cls = SchemaSourceFile if parse_file_type == ParseFileType.Schema else SourceFile
+    source_file = sf_cls(path=path, checksum=checksum,
+                         parse_file_type=parse_file_type, project_name=project_name)
+    source_file.contents = file_contents.strip()
+    if parse_file_type == ParseFileType.Schema and source_file.contents:
+        dfy = yaml_from_file(source_file)
+        validate_yaml(source_file.path.original_file_path, dfy)
+        source_file.dfy = dfy
+    return source_file
+
+
+# Do some minimal validation of the yaml in a schema file.
+# Check version, that key values are lists and that each element in
+# the lists has a 'name' key
+def validate_yaml(file_path, dct):
+    check_format_version(file_path, dct)
+    for key in schema_file_keys:
+        if key in dct:
+            if not isinstance(dct[key], list):
+                msg = (f"The schema file at {file_path} is "
+                       f"invalid because the value of '{key}' is not a list")
+                raise CompilationException(msg)
+            for element in dct[key]:
+                if not isinstance(element, dict):
+                    msg = (f"The schema file at {file_path} is "
+                           f"invalid because a list element for '{key}' is not a dictionary")
+                    raise CompilationException(msg)
+                if 'name' not in element:
+                    msg = (f"The schema file at {file_path} is "
+                           f"invalid because a list element for '{key}' does not have a "
+                           "name attribute.")
+                    raise CompilationException(msg)
+
+
+# Special processing for big seed files
+def load_seed_source_file(match: FilePath, project_name) -> SourceFile:
+    if match.seed_too_large():
+        # We don't want to calculate a hash of this file. Use the path.
+        source_file = SourceFile.big_seed(match)
+    else:
+        file_contents = load_file_contents(match.absolute_path, strip=False)
+        checksum = FileHash.from_contents(file_contents)
+        source_file = SourceFile(path=match, checksum=checksum)
+        source_file.contents = ''
+    source_file.parse_file_type = ParseFileType.Seed
+    source_file.project_name = project_name
+    return source_file
+
+
+# Use the FilesystemSearcher to get a bunch of FilePaths, then turn
+# them into a bunch of FileSource objects
+def get_source_files(project, paths, extension, parse_file_type):
+    # file path list
+    fp_list = list(FilesystemSearcher(
+        project, paths, extension
+    ))
+    # file block list
+    fb_list = []
+    for fp in fp_list:
+        if parse_file_type == ParseFileType.Seed:
+            fb_list.append(load_seed_source_file(fp, project.project_name))
+        else:
+            fb_list.append(load_source_file(
+                fp, parse_file_type, project.project_name))
+    return fb_list
+
+
+def read_files_for_parser(project, files, dirs, extension, parse_ft):
+    parser_files = []
+    source_files = get_source_files(
+        project, dirs, extension, parse_ft
+    )
+    for sf in source_files:
+        files[sf.file_id] = sf
+        parser_files.append(sf.file_id)
+    return parser_files
+
+
+# This needs to read files for multiple projects, so the 'files'
+# dictionary needs to be passed in. What determines the order of
+# the various projects? Is the root project always last? Do the
+# non-root projects need to be done separately in order?
+def read_files(project, files, parser_files):
+
+    project_files = {}
+
+    project_files['MacroParser'] = read_files_for_parser(
+        project, files, project.macro_paths, '.sql', ParseFileType.Macro,
+    )
+
+    project_files['ModelParser'] = read_files_for_parser(
+        project, files, project.source_paths, '.sql', ParseFileType.Model,
+    )
+
+    project_files['SnapshotParser'] = read_files_for_parser(
+        project, files, project.snapshot_paths, '.sql', ParseFileType.Snapshot,
+    )
+
+    project_files['AnalysisParser'] = read_files_for_parser(
+        project, files, project.analysis_paths, '.sql', ParseFileType.Analysis,
+    )
+
+    project_files['DataTestParser'] = read_files_for_parser(
+        project, files, project.test_paths, '.sql', ParseFileType.Test,
+    )
+
+    project_files['SeedParser'] = read_files_for_parser(
+        project, files, project.data_paths, '.csv', ParseFileType.Seed,
+    )
+
+    project_files['DocumentationParser'] = read_files_for_parser(
+        project, files, project.docs_paths, '.md', ParseFileType.Documentation,
+    )
+
+    project_files['SchemaParser'] = read_files_for_parser(
+        project, files, project.all_source_paths, '.yml', ParseFileType.Schema,
+    )
+
+    # Also read .yaml files for schema files. Might be better to change
+    # 'read_files_for_parser' accept an array in the future.
+    yaml_files = read_files_for_parser(
+        project, files, project.all_source_paths, '.yaml', ParseFileType.Schema,
+    )
+    project_files['SchemaParser'].extend(yaml_files)
+
+    # Store the parser files for this particular project
+    parser_files[project.project_name] = project_files

core/dbt/parser/results.py

@@ -1,328 +0,0 @@
-from dataclasses import dataclass, field
-from typing import TypeVar, MutableMapping, Mapping, Union, List
-
-from dbt.dataclass_schema import dbtClassMixin
-
-from dbt.contracts.files import RemoteFile, FileHash, SourceFile
-from dbt.contracts.graph.compiled import CompileResultNode
-from dbt.contracts.graph.parsed import (
-    HasUniqueID,
-    ParsedAnalysisNode,
-    ParsedDataTestNode,
-    ParsedDocumentation,
-    ParsedHookNode,
-    ParsedMacro,
-    ParsedMacroPatch,
-    ParsedModelNode,
-    ParsedNodePatch,
-    ParsedExposure,
-    ParsedRPCNode,
-    ParsedSeedNode,
-    ParsedSchemaTestNode,
-    ParsedSnapshotNode,
-    UnpatchedSourceDefinition,
-)
-from dbt.contracts.graph.unparsed import SourcePatch
-from dbt.contracts.util import Writable, Replaceable, MacroKey, SourceKey
-from dbt.exceptions import (
-    raise_duplicate_resource_name, raise_duplicate_patch_name,
-    raise_duplicate_macro_patch_name, CompilationException, InternalException,
-    raise_compiler_error, raise_duplicate_source_patch_name
-)
-from dbt.node_types import NodeType
-from dbt.ui import line_wrap_message
-from dbt.version import __version__
-
-
-# Parsers can return anything as long as it's a unique ID
-ParsedValueType = TypeVar('ParsedValueType', bound=HasUniqueID)
-
-
-def _check_duplicates(
-    value: HasUniqueID, src: Mapping[str, HasUniqueID]
-):
-    if value.unique_id in src:
-        raise_duplicate_resource_name(value, src[value.unique_id])
-
-
-ManifestNodes = Union[
-    ParsedAnalysisNode,
-    ParsedDataTestNode,
-    ParsedHookNode,
-    ParsedModelNode,
-    ParsedRPCNode,
-    ParsedSchemaTestNode,
-    ParsedSeedNode,
-    ParsedSnapshotNode,
-]
-
-
-def dict_field():
-    return field(default_factory=dict)
-
-
-@dataclass
-class ParseResult(dbtClassMixin, Writable, Replaceable):
-    vars_hash: FileHash
-    profile_hash: FileHash
-    project_hashes: MutableMapping[str, FileHash]
-    nodes: MutableMapping[str, ManifestNodes] = dict_field()
-    sources: MutableMapping[str, UnpatchedSourceDefinition] = dict_field()
-    docs: MutableMapping[str, ParsedDocumentation] = dict_field()
-    macros: MutableMapping[str, ParsedMacro] = dict_field()
-    exposures: MutableMapping[str, ParsedExposure] = dict_field()
-    macro_patches: MutableMapping[MacroKey, ParsedMacroPatch] = dict_field()
-    patches: MutableMapping[str, ParsedNodePatch] = dict_field()
-    source_patches: MutableMapping[SourceKey, SourcePatch] = dict_field()
-    files: MutableMapping[str, SourceFile] = dict_field()
-    disabled: MutableMapping[str, List[CompileResultNode]] = dict_field()
-    dbt_version: str = __version__
-
-    def get_file(self, source_file: SourceFile) -> SourceFile:
-        key = source_file.search_key
-        if key is None:
-            return source_file
-        if key not in self.files:
-            self.files[key] = source_file
-        return self.files[key]
-
-    def add_source(
-        self, source_file: SourceFile, source: UnpatchedSourceDefinition
-    ):
-        # sources can't be overwritten!
-        _check_duplicates(source, self.sources)
-        self.sources[source.unique_id] = source
-        self.get_file(source_file).sources.append(source.unique_id)
-
-    def add_node_nofile(self, node: ManifestNodes):
-        # nodes can't be overwritten!
-        _check_duplicates(node, self.nodes)
-        self.nodes[node.unique_id] = node
-
-    def add_node(self, source_file: SourceFile, node: ManifestNodes):
-        self.add_node_nofile(node)
-        self.get_file(source_file).nodes.append(node.unique_id)
-
-    def add_exposure(self, source_file: SourceFile, exposure: ParsedExposure):
-        _check_duplicates(exposure, self.exposures)
-        self.exposures[exposure.unique_id] = exposure
-        self.get_file(source_file).exposures.append(exposure.unique_id)
-
-    def add_disabled_nofile(self, node: CompileResultNode):
-        if node.unique_id in self.disabled:
-            self.disabled[node.unique_id].append(node)
-        else:
-            self.disabled[node.unique_id] = [node]
-
-    def add_disabled(self, source_file: SourceFile, node: CompileResultNode):
-        self.add_disabled_nofile(node)
-        self.get_file(source_file).nodes.append(node.unique_id)
-
-    def add_macro(self, source_file: SourceFile, macro: ParsedMacro):
-        if macro.unique_id in self.macros:
-            # detect that the macro exists and emit an error
-            other_path = self.macros[macro.unique_id].original_file_path
-            # subtract 2 for the "Compilation Error" indent
-            # note that the line wrap eats newlines, so if you want newlines,
-            # this is the result :(
-            msg = line_wrap_message(
-                f'''\
-                dbt found two macros named "{macro.name}" in the project
-                "{macro.package_name}".
-
-
-                To fix this error, rename or remove one of the following
-                macros:
-
-                    - {macro.original_file_path}
-
-                    - {other_path}
-                ''',
-                subtract=2
-            )
-            raise_compiler_error(msg)
-
-        self.macros[macro.unique_id] = macro
-        self.get_file(source_file).macros.append(macro.unique_id)
-
-    def add_doc(self, source_file: SourceFile, doc: ParsedDocumentation):
-        _check_duplicates(doc, self.docs)
-        self.docs[doc.unique_id] = doc
-        self.get_file(source_file).docs.append(doc.unique_id)
-
-    def add_patch(
-        self, source_file: SourceFile, patch: ParsedNodePatch
-    ) -> None:
-        # patches can't be overwritten
-        if patch.name in self.patches:
-            raise_duplicate_patch_name(patch, self.patches[patch.name])
-        self.patches[patch.name] = patch
-        self.get_file(source_file).patches.append(patch.name)
-
-    def add_macro_patch(
-        self, source_file: SourceFile, patch: ParsedMacroPatch
-    ) -> None:
-        # macros are fully namespaced
-        key = (patch.package_name, patch.name)
-        if key in self.macro_patches:
-            raise_duplicate_macro_patch_name(patch, self.macro_patches[key])
-        self.macro_patches[key] = patch
-        self.get_file(source_file).macro_patches.append(key)
-
-    def add_source_patch(
-        self, source_file: SourceFile, patch: SourcePatch
-    ) -> None:
-        # source patches must be unique
-        key = (patch.overrides, patch.name)
-        if key in self.source_patches:
-            raise_duplicate_source_patch_name(patch, self.source_patches[key])
-        self.source_patches[key] = patch
-        self.get_file(source_file).source_patches.append(key)
-
-    def _get_disabled(
-        self,
-        unique_id: str,
-        match_file: SourceFile,
-    ) -> List[CompileResultNode]:
-        if unique_id not in self.disabled:
-            raise InternalException(
-                'called _get_disabled with id={}, but it does not exist'
-                .format(unique_id)
-            )
-        return [
-            n for n in self.disabled[unique_id]
-            if n.original_file_path == match_file.path.original_file_path
-        ]
-
-    def _process_node(
-        self,
-        node_id: str,
-        source_file: SourceFile,
-        old_file: SourceFile,
-        old_result: 'ParseResult',
-    ) -> None:
-        """Nodes are a special kind of complicated - there can be multiple
-        with the same name, as long as all but one are disabled.
-
-        Only handle nodes where the matching node has the same resource type
-        as the current parser.
-        """
-        source_path = source_file.path.original_file_path
-        found: bool = False
-        if node_id in old_result.nodes:
-            old_node = old_result.nodes[node_id]
-            if old_node.original_file_path == source_path:
-                self.add_node(source_file, old_node)
-                found = True
-
-        if node_id in old_result.disabled:
-            matches = old_result._get_disabled(node_id, source_file)
-            for match in matches:
-                self.add_disabled(source_file, match)
-                found = True
-
-        if not found:
-            raise CompilationException(
-                'Expected to find "{}" in cached "manifest.nodes" or '
-                '"manifest.disabled" based on cached file information: {}!'
-                .format(node_id, old_file)
-            )
-
-    def sanitized_update(
-        self,
-        source_file: SourceFile,
-        old_result: 'ParseResult',
-        resource_type: NodeType,
-    ) -> bool:
-        """Perform a santized update. If the file can't be updated, invalidate
-        it and return false.
-        """
-        if isinstance(source_file.path, RemoteFile):
-            return False
-
-        old_file = old_result.get_file(source_file)
-        for doc_id in old_file.docs:
-            doc = _expect_value(doc_id, old_result.docs, old_file, "docs")
-            self.add_doc(source_file, doc)
-
-        for macro_id in old_file.macros:
-            macro = _expect_value(
-                macro_id, old_result.macros, old_file, "macros"
-            )
-            self.add_macro(source_file, macro)
-
-        for source_id in old_file.sources:
-            source = _expect_value(
-                source_id, old_result.sources, old_file, "sources"
-            )
-            self.add_source(source_file, source)
-
-        # because we know this is how we _parsed_ the node, we can safely
-        # assume if it's disabled it was done by the project or file, and
-        # we can keep our old data
-        # the node ID could be in old_result.disabled AND in old_result.nodes.
-        # In that case, we have to make sure the path also matches.
-        for node_id in old_file.nodes:
-            # cheat: look at the first part of the node ID and compare it to
-            # the parser resource type. On a mismatch, bail out.
-            if resource_type != node_id.split('.')[0]:
-                continue
-            self._process_node(node_id, source_file, old_file, old_result)
-
-        for exposure_id in old_file.exposures:
-            exposure = _expect_value(
-                exposure_id, old_result.exposures, old_file, "exposures"
-            )
-            self.add_exposure(source_file, exposure)
-
-        patched = False
-        for name in old_file.patches:
-            patch = _expect_value(
-                name, old_result.patches, old_file, "patches"
-            )
-            self.add_patch(source_file, patch)
-            patched = True
-        if patched:
-            self.get_file(source_file).patches.sort()
-
-        macro_patched = False
-        for key in old_file.macro_patches:
-            macro_patch = _expect_value(
-                key, old_result.macro_patches, old_file, "macro_patches"
-            )
-            self.add_macro_patch(source_file, macro_patch)
-            macro_patched = True
-        if macro_patched:
-            self.get_file(source_file).macro_patches.sort()
-
-        return True
-
-    def has_file(self, source_file: SourceFile) -> bool:
-        key = source_file.search_key
-        if key is None:
-            return False
-        if key not in self.files:
-            return False
-        my_checksum = self.files[key].checksum
-        return my_checksum == source_file.checksum
-
-    @classmethod
-    def rpc(cls):
-        # ugh!
-        return cls(FileHash.empty(), FileHash.empty(), {})
-
-
-K_T = TypeVar('K_T')
-V_T = TypeVar('V_T')
-
-
-def _expect_value(
-    key: K_T, src: Mapping[K_T, V_T], old_file: SourceFile, name: str
-) -> V_T:
-    if key not in src:
-        raise CompilationException(
-            'Expected to find "{}" in cached "result.{}" based '
-            'on cached file information: {}!'
-            .format(key, name, old_file)
-        )
-    return src[key]

core/dbt/parser/rpc.py

@@ -22,9 +22,6 @@ class RPCBlock(FileBlock):
 
 
 class RPCCallParser(SimpleSQLParser[ParsedRPCNode]):
-    def get_paths(self):
-        return []
-
     def parse_from_dict(self, dct, validate=True) -> ParsedRPCNode:
         if validate:
             ParsedRPCNode.validate(dct)
@@ -45,7 +42,7 @@ class RPCCallParser(SimpleSQLParser[ParsedRPCNode]):
         return os.path.join('rpc', block.name)
 
     def parse_remote(self, sql: str, name: str) -> ParsedRPCNode:
-        source_file = SourceFile.remote(contents=sql)
+        source_file = SourceFile.remote(sql, self.project.project_name)
         contents = RPCBlock(rpc_name=name, file=source_file)
         return self.parse_node(contents)
 

core/dbt/parser/schema_test_builders.py

@@ -41,16 +41,21 @@ def get_nice_schema_test_name(
     clean_flat_args = [re.sub('[^0-9a-zA-Z_]+', '_', arg) for arg in flat_args]
     unique = "__".join(clean_flat_args)
 
-    cutoff = 32
-    if len(unique) <= cutoff:
-        label = unique
+    # for the file path + alias, the name must be <64 characters
+    # if the full name is too long, include the first 30 identifying chars plus
+    # a 32-character hash of the full contents
+
+    test_identifier = '{}_{}'.format(test_type, test_name)
+    full_name = '{}_{}'.format(test_identifier, unique)
+
+    if len(full_name) >= 64:
+        test_trunc_identifier = test_identifier[:30]
+        label = hashlib.md5(full_name.encode('utf-8')).hexdigest()
+        short_name = '{}_{}'.format(test_trunc_identifier, label)
     else:
-        label = hashlib.md5(unique.encode('utf-8')).hexdigest()
+        short_name = full_name
 
-    filename = '{}_{}_{}'.format(test_type, test_name, label)
-    name = '{}_{}_{}'.format(test_type, test_name, unique)
-
-    return filename, name
+    return short_name, full_name
 
 
 @dataclass
@@ -184,8 +189,11 @@ class TestBuilder(Generic[Testable]):
         r'((?P<test_namespace>([a-zA-Z_][0-9a-zA-Z_]*))\.)?'
         r'(?P<test_name>([a-zA-Z_][0-9a-zA-Z_]*))'
     )
-    # map magic keys to default values
-    MODIFIER_ARGS = {'severity': 'ERROR', 'tags': []}
+    # kwargs representing test configs
+    MODIFIER_ARGS = (
+        'severity', 'tags', 'enabled', 'where', 'limit', 'warn_if', 'error_if',
+        'fail_calc', 'store_failures'
+    )
 
     def __init__(
         self,
@@ -217,11 +225,12 @@ class TestBuilder(Generic[Testable]):
         self.name: str = groups['test_name']
         self.namespace: str = groups['test_namespace']
         self.modifiers: Dict[str, Any] = {}
-        for key, default in self.MODIFIER_ARGS.items():
-            value = self.args.pop(key, default)
+        for key in self.MODIFIER_ARGS:
+            value = self.args.pop(key, None)
             if isinstance(value, str):
-                value = get_rendered(value, render_ctx)
-            self.modifiers[key] = value
+                value = get_rendered(value, render_ctx, native=True)
+            if value is not None:
+                self.modifiers[key] = value
 
         if self.namespace is not None:
             self.package_name = self.namespace
@@ -230,6 +239,10 @@ class TestBuilder(Generic[Testable]):
         self.compiled_name: str = compiled_name
         self.fqn_name: str = fqn_name
 
+        # use hashed name as alias if too long
+        if compiled_name != fqn_name:
+            self.modifiers['alias'] = compiled_name
+
     def _bad_type(self) -> TypeError:
         return TypeError('invalid target type "{}"'.format(type(self.target)))
 
@@ -267,8 +280,45 @@ class TestBuilder(Generic[Testable]):
             test_args['column_name'] = name
         return test_name, test_args
 
-    def severity(self) -> str:
-        return self.modifiers.get('severity', 'ERROR').upper()
+    @property
+    def enabled(self) -> Optional[bool]:
+        return self.modifiers.get('enabled')
+
+    @property
+    def alias(self) -> Optional[str]:
+        return self.modifiers.get('alias')
+
+    @property
+    def severity(self) -> Optional[str]:
+        sev = self.modifiers.get('severity')
+        if sev:
+            return sev.upper()
+        else:
+            return None
+
+    @property
+    def store_failures(self) -> Optional[bool]:
+        return self.modifiers.get('store_failures')
+
+    @property
+    def where(self) -> Optional[str]:
+        return self.modifiers.get('where')
+
+    @property
+    def limit(self) -> Optional[int]:
+        return self.modifiers.get('limit')
+
+    @property
+    def warn_if(self) -> Optional[str]:
+        return self.modifiers.get('warn_if')
+
+    @property
+    def error_if(self) -> Optional[str]:
+        return self.modifiers.get('error_if')
+
+    @property
+    def fail_calc(self) -> Optional[str]:
+        return self.modifiers.get('fail_calc')
 
     def tags(self) -> List[str]:
         tags = self.modifiers.get('tags', [])
@@ -303,23 +353,39 @@ class TestBuilder(Generic[Testable]):
             name = '{}_{}'.format(self.namespace, name)
         return get_nice_schema_test_name(name, self.target.name, self.args)
 
+    def construct_config(self) -> str:
+        configs = ",".join([
+            f"{key}=" + (
+                ("\"" + value.replace('\"', '\\\"') + "\"") if isinstance(value, str)
+                else str(value)
+            )
+            for key, value
+            in self.modifiers.items()
+        ])
+        if configs:
+            return f"{{{{ config({configs}) }}}}"
+        else:
+            return ""
+
     # this is the 'raw_sql' that's used in 'render_update' and execution
     # of the test macro
     def build_raw_sql(self) -> str:
         return (
-            "{{{{ config(severity='{severity}') }}}}"
-            "{{{{ {macro}(**{kwargs_name}) }}}}"
+            "{{{{ {macro}(**{kwargs_name}) }}}}{config}"
         ).format(
             macro=self.macro_name(),
-            severity=self.severity(),
+            config=self.construct_config(),
             kwargs_name=SCHEMA_TEST_KWARGS_NAME,
         )
 
     def build_model_str(self):
+        targ = self.target
+        cfg_where = "config.get('where')"
         if isinstance(self.target, UnparsedNodeUpdate):
-            fmt = "{{{{ ref('{0.name}') }}}}"
+            identifier = self.target.name
+            target_str = f"{{{{ ref('{targ.name}') }}}}"
         elif isinstance(self.target, UnpatchedSourceDefinition):
-            fmt = "{{{{ source('{0.source.name}', '{0.table.name}') }}}}"
-        else:
-            raise self._bad_type()
-        return fmt.format(self.target)
+            identifier = self.target.table.name
+            target_str = f"{{{{ source('{targ.source.name}', '{targ.table.name}') }}}}"
+        filtered = f"(select * from {target_str} where {{{{{cfg_where}}}}}) {identifier}"
+        return f"{{% if {cfg_where} %}}{filtered}{{% else %}}{target_str}{{% endif %}}"

core/dbt/parser/schemas.py

@@ -2,6 +2,7 @@ import itertools
 import os
 
 from abc import ABCMeta, abstractmethod
+from hashlib import md5
 from typing import (
     Iterable, Dict, Any, Union, List, Optional, Generic, TypeVar, Type
 )
@@ -13,10 +14,7 @@ from dbt.clients.jinja import get_rendered, add_rendered_test_kwargs
 from dbt.clients.yaml_helper import load_yaml_text
 from dbt.config.renderer import SchemaYamlRenderer
 from dbt.context.context_config import (
-    BaseContextConfigGenerator,
     ContextConfig,
-    ContextConfigGenerator,
-    UnrenderedConfigGenerator,
 )
 from dbt.context.configured import generate_schema_yml
 from dbt.context.target import generate_target_context
@@ -25,11 +23,9 @@ from dbt.context.providers import (
 )
 from dbt.context.macro_resolver import MacroResolver
 from dbt.contracts.files import FileHash
-from dbt.contracts.graph.manifest import SourceFile
-from dbt.contracts.graph.model_config import SourceConfig
+from dbt.contracts.graph.manifest import SchemaSourceFile
 from dbt.contracts.graph.parsed import (
     ParsedNodePatch,
-    ParsedSourceDefinition,
     ColumnInfo,
     ParsedSchemaTestNode,
     ParsedMacroPatch,
@@ -37,7 +33,6 @@ from dbt.contracts.graph.parsed import (
     ParsedExposure,
 )
 from dbt.contracts.graph.unparsed import (
-    FreshnessThreshold,
     HasColumnDocs,
     HasColumnTests,
     HasDocs,
@@ -52,11 +47,11 @@ from dbt.contracts.graph.unparsed import (
 from dbt.exceptions import (
     validator_error_message, JSONValidationException,
     raise_invalid_schema_yml_version, ValidationException,
-    CompilationException, warn_or_error, InternalException
+    CompilationException,
 )
 from dbt.node_types import NodeType
 from dbt.parser.base import SimpleParser
-from dbt.parser.search import FileBlock, FilesystemSearcher
+from dbt.parser.search import FileBlock
 from dbt.parser.schema_test_builders import (
     TestBuilder, SchemaTestBlock, TargetBlock, YamlBlock,
     TestBlock, Testable
@@ -75,6 +70,11 @@ UnparsedSchemaYaml = Union[
 
 TestDef = Union[str, Dict[str, Any]]
 
+schema_file_keys = (
+    'models', 'seeds', 'snapshots', 'sources',
+    'macros', 'analyses', 'exposures',
+)
+
 
 def error_context(
     path: str,
@@ -96,6 +96,22 @@ def error_context(
     )
 
 
+def yaml_from_file(
+    source_file: SchemaSourceFile
+) -> Dict[str, Any]:
+    """If loading the yaml fails, raise an exception.
+    """
+    path = source_file.path.relative_path
+    try:
+        return load_yaml_text(source_file.contents)
+    except ValidationException as e:
+        reason = validator_error_message(e)
+        raise CompilationException(
+            'Error reading {}: {} - {}'
+            .format(source_file.project_name, path, reason)
+        )
+
+
 class ParserRef:
     """A helper object to hold parse-time references."""
 
@@ -145,22 +161,11 @@ def _trimmed(inp: str) -> str:
     return inp[:44] + '...' + inp[-3:]
 
 
-def merge_freshness(
-    base: Optional[FreshnessThreshold], update: Optional[FreshnessThreshold]
-) -> Optional[FreshnessThreshold]:
-    if base is not None and update is not None:
-        return base.merged(update)
-    elif base is None and update is not None:
-        return update
-    else:
-        return None
-
-
 class SchemaParser(SimpleParser[SchemaTestBlock, ParsedSchemaTestNode]):
     def __init__(
-        self, results, project, root_project, macro_manifest,
+        self, project, manifest, root_project,
     ) -> None:
-        super().__init__(results, project, root_project, macro_manifest)
+        super().__init__(project, manifest, root_project)
         all_v_2 = (
             self.root_project.config_version == 2 and
             self.project.config_version == 2
@@ -180,7 +185,7 @@ class SchemaParser(SimpleParser[SchemaTestBlock, ParsedSchemaTestNode]):
             self.root_project.credentials.type
         )
         self.macro_resolver = MacroResolver(
-            self.macro_manifest.macros,
+            self.manifest.macros,
             self.root_project.project_name,
             internal_package_names
         )
@@ -194,65 +199,11 @@ class SchemaParser(SimpleParser[SchemaTestBlock, ParsedSchemaTestNode]):
     def resource_type(self) -> NodeType:
         return NodeType.Test
 
-    def get_paths(self):
-        # TODO: In order to support this, make FilesystemSearcher accept a list
-        # of file patterns. eg: ['.yml', '.yaml']
-        yaml_files = list(FilesystemSearcher(
-            self.project, self.project.all_source_paths, '.yaml'
-        ))
-        if yaml_files:
-            warn_or_error(
-                'A future version of dbt will parse files with both'
-                ' .yml and .yaml file extensions. dbt found'
-                f' {len(yaml_files)} files with .yaml extensions in'
-                ' your dbt project. To avoid errors when upgrading'
-                ' to a future release, either remove these files from'
-                ' your dbt project, or change their extensions.'
-            )
-        return FilesystemSearcher(
-            self.project, self.project.all_source_paths, '.yml'
-        )
-
     def parse_from_dict(self, dct, validate=True) -> ParsedSchemaTestNode:
         if validate:
             ParsedSchemaTestNode.validate(dct)
         return ParsedSchemaTestNode.from_dict(dct)
 
-    def _check_format_version(
-        self, yaml: YamlBlock
-    ) -> None:
-        path = yaml.path.relative_path
-        if 'version' not in yaml.data:
-            raise_invalid_schema_yml_version(path, 'no version is specified')
-
-        version = yaml.data['version']
-        # if it's not an integer, the version is malformed, or not
-        # set. Either way, only 'version: 2' is supported.
-        if not isinstance(version, int):
-            raise_invalid_schema_yml_version(
-                path, 'the version is not an integer'
-            )
-        if version != 2:
-            raise_invalid_schema_yml_version(
-                path, 'version {} is not supported'.format(version)
-            )
-
-    def _yaml_from_file(
-        self, source_file: SourceFile
-    ) -> Optional[Dict[str, Any]]:
-        """If loading the yaml fails, raise an exception.
-        """
-        path: str = source_file.path.relative_path
-        try:
-            return load_yaml_text(source_file.contents)
-        except ValidationException as e:
-            reason = validator_error_message(e)
-            raise CompilationException(
-                'Error reading {}: {} - {}'
-                .format(self.project.project_name, path, reason)
-            )
-        return None
-
     def parse_column_tests(
         self, block: TestBlock, column: UnparsedColumn
     ) -> None:
@@ -262,99 +213,6 @@ class SchemaParser(SimpleParser[SchemaTestBlock, ParsedSchemaTestNode]):
         for test in column.tests:
             self.parse_test(block, test, column)
 
-    def _generate_source_config(self, fqn: List[str], rendered: bool):
-        generator: BaseContextConfigGenerator
-        if rendered:
-            generator = ContextConfigGenerator(self.root_project)
-        else:
-            generator = UnrenderedConfigGenerator(
-                self.root_project
-            )
-
-        return generator.calculate_node_config(
-            config_calls=[],
-            fqn=fqn,
-            resource_type=NodeType.Source,
-            project_name=self.project.project_name,
-            base=False,
-        )
-
-    def _get_relation_name(self, node: ParsedSourceDefinition):
-        adapter = get_adapter(self.root_project)
-        relation_cls = adapter.Relation
-        return str(relation_cls.create_from(self.root_project, node))
-
-    def parse_source(
-        self, target: UnpatchedSourceDefinition
-    ) -> ParsedSourceDefinition:
-        source = target.source
-        table = target.table
-        refs = ParserRef.from_target(table)
-        unique_id = target.unique_id
-        description = table.description or ''
-        meta = table.meta or {}
-        source_description = source.description or ''
-        loaded_at_field = table.loaded_at_field or source.loaded_at_field
-
-        freshness = merge_freshness(source.freshness, table.freshness)
-        quoting = source.quoting.merged(table.quoting)
-        # path = block.path.original_file_path
-        source_meta = source.meta or {}
-
-        # make sure we don't do duplicate tags from source + table
-        tags = sorted(set(itertools.chain(source.tags, table.tags)))
-
-        config = self._generate_source_config(
-            fqn=target.fqn,
-            rendered=True,
-        )
-
-        unrendered_config = self._generate_source_config(
-            fqn=target.fqn,
-            rendered=False,
-        )
-
-        if not isinstance(config, SourceConfig):
-            raise InternalException(
-                f'Calculated a {type(config)} for a source, but expected '
-                f'a SourceConfig'
-            )
-
-        default_database = self.root_project.credentials.database
-
-        parsed_source = ParsedSourceDefinition(
-            package_name=target.package_name,
-            database=(source.database or default_database),
-            schema=(source.schema or source.name),
-            identifier=(table.identifier or table.name),
-            root_path=target.root_path,
-            path=target.path,
-            original_file_path=target.original_file_path,
-            columns=refs.column_info,
-            unique_id=unique_id,
-            name=table.name,
-            description=description,
-            external=table.external,
-            source_name=source.name,
-            source_description=source_description,
-            source_meta=source_meta,
-            meta=meta,
-            loader=source.loader,
-            loaded_at_field=loaded_at_field,
-            freshness=freshness,
-            quoting=quoting,
-            resource_type=NodeType.Source,
-            fqn=target.fqn,
-            tags=tags,
-            config=config,
-            unrendered_config=unrendered_config,
-        )
-
-        # relation name is added after instantiation because the adapter does
-        # not provide the relation name for a UnpatchedSourceDefinition object
-        parsed_source.relation_name = self._get_relation_name(parsed_source)
-        return parsed_source
-
     def create_test_node(
         self,
         target: Union[UnpatchedSourceDefinition, UnparsedNodeUpdate],
@@ -368,6 +226,25 @@ class SchemaParser(SimpleParser[SchemaTestBlock, ParsedSchemaTestNode]):
         column_name: Optional[str],
     ) -> ParsedSchemaTestNode:
 
+        HASH_LENGTH = 10
+
+        # N.B: This function builds a hashable string from any given test_metadata dict.
+        #   it's a bit fragile for general use (only supports str, int, float, List, Dict)
+        #   but it gets the job done here without the overhead of complete ser(de).
+        def get_hashable_md(
+            data: Union[str, int, float, List, Dict]
+        ) -> Union[str, List, Dict]:
+            if type(data) == dict:
+                return {k: get_hashable_md(data[k]) for k in sorted(data.keys())}  # type: ignore
+            elif type(data) == list:
+                return [get_hashable_md(val) for val in data]  # type: ignore
+            else:
+                return str(data)
+
+        hashable_metadata = repr(get_hashable_md(test_metadata))
+        hash_string = ''.join([name, hashable_metadata]).encode('utf-8')
+        test_hash = md5(hash_string).hexdigest()[-HASH_LENGTH:]
+
         dct = {
             'alias': name,
             'schema': self.default_schema,
@@ -381,7 +258,7 @@ class SchemaParser(SimpleParser[SchemaTestBlock, ParsedSchemaTestNode]):
             'original_file_path': target.original_file_path,
             'package_name': self.project.project_name,
             'raw_sql': raw_sql,
-            'unique_id': self.generate_unique_id(name),
+            'unique_id': self.generate_unique_id(name, test_hash),
             'config': self.config_dict(config),
             'test_metadata': test_metadata,
             'column_name': column_name,
@@ -484,8 +361,36 @@ class SchemaParser(SimpleParser[SchemaTestBlock, ParsedSchemaTestNode]):
         if (macro_unique_id in
                 ['macro.dbt.test_not_null', 'macro.dbt.test_unique']):
             self.update_parsed_node(node, config)
-            node.unrendered_config['severity'] = builder.severity()
-            node.config['severity'] = builder.severity()
+            # manually set configs
+            # note: this does not respect generate_alias_name() macro
+            if builder.alias is not None:
+                node.unrendered_config['alias'] = builder.alias
+                node.config['alias'] = builder.alias
+                node.alias = builder.alias
+            if builder.severity is not None:
+                node.unrendered_config['severity'] = builder.severity
+                node.config['severity'] = builder.severity
+            if builder.enabled is not None:
+                node.unrendered_config['enabled'] = builder.enabled
+                node.config['enabled'] = builder.enabled
+            if builder.where is not None:
+                node.unrendered_config['where'] = builder.where
+                node.config['where'] = builder.where
+            if builder.limit is not None:
+                node.unrendered_config['limit'] = builder.limit
+                node.config['limit'] = builder.limit
+            if builder.warn_if is not None:
+                node.unrendered_config['warn_if'] = builder.warn_if
+                node.config['warn_if'] = builder.warn_if
+            if builder.error_if is not None:
+                node.unrendered_config['error_if'] = builder.error_if
+                node.config['error_if'] = builder.error_if
+            if builder.fail_calc is not None:
+                node.unrendered_config['fail_calc'] = builder.fail_calc
+                node.config['fail_calc'] = builder.fail_calc
+            if builder.store_failures is not None:
+                node.unrendered_config['store_failures'] = builder.store_failures
+                node.config['store_failures'] = builder.store_failures
             # source node tests are processed at patch_source time
             if isinstance(builder.target, UnpatchedSourceDefinition):
                 sources = [builder.target.fqn[-2], builder.target.fqn[-1]]
@@ -496,7 +401,7 @@ class SchemaParser(SimpleParser[SchemaTestBlock, ParsedSchemaTestNode]):
             try:
                 # make a base context that doesn't have the magic kwargs field
                 context = generate_test_context(
-                    node, self.root_project, self.macro_manifest, config,
+                    node, self.root_project, self.manifest, config,
                     self.macro_resolver,
                 )
                 # update with rendered test kwargs (which collects any refs)
@@ -511,42 +416,6 @@ class SchemaParser(SimpleParser[SchemaTestBlock, ParsedSchemaTestNode]):
                 msg = validator_error_message(exc)
                 raise CompilationException(msg, node=node) from exc
 
-    def parse_source_test(
-        self,
-        target: UnpatchedSourceDefinition,
-        test: Dict[str, Any],
-        column: Optional[UnparsedColumn],
-    ) -> ParsedSchemaTestNode:
-        column_name: Optional[str]
-        if column is None:
-            column_name = None
-        else:
-            column_name = column.name
-            should_quote = (
-                column.quote or
-                (column.quote is None and target.quote_columns)
-            )
-            if should_quote:
-                column_name = get_adapter(self.root_project).quote(column_name)
-
-        tags_sources = [target.source.tags, target.table.tags]
-        if column is not None:
-            tags_sources.append(column.tags)
-        tags = list(itertools.chain.from_iterable(tags_sources))
-
-        node = self._parse_generic_test(
-            target=target,
-            test=test,
-            tags=tags,
-            column_name=column_name
-        )
-        # we can't go through result.add_node - no file... instead!
-        if node.config.enabled:
-            self.results.add_node_nofile(node)
-        else:
-            self.results.add_disabled_nofile(node)
-        return node
-
     def parse_node(self, block: SchemaTestBlock) -> ParsedSchemaTestNode:
         """In schema parsing, we rewrite most of the part of parse_node that
         builds the initial node to be parsed, but rendering is basically the
@@ -558,9 +427,16 @@ class SchemaParser(SimpleParser[SchemaTestBlock, ParsedSchemaTestNode]):
             tags=block.tags,
             column_name=block.column_name,
         )
-        self.add_result_node(block, node)
+        self.add_test_node(block, node)
         return node
 
+    def add_test_node(self, block: SchemaTestBlock, node: ParsedSchemaTestNode):
+        test_from = {"key": block.target.yaml_key, "name": block.target.name}
+        if node.config.enabled:
+            self.manifest.add_node(block.file, node, test_from)
+        else:
+            self.manifest.add_disabled(block.file, node, test_from)
+
     def render_with_context(
         self, node: ParsedSchemaTestNode, config: ContextConfig,
     ) -> None:
@@ -615,20 +491,14 @@ class SchemaParser(SimpleParser[SchemaTestBlock, ParsedSchemaTestNode]):
         for test in block.tests:
             self.parse_test(block, test, None)
 
-    def parse_exposures(self, block: YamlBlock) -> None:
-        parser = ExposureParser(self, block)
-        for node in parser.parse():
-            self.results.add_exposure(block.file, node)
-
-    def parse_file(self, block: FileBlock) -> None:
-        dct = self._yaml_from_file(block.file)
-
-        # mark the file as seen, in ParseResult.files
-        self.results.get_file(block.file)
+    def parse_file(self, block: FileBlock, dct: Dict = None) -> None:
+        assert isinstance(block.file, SchemaSourceFile)
+        if not dct:
+            dct = yaml_from_file(block.file)
 
         if dct:
             try:
-                # This does a deep_map to check for circular references
+                # This does a deep_map which will fail if there are circular references
                 dct = self.raw_renderer.render_data(dct)
             except CompilationException as exc:
                 raise CompilationException(
@@ -639,9 +509,6 @@ class SchemaParser(SimpleParser[SchemaTestBlock, ParsedSchemaTestNode]):
             # contains the FileBlock and the data (dictionary)
             yaml_block = YamlBlock.from_file_block(block, dct)
 
-            # checks version
-            self._check_format_version(yaml_block)
-
             parser: YamlDocsReader
 
             # There are 7 kinds of parsers:
@@ -673,21 +540,40 @@ class SchemaParser(SimpleParser[SchemaTestBlock, ParsedSchemaTestNode]):
                 parser = SourceParser(self, yaml_block, 'sources')
                 parser.parse()
 
-            # NonSourceParser.parse()
+            # NonSourceParser.parse() (but never test_blocks)
             if 'macros' in dct:
                 parser = MacroPatchParser(self, yaml_block, 'macros')
-                for test_block in parser.parse():
-                    self.parse_tests(test_block)
+                parser.parse()
 
-            # NonSourceParser.parse()
+            # NonSourceParser.parse() (but never test_blocks)
             if 'analyses' in dct:
                 parser = AnalysisPatchParser(self, yaml_block, 'analyses')
-                for test_block in parser.parse():
-                    self.parse_tests(test_block)
+                parser.parse()
 
             # parse exposures
             if 'exposures' in dct:
-                self.parse_exposures(yaml_block)
+                exp_parser = ExposureParser(self, yaml_block)
+                for node in exp_parser.parse():
+                    self.manifest.add_exposure(yaml_block.file, node)
+
+
+def check_format_version(
+    file_path, yaml_dct
+) -> None:
+    if 'version' not in yaml_dct:
+        raise_invalid_schema_yml_version(file_path, 'no version is specified')
+
+    version = yaml_dct['version']
+    # if it's not an integer, the version is malformed, or not
+    # set. Either way, only 'version: 2' is supported.
+    if not isinstance(version, int):
+        raise_invalid_schema_yml_version(
+            file_path, 'the version is not an integer'
+        )
+    if version != 2:
+        raise_invalid_schema_yml_version(
+            file_path, 'version {} is not supported'.format(version)
+        )
 
 
 Parsed = TypeVar(
@@ -716,8 +602,8 @@ class YamlReader(metaclass=ABCMeta):
         self.yaml = yaml
 
     @property
-    def results(self):
-        return self.schema_parser.results
+    def manifest(self):
+        return self.schema_parser.manifest
 
     @property
     def project(self):
@@ -765,6 +651,7 @@ class YamlDocsReader(YamlReader):
 T = TypeVar('T', bound=dbtClassMixin)
 
 
+# This parses the 'sources' keys in yaml files.
 class SourceParser(YamlDocsReader):
     def _target_from_dict(self, cls: Type[T], data: Dict[str, Any]) -> T:
         path = self.yaml.path.original_file_path
@@ -775,7 +662,11 @@ class SourceParser(YamlDocsReader):
             msg = error_context(path, self.key, data, exc)
             raise CompilationException(msg) from exc
 
-    # the other parse method returns TestBlocks. This one doesn't.
+    # The other parse method returns TestBlocks. This one doesn't.
+    # This takes the yaml dictionaries in 'sources' keys and uses them
+    # to create UnparsedSourceDefinition objects. They are then turned
+    # into UnpatchedSourceDefinition objects in 'add_source_definitions'
+    # or SourcePatch objects in 'add_source_patch'
     def parse(self) -> List[TestBlock]:
         # get a verified list of dicts for the key handled by this parser
         for data in self.get_key_dicts():
@@ -787,7 +678,7 @@ class SourceParser(YamlDocsReader):
             if is_override:
                 data['path'] = self.yaml.path.original_file_path
                 patch = self._target_from_dict(SourcePatch, data)
-                self.results.add_source_patch(self.yaml.file, patch)
+                self.manifest.add_source_patch(self.yaml.file, patch)
             else:
                 source = self._target_from_dict(UnparsedSourceDefinition, data)
                 self.add_source_definitions(source)
@@ -806,7 +697,7 @@ class SourceParser(YamlDocsReader):
             fqn = self.schema_parser.get_fqn_prefix(fqn_path)
             fqn.extend([source.name, table.name])
 
-            result = UnpatchedSourceDefinition(
+            source_def = UnpatchedSourceDefinition(
                 source=source,
                 table=table,
                 path=original_file_path,
@@ -817,7 +708,7 @@ class SourceParser(YamlDocsReader):
                 resource_type=NodeType.Source,
                 fqn=fqn,
             )
-            self.results.add_source(self.yaml.file, result)
+            self.manifest.add_source(self.yaml.file, source_def)
 
 
 # This class has three main subclasses: TestablePatchParser (models,
@@ -860,9 +751,10 @@ class NonSourceParser(YamlDocsReader, Generic[NonSourceTarget, Parsed]):
                 refs: ParserRef = ParserRef.from_target(node)
             else:
                 refs = ParserRef()
-            # This adds the node_block to self.results (a ParseResult
-            # object) as a ParsedNodePatch or ParsedMacroPatch
+            # This adds the node_block to self.manifest
+            # as a ParsedNodePatch or ParsedMacroPatch
             self.parse_patch(node_block, refs)
+        # This will always be empty if the node a macro or analysis
         return test_blocks
 
     def get_unparsed_target(self) -> Iterable[NonSourceTarget]:
@@ -898,7 +790,7 @@ class NodePatchParser(
     def parse_patch(
         self, block: TargetBlock[NodeTarget], refs: ParserRef
     ) -> None:
-        result = ParsedNodePatch(
+        patch = ParsedNodePatch(
             name=block.target.name,
             original_file_path=block.target.original_file_path,
             yaml_key=block.target.yaml_key,
@@ -908,7 +800,7 @@ class NodePatchParser(
             meta=block.target.meta,
             docs=block.target.docs,
         )
-        self.results.add_patch(self.yaml.file, result)
+        self.manifest.add_patch(self.yaml.file, patch)
 
 
 class TestablePatchParser(NodePatchParser[UnparsedNodeUpdate]):
@@ -937,7 +829,7 @@ class MacroPatchParser(NonSourceParser[UnparsedMacroUpdate, ParsedMacroPatch]):
     def parse_patch(
         self, block: TargetBlock[UnparsedMacroUpdate], refs: ParserRef
     ) -> None:
-        result = ParsedMacroPatch(
+        patch = ParsedMacroPatch(
             name=block.target.name,
             original_file_path=block.target.original_file_path,
             yaml_key=block.target.yaml_key,
@@ -947,7 +839,7 @@ class MacroPatchParser(NonSourceParser[UnparsedMacroUpdate, ParsedMacroPatch]):
             meta=block.target.meta,
             docs=block.target.docs,
         )
-        self.results.add_macro_patch(self.yaml.file, result)
+        self.manifest.add_macro_patch(self.yaml.file, patch)
 
 
 class ExposureParser(YamlReader):
@@ -974,6 +866,8 @@ class ExposureParser(YamlReader):
             name=unparsed.name,
             type=unparsed.type,
             url=unparsed.url,
+            meta=unparsed.meta,
+            tags=unparsed.tags,
             description=unparsed.description,
             owner=unparsed.owner,
             maturity=unparsed.maturity,
@@ -981,7 +875,7 @@ class ExposureParser(YamlReader):
         ctx = generate_parse_exposure(
             parsed,
             self.root_project,
-            self.schema_parser.macro_manifest,
+            self.schema_parser.manifest,
             package_name,
         )
         depends_on_jinja = '\n'.join(