Initial file creation of code documentation READMEs

* add lazy type and apply to cache events

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 1.0.0rc3
+current_version = 1.0.1
 parse = (?P<major>\d+)
 	\.(?P<minor>\d+)
 	\.(?P<patch>\d+)
@@ -35,5 +35,3 @@ first_value = 1
 [bumpversion:file:plugins/postgres/setup.py]
 
 [bumpversion:file:plugins/postgres/dbt/adapters/postgres/__version__.py]
-
-[bumpversion:file:docker/requirements/requirements.txt]

.github/CODEOWNERS

@@ -0,0 +1,43 @@
+# This file contains the code owners for the dbt-core repo.
+# PRs will be automatically assigned for review to the associated 
+# team(s) or person(s) that touches any files that are mapped to them.
+#
+# A statement takes precedence over the statements above it so more general
+# assignments are found at the top with specific assignments being lower in
+# the ordering (i.e. catch all assignment should be the first item)
+#
+# Consult GitHub documentation for formatting guidelines:
+# https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/about-code-owners#example-of-a-codeowners-file
+
+# As a default for areas with no assignment, 
+# the core team as a whole will be assigned
+*       @dbt-labs/core
+
+# Changes to GitHub configurations including Actions
+/.github/ @leahwicz
+
+# Language core modules
+/core/dbt/config/     @dbt-labs/core-language
+/core/dbt/context/    @dbt-labs/core-language
+/core/dbt/contracts/  @dbt-labs/core-language
+/core/dbt/deps/       @dbt-labs/core-language
+/core/dbt/parser/     @dbt-labs/core-language
+
+# Execution core modules
+/core/dbt/events/     @dbt-labs/core-execution @dbt-labs/core-language # eventually remove language but they have knowledge here now
+/core/dbt/graph/      @dbt-labs/core-execution
+/core/dbt/task/       @dbt-labs/core-execution
+
+# Adapter interface, scaffold, Postgres plugin
+/core/dbt/adapters    @dbt-labs/core-adapters
+/core/scripts/create_adapter_plugin.py    @dbt-labs/core-adapters
+/plugins/             @dbt-labs/core-adapters
+
+# Global project: default macros, including generic tests + materializations
+/core/dbt/include/global_project    @dbt-labs/core-execution @dbt-labs/core-adapters
+
+# Perf regression testing framework
+# This excludes the test project files itself since those aren't specific 
+# framework changes (excluded by not setting an owner next to it- no owner)
+/performance @nathaniel-may
+/performance/projects

.github/workflows/backport.yml

@@ -0,0 +1,34 @@
+# **what?**
+# When a PR is merged, if it has the backport label, it will create 
+# a new PR to backport those changes to the given branch. If it can't
+# cleanly do a backport, it will comment on the merged PR of the failure.
+#
+# Label naming convention: "backport <branch name to backport to>"
+# Example: backport 1.0.latest
+#
+# You MUST "Squash and merge" the original PR or this won't work.
+
+# **why?**
+# Changes sometimes need to be backported to release branches.
+# This automates the backporting process
+
+# **when?**
+# Once a PR is "Squash and merge"'d and it has been correctly labeled
+# according to the naming convention.
+
+name: Backport
+on:
+  pull_request:
+    types:
+      - closed
+      - labeled
+
+jobs:
+  backport:
+    runs-on: ubuntu-18.04
+    name: Backport
+    steps:
+      - name: Backport
+        uses: tibdex/backport@v1.1.1
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/jira-creation.yml

@@ -0,0 +1,26 @@
+# **what?**
+# Mirrors issues into Jira. Includes the information: title,
+# GitHub Issue ID and URL
+
+# **why?**
+# Jira is our tool for tracking and we need to see these issues in there
+
+# **when?**
+# On issue creation or when an issue is labeled `Jira`
+
+name: Jira Issue Creation
+
+on:
+  issues:
+    types: [opened, labeled]
+    
+permissions:
+  issues: write
+
+jobs:
+  call-label-action:
+    uses: dbt-labs/jira-actions/.github/workflows/jira-creation.yml@main
+    secrets:
+      JIRA_BASE_URL: ${{ secrets.JIRA_BASE_URL }}
+      JIRA_USER_EMAIL: ${{ secrets.JIRA_USER_EMAIL }}
+      JIRA_API_TOKEN: ${{ secrets.JIRA_API_TOKEN }}

.github/workflows/jira-label.yml

@@ -0,0 +1,27 @@
+# **what?**
+# Calls mirroring Jira label Action. Includes adding a new label
+# to an existing issue or removing a label as well
+
+# **why?**
+# Jira is our tool for tracking and we need to see these labels in there
+
+# **when?**
+# On labels being added or removed from issues
+
+name: Jira Label Mirroring
+
+on:
+  issues:
+    types: [labeled, unlabeled]
+    
+permissions:
+  issues: read
+
+jobs:
+  call-label-action:
+    uses: dbt-labs/jira-actions/.github/workflows/jira-label.yml@main
+    secrets:
+      JIRA_BASE_URL: ${{ secrets.JIRA_BASE_URL }}
+      JIRA_USER_EMAIL: ${{ secrets.JIRA_USER_EMAIL }}
+      JIRA_API_TOKEN: ${{ secrets.JIRA_API_TOKEN }}
+    

.github/workflows/jira-transition.yml

@@ -0,0 +1,24 @@
+# **what?**
+# Transition a Jira issue to a new state
+# Only supports these GitHub Issue transitions:
+#   closed, deleted, reopened
+
+# **why?**
+# Jira needs to be kept up-to-date
+
+# **when?**
+# On issue closing, deletion, reopened
+
+name: Jira Issue Transition
+
+on:
+  issues:
+    types: [closed, deleted, reopened]
+
+jobs:
+  call-label-action:
+    uses: dbt-labs/jira-actions/.github/workflows/jira-transition.yml@main
+    secrets:
+      JIRA_BASE_URL: ${{ secrets.JIRA_BASE_URL }}
+      JIRA_USER_EMAIL: ${{ secrets.JIRA_USER_EMAIL }}
+      JIRA_API_TOKEN: ${{ secrets.JIRA_API_TOKEN }}

.github/workflows/release.yml

@@ -95,7 +95,9 @@ jobs:
       - uses: actions/upload-artifact@v2
         with:
           name: dist
-          path: dist/
+          path: |
+            dist/
+            !dist/dbt-${{github.event.inputs.version_number}}.tar.gz
 
   test-build:
     name: verify packages

.github/workflows/structured-logging-schema-check.yml

@@ -0,0 +1,71 @@
+# This Action checks makes a dbt run to sample json structured logs
+# and checks that they conform to the currently documented schema.
+# 
+# If this action fails it either means we have unintentionally deviated
+# from our documented structured logging schema, or we need to bump the
+# version of our structured logging and add new documentation to
+# communicate these changes.
+
+
+name: Structured Logging Schema Check
+on:
+  push:
+    branches:
+      - "main"
+      - "*.latest"
+      - "releases/*"
+  pull_request:
+  workflow_dispatch:
+
+permissions: read-all
+
+jobs:
+  # run the performance measurements on the current or default branch
+  test-schema:
+    name: Test Log Schema
+    runs-on: ubuntu-latest
+    env:
+      # turns warnings into errors
+      RUSTFLAGS: "-D warnings"
+      # points tests to the log file
+      LOG_DIR: "/home/runner/work/dbt-core/dbt-core/logs"
+      # tells integration tests to output into json format
+      DBT_LOG_FORMAT: 'json'
+    steps:
+
+      - name: checkout dev
+        uses: actions/checkout@v2
+        with:
+          persist-credentials: false
+
+      - name: Setup Python
+        uses: actions/setup-python@v2.2.2
+        with:
+          python-version: "3.8"
+
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: stable
+          override: true
+
+      - name: install dbt
+        run: pip install -r dev-requirements.txt -r editable-requirements.txt
+
+      - name: Set up postgres
+        uses: ./.github/actions/setup-postgres-linux
+
+      - name: ls
+        run: ls
+
+      # integration tests generate a ton of logs in different files. the next step will find them all.
+      # we actually care if these pass, because the normal test run doesn't usually include many json log outputs
+      - name: Run integration tests
+        run: tox -e py38-postgres -- -nauto
+
+      # apply our schema tests to every log event from the previous step
+      # skips any output that isn't valid json
+      - uses: actions-rs/cargo@v1
+        with:
+          command: run
+          args: --manifest-path test/interop/log_parsing/Cargo.toml

.github/workflows/version-bump.yml

@@ -66,12 +66,12 @@ jobs:
           git push origin bumping-version/${{steps.variables.outputs.VERSION_NUMBER}}_$GITHUB_RUN_ID
           git branch --set-upstream-to=origin/bumping-version/${{steps.variables.outputs.VERSION_NUMBER}}_$GITHUB_RUN_ID bumping-version/${{steps.variables.outputs.VERSION_NUMBER}}_$GITHUB_RUN_ID
       
-      - name: Generate Docker requirements
-        run: |
-          source env/bin/activate
-          pip install -r requirements.txt 
-          pip freeze -l > docker/requirements/requirements.txt
-          git status
+      # - name: Generate Docker requirements
+      #  run: |
+      #    source env/bin/activate
+      #    pip install -r requirements.txt 
+      #    pip freeze -l > docker/requirements/requirements.txt
+      #    git status
 
       - name: Bump version
         run: |

ARCHITECTURE.md

@@ -4,16 +4,17 @@ The core function of dbt is SQL compilation and execution. Users create projects
 
 Most of the python code in the repository is within the `core/dbt` directory. Currently the main subdirectories are:
 
-- [`adapters`](core/dbt/adapters): Define base classes for behavior that is likely to differ across databases
-- [`clients`](core/dbt/clients): Interface with dependencies (agate, jinja) or across operating systems
-- [`config`](core/dbt/config): Reconcile user-supplied configuration from connection profiles, project files, and Jinja macros
-- [`context`](core/dbt/context): Build and expose dbt-specific Jinja functionality
-- [`contracts`](core/dbt/contracts): Define Python objects (dataclasses) that dbt expects to create and validate
-- [`deps`](core/dbt/deps): Package installation and dependency resolution
-- [`graph`](core/dbt/graph): Produce a `networkx` DAG of project resources, and selecting those resources given user-supplied criteria
-- [`include`](core/dbt/include): The dbt "global project," which defines default implementations of Jinja2 macros
-- [`parser`](core/dbt/parser): Read project files, validate, construct python objects
-- [`task`](core/dbt/task): Set forth the actions that dbt can perform when invoked
+- [`adapters`](core/dbt/adapters/README.md): Define base classes for behavior that is likely to differ across databases
+- [`clients`](core/dbt/clients/README.md): Interface with dependencies (agate, jinja) or across operating systems
+- [`config`](core/dbt/config/README.md): Reconcile user-supplied configuration from connection profiles, project files, and Jinja macros
+- [`context`](core/dbt/context/README.md): Build and expose dbt-specific Jinja functionality
+- [`contracts`](core/dbt/contracts/README.md): Define Python objects (dataclasses) that dbt expects to create and validate
+- [`deps`](core/dbt/deps/README.md): Package installation and dependency resolution
+- [`events`](core/dbt/events/README.md): Logging events
+- [`graph`](core/dbt/graph/README.md): Produce a `networkx` DAG of project resources, and selecting those resources given user-supplied criteria
+- [`include`](core/dbt/include/README.md): The dbt "global project," which defines default implementations of Jinja2 macros
+- [`parser`](core/dbt/parser/README.md): Read project files, validate, construct python objects
+- [`task`](core/dbt/task/README.md): Set forth the actions that dbt can perform when invoked
 
 ### Invoking dbt
 
@@ -44,4 +45,4 @@ The [`test/`](test/) subdirectory includes unit and integration tests that run a
 
 - [docker](docker/): All dbt versions are published as Docker images on DockerHub. This subfolder contains the `Dockerfile` (constant) and `requirements.txt` (one for each version).
 - [etc](etc/): Images for README
-- [scripts](scripts/): Helper scripts for testing, releasing, and producing JSON schemas. These are not included in distributions of dbt, not are they rigorously tested—they're just handy tools for the dbt maintainers :)
+- [scripts](scripts/): Helper scripts for testing, releasing, and producing JSON schemas. These are not included in distributions of dbt, nor are they rigorously tested—they're just handy tools for the dbt maintainers :)

CHANGELOG.md

@@ -1,4 +1,79 @@
-## dbt-core 1.0.0 (Release TBD)
+## dbt-core 1.1.0 (TBD)
+
+### Features
+- New Dockerfile to support specific db adapters and platforms.  See docker/README.md for details ([#4495](https://github.com/dbt-labs/dbt-core/issues/4495), [#4487](https://github.com/dbt-labs/dbt-core/pull/4487))
+
+### Fixes
+- User wasn't asked for permission to overwite a profile entry when running init inside an existing project ([#4375](https://github.com/dbt-labs/dbt-core/issues/4375), [#4447](https://github.com/dbt-labs/dbt-core/pull/4447))
+- Add project name validation to `dbt init` ([#4490](https://github.com/dbt-labs/dbt-core/issues/4490),[#4536](https://github.com/dbt-labs/dbt-core/pull/4536))
+
+### Under the hood
+- Testing cleanup ([#4496](https://github.com/dbt-labs/dbt-core/pull/4496), [#4509](https://github.com/dbt-labs/dbt-core/pull/4509))
+- Clean up test deprecation warnings ([#3988](https://github.com/dbt-labs/dbt-core/issue/3988), [#4556](https://github.com/dbt-labs/dbt-core/pull/4556))
+- Use mashumaro for serialization in event logging ([#4504](https://github.com/dbt-labs/dbt-core/issues/4504), [#4505](https://github.com/dbt-labs/dbt-core/pull/4505))
+- Drop support for Python 3.7.0 + 3.7.1 ([#4584](https://github.com/dbt-labs/dbt-core/issues/4584), [#4585](https://github.com/dbt-labs/dbt-core/pull/4585), [#4643](https://github.com/dbt-labs/dbt-core/pull/4643))
+
+Contributors:
+- [@NiallRees](https://github.com/NiallRees) ([#4447](https://github.com/dbt-labs/dbt-core/pull/4447))
+
+## dbt-core 1.0.2 (TBD)
+
+### Fixes
+- Projects created using `dbt init` now have the correct `seeds` directory created (instead of `data`) ([#4588](https://github.com/dbt-labs/dbt-core/issues/4588), [#4599](https://github.com/dbt-labs/dbt-core/pull/4589))
+- Don't require a profile for dbt deps and clean commands ([#4554](https://github.com/dbt-labs/dbt-core/issues/4554), [#4610](https://github.com/dbt-labs/dbt-core/pull/4610))
+- Select modified.body works correctly when new model added([#4570](https://github.com/dbt-labs/dbt-core/issues/4570), [#4631](https://github.com/dbt-labs/dbt-core/pull/4631))
+- Fix bug in retry logic for bad response from hub and when there is a bad git tarball download. ([#4577](https://github.com/dbt-labs/dbt-core/issues/4577), [#4579](https://github.com/dbt-labs/dbt-core/issues/4579), [#4609](https://github.com/dbt-labs/dbt-core/pull/4609)) 
+- Restore previous log level (DEBUG) when a test depends on a disabled resource. Still WARN if the resource is missing ([#4594](https://github.com/dbt-labs/dbt-core/issues/4594), [#4647](https://github.com/dbt-labs/dbt-core/pull/4647))
+
+## dbt-core 1.0.1 (January 03, 2022)
+
+* [@amirkdv](https://github.com/amirkdv) ([#4536](https://github.com/dbt-labs/dbt-core/pull/4536))
+
+
+## dbt-core 1.0.1rc1 (December 20, 2021)
+
+### Fixes
+- Fix wrong url in the dbt docs overview homepage ([#4442](https://github.com/dbt-labs/dbt-core/pull/4442))
+- Fix redefined status param of SQLQueryStatus to typecheck the string which passes on `._message` value of `AdapterResponse` or the `str` value sent by adapter plugin.  ([#4463](https://github.com/dbt-labs/dbt-core/pull/4463#issuecomment-990174166))
+- Fix `DepsStartPackageInstall` event to use package name instead of version number. ([#4482](https://github.com/dbt-labs/dbt-core/pull/4482))
+- Reimplement log message to use adapter name instead of the object method. ([#4501](https://github.com/dbt-labs/dbt-core/pull/4501))
+- Issue better error message for incompatible schemas ([#4470](https://github.com/dbt-labs/dbt-core/pull/4442), [#4497](https://github.com/dbt-labs/dbt-core/pull/4497))
+- Remove secrets from error related to packages. ([#4507](https://github.com/dbt-labs/dbt-core/pull/4507))
+- Prevent coercion of boolean values (`True`, `False`) to numeric values (`0`, `1`) in query results ([#4511](https://github.com/dbt-labs/dbt-core/issues/4511), [#4512](https://github.com/dbt-labs/dbt-core/pull/4512))
+- Fix error with an env_var in a project hook ([#4523](https://github.com/dbt-labs/dbt-core/issues/4523), [#4524](https://github.com/dbt-labs/dbt-core/pull/4524))
+- Add additional windows compat logic for colored log output. ([#4443](https://github.com/dbt-labs/dbt-core/issues/4443))
+
+###  Docs
+- Fix missing data on exposures in docs ([#4467](https://github.com/dbt-labs/dbt-core/issues/4467))
+
+Contributors:
+- [remoyson](https://github.com/remoyson) ([#4442](https://github.com/dbt-labs/dbt-core/pull/4442))
+
+## dbt-core 1.0.0 (December 3, 2021)
+
+### Fixes
+- Configure the CLI logger destination to use stdout instead of stderr ([#4368](https://github.com/dbt-labs/dbt-core/pull/4368))
+- Make the size of `EVENT_HISTORY` configurable, via `EVENT_BUFFER_SIZE` global config ([#4411](https://github.com/dbt-labs/dbt-core/pull/4411), [#4416](https://github.com/dbt-labs/dbt-core/pull/4416))
+- Change type of `log_format` in `profiles.yml` user config to be string, not boolean ([#4394](https://github.com/dbt-labs/dbt-core/pull/4394))
+
+### Under the hood
+- Only log cache events if `LOG_CACHE_EVENTS` is enabled, and disable by default. This restores previous behavior ([#4369](https://github.com/dbt-labs/dbt-core/pull/4369))
+- Move event codes to be a top-level attribute of JSON-formatted logs, rather than nested in `data` ([#4381](https://github.com/dbt-labs/dbt-core/pull/4381))
+- Fix failing integration test on Windows ([#4380](https://github.com/dbt-labs/dbt-core/pull/4380))
+- Clean up warning messages for `clean` + `deps` ([#4366](https://github.com/dbt-labs/dbt-core/pull/4366))
+- Use RFC3339 timestamps for log messages ([#4384](https://github.com/dbt-labs/dbt-core/pull/4384))
+- Different text output for console (info) and file (debug) logs ([#4379](https://github.com/dbt-labs/dbt-core/pull/4379), [#4418](https://github.com/dbt-labs/dbt-core/pull/4418))
+- Remove unused events. More structured `ConcurrencyLine`. Replace `\n` message starts/ends with `EmptyLine` events, and exclude `EmptyLine` from JSON-formatted output ([#4388](https://github.com/dbt-labs/dbt-core/pull/4388))
+- Update `events` module README ([#4395](https://github.com/dbt-labs/dbt-core/pull/4395))
+- Rework approach to JSON serialization for events with non-standard properties ([#4396](https://github.com/dbt-labs/dbt-core/pull/4396))
+- Update legacy logger file name to `dbt.log.legacy` ([#4402](https://github.com/dbt-labs/dbt-core/pull/4402))
+- Rollover `dbt.log` at 10 MB, and keep up to 5 backups, restoring previous behavior ([#4405](https://github.com/dbt-labs/dbt-core/pull/4405))
+- Use reference keys instead of full relation objects in cache events ([#4410](https://github.com/dbt-labs/dbt-core/pull/4410))
+- Add `node_type` contextual info to more events ([#4378](https://github.com/dbt-labs/dbt-core/pull/4378))
+- Make `materialized` config optional in `node_type` ([#4417](https://github.com/dbt-labs/dbt-core/pull/4417))
+- Stringify exception in `GenericExceptionOnRun` to support JSON serialization ([#4424](https://github.com/dbt-labs/dbt-core/pull/4424))
+- Add "interop" tests for machine consumption of structured log output ([#4327](https://github.com/dbt-labs/dbt-core/pull/4327))
+- Relax version specifier for `dbt-extractor` to `~=0.4.0`, to support compiled wheels for additional architectures when available ([#4427](https://github.com/dbt-labs/dbt-core/pull/4427))
 
 ## dbt-core 1.0.0rc3 (November 30, 2021)
 
@@ -176,7 +251,7 @@ Contributors:
 - [@laxjesse](https://github.com/laxjesse) ([#4019](https://github.com/dbt-labs/dbt-core/pull/4019))
 - [@gitznik](https://github.com/Gitznik) ([#4124](https://github.com/dbt-labs/dbt-core/pull/4124))
 
-## dbt 0.21.1 (Release TBD)
+## dbt 0.21.1 (November 29, 2021)
 
 ### Fixes
 - Add `get_where_subquery` to test macro namespace, fixing custom generic tests that rely on introspecting the `model` arg at parse time ([#4195](https://github.com/dbt-labs/dbt/issues/4195), [#4197](https://github.com/dbt-labs/dbt/pull/4197))
@@ -320,7 +395,7 @@ Contributors:
 - [@jmriego](https://github.com/jmriego) ([#3526](https://github.com/dbt-labs/dbt-core/pull/3526))
 - [@danielefrigo](https://github.com/danielefrigo) ([#3547](https://github.com/dbt-labs/dbt-core/pull/3547))
 
-## dbt 0.20.2 (Release TBD)
+## dbt 0.20.2 (September 07, 2021)
 
 ### Under the hood
 

CONTRIBUTING.md

@@ -10,7 +10,7 @@
 
 ## About this document
 
-This document is a guide intended for folks interested in contributing to `dbt`. Below, we document the process by which members of the community should create issues and submit pull requests (PRs) in this repository. It is not intended as a guide for using `dbt`, and it assumes a certain level of familiarity with Python concepts such as virtualenvs, `pip`, python modules, filesystems, and so on. This guide assumes you are using macOS or Linux and are comfortable with the command line.
+This document is a guide intended for folks interested in contributing to `dbt-core`. Below, we document the process by which members of the community should create issues and submit pull requests (PRs) in this repository. It is not intended as a guide for using `dbt-core`, and it assumes a certain level of familiarity with Python concepts such as virtualenvs, `pip`, python modules, filesystems, and so on. This guide assumes you are using macOS or Linux and are comfortable with the command line.
 
 If you're new to python development or contributing to open-source software, we encourage you to read this document from start to finish. If you get stuck, drop us a line in the `#dbt-core-development` channel on [slack](https://community.getdbt.com).
 
@@ -20,101 +20,101 @@ If you have an issue or code change suggestion related to a specific database [a
 
 ### Signing the CLA
 
-Please note that all contributors to `dbt` must sign the [Contributor License Agreement](https://docs.getdbt.com/docs/contributor-license-agreements) to have their Pull Request merged into the `dbt` codebase. If you are unable to sign the CLA, then the `dbt` maintainers will unfortunately be unable to merge your Pull Request. You are, however, welcome to open issues and comment on existing ones.
+Please note that all contributors to `dbt-core` must sign the [Contributor License Agreement](https://docs.getdbt.com/docs/contributor-license-agreements) to have their Pull Request merged into the `dbt-core` codebase. If you are unable to sign the CLA, then the `dbt-core` maintainers will unfortunately be unable to merge your Pull Request. You are, however, welcome to open issues and comment on existing ones.
 
 ## Proposing a change
 
-`dbt` is Apache 2.0-licensed open source software. `dbt` is what it is today because community members like you have opened issues, provided feedback, and contributed to the knowledge loop for the entire communtiy. Whether you are a seasoned open source contributor or a first-time committer, we welcome and encourage you to contribute code, documentation, ideas, or problem statements to this project.
+`dbt-core` is Apache 2.0-licensed open source software. `dbt-core` is what it is today because community members like you have opened issues, provided feedback, and contributed to the knowledge loop for the entire communtiy. Whether you are a seasoned open source contributor or a first-time committer, we welcome and encourage you to contribute code, documentation, ideas, or problem statements to this project.
 
 ### Defining the problem
 
-If you have an idea for a new feature or if you've discovered a bug in `dbt`, the first step is to open an issue. Please check the list of [open issues](https://github.com/dbt-labs/dbt-core/issues) before creating a new one. If you find a relevant issue, please add a comment to the open issue instead of creating a new one. There are hundreds of open issues in this repository and it can be hard to know where to look for a relevant open issue. **The `dbt` maintainers are always happy to point contributors in the right direction**, so please err on the side of documenting your idea in a new issue if you are unsure where a problem statement belongs.
+If you have an idea for a new feature or if you've discovered a bug in `dbt-core`, the first step is to open an issue. Please check the list of [open issues](https://github.com/dbt-labs/dbt-core/issues) before creating a new one. If you find a relevant issue, please add a comment to the open issue instead of creating a new one. There are hundreds of open issues in this repository and it can be hard to know where to look for a relevant open issue. **The `dbt-core` maintainers are always happy to point contributors in the right direction**, so please err on the side of documenting your idea in a new issue if you are unsure where a problem statement belongs.
 
 > **Note:** All community-contributed Pull Requests _must_ be associated with an open issue. If you submit a Pull Request that does not pertain to an open issue, you will be asked to create an issue describing the problem before the Pull Request can be reviewed.
 
 ### Discussing the idea
 
-After you open an issue, a `dbt` maintainer will follow up by commenting on your issue (usually within 1-3 days) to explore your idea further and advise on how to implement the suggested changes. In many cases, community members will chime in with their own thoughts on the problem statement. If you as the issue creator are interested in submitting a Pull Request to address the issue, you should indicate this in the body of the issue. The `dbt` maintainers are _always_ happy to help contributors with the implementation of fixes and features, so please also indicate if there's anything you're unsure about or could use guidance around in the issue.
+After you open an issue, a `dbt-core` maintainer will follow up by commenting on your issue (usually within 1-3 days) to explore your idea further and advise on how to implement the suggested changes. In many cases, community members will chime in with their own thoughts on the problem statement. If you as the issue creator are interested in submitting a Pull Request to address the issue, you should indicate this in the body of the issue. The `dbt-core` maintainers are _always_ happy to help contributors with the implementation of fixes and features, so please also indicate if there's anything you're unsure about or could use guidance around in the issue.
 
 ### Submitting a change
 
-If an issue is appropriately well scoped and describes a beneficial change to the `dbt` codebase, then anyone may submit a Pull Request to implement the functionality described in the issue. See the sections below on how to do this.
+If an issue is appropriately well scoped and describes a beneficial change to the `dbt-core` codebase, then anyone may submit a Pull Request to implement the functionality described in the issue. See the sections below on how to do this.
 
-The `dbt` maintainers will add a `good first issue` label if an issue is suitable for a first-time contributor. This label often means that the required code change is small, limited to one database adapter, or a net-new addition that does not impact existing functionality. You can see the list of currently open issues on the [Contribute](https://github.com/dbt-labs/dbt-core/contribute) page.
+The `dbt-core` maintainers will add a `good first issue` label if an issue is suitable for a first-time contributor. This label often means that the required code change is small, limited to one database adapter, or a net-new addition that does not impact existing functionality. You can see the list of currently open issues on the [Contribute](https://github.com/dbt-labs/dbt-core/contribute) page.
 
 Here's a good workflow:
 - Comment on the open issue, expressing your interest in contributing the required code change
 - Outline your planned implementation. If you want help getting started, ask!
-- Follow the steps outlined below to develop locally. Once you have opened a PR, one of the `dbt` maintainers will work with you to review your code.
-- Add a test! Tests are crucial for both fixes and new features alike. We want to make sure that code works as intended, and that it avoids any bugs  previously encountered. Currently, the best resource for understanding `dbt`'s [unit](test/unit) and [integration](test/integration) tests is the tests themselves. One of the maintainers can help by pointing out relevant examples.
+- Follow the steps outlined below to develop locally. Once you have opened a PR, one of the `dbt-core` maintainers will work with you to review your code.
+- Add a test! Tests are crucial for both fixes and new features alike. We want to make sure that code works as intended, and that it avoids any bugs  previously encountered. Currently, the best resource for understanding `dbt-core`'s [unit](test/unit) and [integration](test/integration) tests is the tests themselves. One of the maintainers can help by pointing out relevant examples.
 
-In some cases, the right resolution to an open issue might be tangential to the `dbt` codebase. The right path forward might be a documentation update or a change that can be made in user-space. In other cases, the issue might describe functionality that the `dbt` maintainers are unwilling or unable to incorporate into the `dbt` codebase. When it is determined that an open issue describes functionality that will not translate to a code change in the `dbt` repository, the issue will be tagged with the `wontfix` label (see below) and closed.
+In some cases, the right resolution to an open issue might be tangential to the `dbt-core` codebase. The right path forward might be a documentation update or a change that can be made in user-space. In other cases, the issue might describe functionality that the `dbt-core` maintainers are unwilling or unable to incorporate into the `dbt-core` codebase. When it is determined that an open issue describes functionality that will not translate to a code change in the `dbt-core` repository, the issue will be tagged with the `wontfix` label (see below) and closed.
 
 ### Using issue labels
 
-The `dbt` maintainers use labels to categorize open issues. Some labels indicate the databases impacted by the issue, while others describe the domain in the `dbt` codebase germane to the discussion. While most of these labels are self-explanatory (eg. `snowflake` or `bigquery`), there are others that are worth describing.
+The `dbt-core` maintainers use labels to categorize open issues. Most labels describe the domain in the `dbt-core` codebase germane to the discussion.
 
 | tag | description |
 | --- | ----------- |
-| [triage](https://github.com/dbt-labs/dbt-core/labels/triage) | This is a new issue which has not yet been reviewed by a `dbt` maintainer. This label is removed when a maintainer reviews and responds to the issue. |
-| [bug](https://github.com/dbt-labs/dbt-core/labels/bug) | This issue represents a defect or regression in `dbt` |
-| [enhancement](https://github.com/dbt-labs/dbt-core/labels/enhancement) | This issue represents net-new functionality in `dbt` |
-| [good first issue](https://github.com/dbt-labs/dbt-core/labels/good%20first%20issue) | This issue does not require deep knowledge of the `dbt` codebase to implement. This issue is appropriate for a first-time contributor. |
+| [triage](https://github.com/dbt-labs/dbt-core/labels/triage) | This is a new issue which has not yet been reviewed by a `dbt-core` maintainer. This label is removed when a maintainer reviews and responds to the issue. |
+| [bug](https://github.com/dbt-labs/dbt-core/labels/bug) | This issue represents a defect or regression in `dbt-core` |
+| [enhancement](https://github.com/dbt-labs/dbt-core/labels/enhancement) | This issue represents net-new functionality in `dbt-core` |
+| [good first issue](https://github.com/dbt-labs/dbt-core/labels/good%20first%20issue) | This issue does not require deep knowledge of the `dbt-core` codebase to implement. This issue is appropriate for a first-time contributor. |
 | [help wanted](https://github.com/dbt-labs/dbt-core/labels/help%20wanted) / [discussion](https://github.com/dbt-labs/dbt-core/labels/discussion) | Conversation around this issue in ongoing, and there isn't yet a clear path forward. Input from community members is most welcome. |
-| [duplicate](https://github.com/dbt-labs/dbt-core/issues/duplicate) | This issue is functionally identical to another open issue. The `dbt` maintainers will close this issue and encourage community members to focus conversation on the other one. |
-| [snoozed](https://github.com/dbt-labs/dbt-core/labels/snoozed) | This issue describes a good idea, but one which will probably not be addressed in a six-month time horizon. The `dbt` maintainers will revist these issues periodically and re-prioritize them accordingly. |
-| [stale](https://github.com/dbt-labs/dbt-core/labels/stale) | This is an old issue which has not recently been updated. Stale issues will periodically be closed by `dbt` maintainers, but they can be re-opened if the discussion is restarted. |
-| [wontfix](https://github.com/dbt-labs/dbt-core/labels/wontfix) | This issue does not require a code change in the `dbt` repository, or the maintainers are unwilling/unable to merge a Pull Request which implements the behavior described in the issue. |
+| [duplicate](https://github.com/dbt-labs/dbt-core/issues/duplicate) | This issue is functionally identical to another open issue. The `dbt-core` maintainers will close this issue and encourage community members to focus conversation on the other one. |
+| [snoozed](https://github.com/dbt-labs/dbt-core/labels/snoozed) | This issue describes a good idea, but one which will probably not be addressed in a six-month time horizon. The `dbt-core` maintainers will revist these issues periodically and re-prioritize them accordingly. |
+| [stale](https://github.com/dbt-labs/dbt-core/labels/stale) | This is an old issue which has not recently been updated. Stale issues will periodically be closed by `dbt-core` maintainers, but they can be re-opened if the discussion is restarted. |
+| [wontfix](https://github.com/dbt-labs/dbt-core/labels/wontfix) | This issue does not require a code change in the `dbt-core` repository, or the maintainers are unwilling/unable to merge a Pull Request which implements the behavior described in the issue. |
 
 #### Branching Strategy
 
-`dbt` has three types of branches:
+`dbt-core` has three types of branches:
 
-- **Trunks** are where active development of the next release takes place. There is one trunk named `develop` at the time of writing this, and will be the default branch of the repository.
-- **Release Branches** track a specific, not yet complete release of `dbt`. Each minor version release has a corresponding release branch. For example, the `0.11.x` series of releases has a branch called `0.11.latest`. This allows us to release new patch versions under `0.11` without necessarily needing to pull them into the latest version of `dbt`.
+- **Trunks** are where active development of the next release takes place. There is one trunk named `main` at the time of writing this, and will be the default branch of the repository.
+- **Release Branches** track a specific, not yet complete release of `dbt-core`. Each minor version release has a corresponding release branch. For example, the `0.11.x` series of releases has a branch called `0.11.latest`. This allows us to release new patch versions under `0.11` without necessarily needing to pull them into the latest version of `dbt-core`.
 - **Feature Branches** track individual features and fixes. On completion they should be merged into the trunk branch or a specific release branch.
 
 ## Getting the code
 
 ### Installing git
 
-You will need `git` in order to download and modify the `dbt` source code. On macOS, the best way to download git is to just install [Xcode](https://developer.apple.com/support/xcode/).
+You will need `git` in order to download and modify the `dbt-core` source code. On macOS, the best way to download git is to just install [Xcode](https://developer.apple.com/support/xcode/).
 
 ### External contributors
 
-If you are not a member of the `dbt-labs` GitHub organization, you can contribute to `dbt` by forking the `dbt` repository. For a detailed overview on forking, check out the [GitHub docs on forking](https://help.github.com/en/articles/fork-a-repo). In short, you will need to:
+If you are not a member of the `dbt-labs` GitHub organization, you can contribute to `dbt-core` by forking the `dbt-core` repository. For a detailed overview on forking, check out the [GitHub docs on forking](https://help.github.com/en/articles/fork-a-repo). In short, you will need to:
 
-1. fork the `dbt` repository
+1. fork the `dbt-core` repository
 2. clone your fork locally
 3. check out a new branch for your proposed changes
 4. push changes to your fork
 5. open a pull request against `dbt-labs/dbt` from your forked repository
 
-### Core contributors
+### dbt Labs contributors
 
-If you are a member of the `dbt-labs` GitHub organization, you will have push access to the `dbt` repo. Rather than forking `dbt` to make your changes, just clone the repository, check out a new branch, and push directly to that branch.
+If you are a member of the `dbt-labs` GitHub organization, you will have push access to the `dbt-core` repo. Rather than forking `dbt-core` to make your changes, just clone the repository, check out a new branch, and push directly to that branch.
 
 ## Setting up an environment
 
-There are some tools that will be helpful to you in developing locally. While this is the list relevant for `dbt` development, many of these tools are used commonly across open-source python projects.
+There are some tools that will be helpful to you in developing locally. While this is the list relevant for `dbt-core` development, many of these tools are used commonly across open-source python projects.
 
 ### Tools
 
-A short list of tools used in `dbt` testing that will be helpful to your understanding:
+A short list of tools used in `dbt-core` testing that will be helpful to your understanding:
 
-- [`tox`](https://tox.readthedocs.io/en/latest/) to manage virtualenvs across python versions. We currently target the latest patch releases for Python 3.6, Python 3.7, Python 3.8, and Python 3.9
+- [`tox`](https://tox.readthedocs.io/en/latest/) to manage virtualenvs across python versions. We currently target the latest patch releases for Python 3.7, Python 3.8, and Python 3.9
 - [`pytest`](https://docs.pytest.org/en/latest/) to discover/run tests
 - [`make`](https://users.cs.duke.edu/~ola/courses/programming/Makefiles/Makefiles.html) - but don't worry too much, nobody _really_ understands how make works and our Makefile is super simple
 - [`flake8`](https://flake8.pycqa.org/en/latest/) for code linting
 - [`mypy`](https://mypy.readthedocs.io/en/stable/) for static type checking
 - [Github Actions](https://github.com/features/actions)
 
-A deep understanding of these tools in not required to effectively contribute to `dbt`, but we recommend checking out the attached documentation if you're interested in learning more about them.
+A deep understanding of these tools in not required to effectively contribute to `dbt-core`, but we recommend checking out the attached documentation if you're interested in learning more about them.
 
 #### virtual environments
 
-We strongly recommend using virtual environments when developing code in `dbt`. We recommend creating this virtualenv
-in the root of the `dbt` repository. To create a new virtualenv, run:
+We strongly recommend using virtual environments when developing code in `dbt-core`. We recommend creating this virtualenv
+in the root of the `dbt-core` repository. To create a new virtualenv, run:
 ```sh
 python3 -m venv env
 source env/bin/activate
@@ -135,11 +135,11 @@ For testing, and later in the examples in this document, you may want to have `p
 brew install postgresql
 ```
 
-## Running `dbt` in development
+## Running `dbt-core` in development
 
 ### Installation
 
-First make sure that you set up your `virtualenv` as described in [Setting up an environment](#setting-up-an-environment).  Also ensure you have the latest version of pip installed with `pip install --upgrade pip`. Next, install `dbt` (and its dependencies) with:
+First make sure that you set up your `virtualenv` as described in [Setting up an environment](#setting-up-an-environment).  Also ensure you have the latest version of pip installed with `pip install --upgrade pip`. Next, install `dbt-core` (and its dependencies) with:
 
 ```sh
 make dev
@@ -147,23 +147,24 @@ make dev
 pip install -r dev-requirements.txt -r editable-requirements.txt
 ```
 
-When `dbt` is installed this way, any changes you make to the `dbt` source code will be reflected immediately in your next `dbt` run.
+When `dbt-core` is installed this way, any changes you make to the `dbt-core` source code will be reflected immediately in your next `dbt-core` run.
 
-### Running `dbt`
 
-With your virtualenv activated, the `dbt` script should point back to the source code you've cloned on your machine. You can verify this by running `which dbt`. This command should show you a path to an executable in your virtualenv.
+### Running `dbt-core`
+
+With your virtualenv activated, the `dbt-core` script should point back to the source code you've cloned on your machine. You can verify this by running `which dbt`. This command should show you a path to an executable in your virtualenv.
 
 Configure your [profile](https://docs.getdbt.com/docs/configure-your-profile) as necessary to connect to your target databases. It may be a good idea to add a new profile pointing to a local postgres instance, or a specific test sandbox within your data warehouse if appropriate.
 
 ## Testing
 
-Getting the `dbt` integration tests set up in your local environment will be very helpful as you start to make changes to your local version of `dbt`. The section that follows outlines some helpful tips for setting up the test environment.
+Getting the `dbt-core` integration tests set up in your local environment will be very helpful as you start to make changes to your local version of `dbt-core`. The section that follows outlines some helpful tips for setting up the test environment.
 
-Although `dbt` works with a number of different databases, you won't need to supply credentials for every one of these databases in your test environment. Instead you can test all dbt-core code changes with Python and Postgres.
+Although `dbt-core` works with a number of different databases, you won't need to supply credentials for every one of these databases in your test environment. Instead you can test all dbt-core code changes with Python and Postgres.
 
 ### Initial setup
 
-We recommend starting with `dbt`'s Postgres tests. These tests cover most of the functionality in `dbt`, are the fastest to run, and are the easiest to set up. To run the Postgres integration tests, you'll have to do one extra step of setting up the test database:
+We recommend starting with `dbt-core`'s Postgres tests. These tests cover most of the functionality in `dbt-core`, are the fastest to run, and are the easiest to set up. To run the Postgres integration tests, you'll have to do one extra step of setting up the test database:
 
 ```sh
 make setup-db
@@ -174,15 +175,6 @@ docker-compose up -d database
 PGHOST=localhost PGUSER=root PGPASSWORD=password PGDATABASE=postgres bash test/setup_db.sh
 ```
 
-`dbt` uses test credentials specified in a `test.env` file in the root of the repository for non-Postgres databases. This `test.env` file is git-ignored, but please be _extra_ careful to never check in credentials or other sensitive information when developing against `dbt`. To create your `test.env` file, copy the provided sample file, then supply your relevant credentials. This step is only required to use non-Postgres databases.
-
-```
-cp test.env.sample test.env
-$EDITOR test.env
-```
-
-> In general, it's most important to have successful unit and Postgres tests. Once you open a PR, `dbt` will automatically run integration tests for the other three core database adapters. Of course, if you are a BigQuery user, contributing a BigQuery-only feature, it's important to run BigQuery tests as well.
-
 ### Test commands
 
 There are a few methods for running tests locally.
@@ -208,9 +200,9 @@ suites.
 
 [`tox`](https://tox.readthedocs.io/en/latest/) takes care of managing virtualenvs and install dependencies in order to run
 tests. You can also run tests in parallel, for example, you can run unit tests
-for Python 3.6, Python 3.7, Python 3.8, `flake8` checks, and `mypy` checks in
+for Python 3.7, Python 3.8, Python 3.9, `flake8` checks, and `mypy` checks in
 parallel with `tox -p`. Also, you can run unit tests for specific python versions
-with `tox -e py36`. The configuration for these tests in located in `tox.ini`.
+with `tox -e py37`. The configuration for these tests in located in `tox.ini`.
 
 #### `pytest`
 
@@ -230,6 +222,8 @@ python -m pytest test/unit/test_graph.py::GraphTest::test__dependency_list
 
 dbt Labs provides a CI environment to test changes to specific adapters, and periodic maintenance checks of `dbt-core` through Github Actions. For example, if you submit a pull request to the `dbt-redshift` repo, GitHub will trigger automated code checks and tests against Redshift.
 
-A `dbt` maintainer will review your PR. They may suggest code revision for style or clarity, or request that you add unit or integration test(s). These are good things! We believe that, with a little bit of help, anyone can contribute high-quality code.
+A `dbt-core` maintainer will review your PR. They may suggest code revision for style or clarity, or request that you add unit or integration test(s). These are good things! We believe that, with a little bit of help, anyone can contribute high-quality code.
+- First time contributors should note code checks + unit tests require a maintainer to approve.
 
-Once all tests are passing and your PR has been approved, a `dbt` maintainer will merge your changes into the active development branch. And that's it! Happy developing :tada:
+
+Once all tests are passing and your PR has been approved, a `dbt-core` maintainer will merge your changes into the active development branch. And that's it! Happy developing :tada:

Dockerfile.test

@@ -1,3 +1,8 @@
+##
+#  This dockerfile is used for local development and adapter testing only.
+#  See `/docker` for a generic and production-ready docker file
+##
+
 FROM ubuntu:20.04
 
 ENV DEBIAN_FRONTEND noninteractive

core/dbt/README.md

@@ -0,0 +1,52 @@
+# core/dbt directory README
+
+## The following are individual files in this directory.
+
+### deprecations.py
+
+### flags.py		
+
+### main.py			
+
+### tracking.py		
+
+### version.py
+
+### lib.py
+
+### node_types.py
+
+### helper_types.py		
+
+### links.py		
+
+### semver.py		
+
+### ui.py
+
+### compilation.py		
+
+### dataclass_schema.py	
+
+### exceptions.py		
+
+### hooks.py		
+
+### logger.py		
+
+### profiler.py		
+
+### utils.py
+
+
+## The subdirectories will be documented in a README in the subdirectory
+* config
+* include
+* adapters
+* context
+* deps
+* graph
+* task
+* clients
+* events
+

core/dbt/adapters/README.md

@@ -0,0 +1 @@
+# Adapters README

core/dbt/adapters/base/impl.py

@@ -39,7 +39,7 @@ from dbt.adapters.base.relation import (
     ComponentName, BaseRelation, InformationSchema, SchemaSearchMap
 )
 from dbt.adapters.base import Column as BaseColumn
-from dbt.adapters.cache import RelationsCache
+from dbt.adapters.cache import RelationsCache, _make_key
 
 
 SeedModel = Union[ParsedSeedNode, CompiledSeedNode]
@@ -291,7 +291,7 @@ class BaseAdapter(metaclass=AdapterMeta):
         if (database, schema) not in self.cache:
             fire_event(
                 CacheMiss(
-                    conn_name=self.nice_connection_name,
+                    conn_name=self.nice_connection_name(),
                     database=database,
                     schema=schema
                 )
@@ -676,7 +676,11 @@ class BaseAdapter(metaclass=AdapterMeta):
         relations = self.list_relations_without_caching(
             schema_relation
         )
-        fire_event(ListRelations(database=database, schema=schema, relations=relations))
+        fire_event(ListRelations(
+            database=database,
+            schema=schema,
+            relations=[_make_key(x) for x in relations]
+        ))
 
         return relations
 

core/dbt/adapters/cache.py

@@ -1,8 +1,8 @@
 import threading
-from collections import namedtuple
 from copy import deepcopy
 from typing import Any, Dict, Iterable, List, Optional, Set, Tuple
 
+from dbt.adapters.reference_keys import _make_key, _ReferenceKey
 import dbt.exceptions
 from dbt.events.functions import fire_event
 from dbt.events.types import (
@@ -21,18 +21,7 @@ from dbt.events.types import (
     UpdateReference
 )
 from dbt.utils import lowercase
-
-_ReferenceKey = namedtuple('_ReferenceKey', 'database schema identifier')
-
-
-def _make_key(relation) -> _ReferenceKey:
-    """Make _ReferenceKeys with lowercase values for the cache so we don't have
-    to keep track of quoting
-    """
-    # databases and schemas can both be None
-    return _ReferenceKey(lowercase(relation.database),
-                         lowercase(relation.schema),
-                         lowercase(relation.identifier))
+from dbt.helper_types import Lazy
 
 
 def dot_separated(key: _ReferenceKey) -> str:
@@ -303,11 +292,12 @@ class RelationsCache:
         :raises InternalError: If either entry does not exist.
         """
         ref_key = _make_key(referenced)
+        dep_key = _make_key(dependent)
         if (ref_key.database, ref_key.schema) not in self:
             # if we have not cached the referenced schema at all, we must be
             # referring to a table outside our control. There's no need to make
             # a link - we will never drop the referenced relation during a run.
-            fire_event(UncachedRelation(dep_key=dependent, ref_key=ref_key))
+            fire_event(UncachedRelation(dep_key=dep_key, ref_key=ref_key))
             return
         if ref_key not in self.relations:
             # Insert a dummy "external" relation.
@@ -315,8 +305,6 @@ class RelationsCache:
                 type=referenced.External
             )
             self.add(referenced)
-
-        dep_key = _make_key(dependent)
         if dep_key not in self.relations:
             # Insert a dummy "external" relation.
             dependent = dependent.replace(
@@ -334,12 +322,12 @@ class RelationsCache:
         :param BaseRelation relation: The underlying relation.
         """
         cached = _CachedRelation(relation)
-        fire_event(AddRelation(relation=cached))
-        fire_event(DumpBeforeAddGraph(dump=self.dump_graph()))
+        fire_event(AddRelation(relation=_make_key(cached)))
+        fire_event(DumpBeforeAddGraph(dump=Lazy.defer(lambda: self.dump_graph())))
 
         with self.lock:
             self._setdefault(cached)
-        fire_event(DumpAfterAddGraph(dump=self.dump_graph()))
+        fire_event(DumpAfterAddGraph(dump=Lazy.defer(lambda: self.dump_graph())))
 
     def _remove_refs(self, keys):
         """Removes all references to all entries in keys. This does not
@@ -354,17 +342,17 @@ class RelationsCache:
         for cached in self.relations.values():
             cached.release_references(keys)
 
-    def _drop_cascade_relation(self, dropped):
+    def _drop_cascade_relation(self, dropped_key):
         """Drop the given relation and cascade it appropriately to all
         dependent relations.
 
         :param _CachedRelation dropped: An existing _CachedRelation to drop.
         """
-        if dropped not in self.relations:
-            fire_event(DropMissingRelation(relation=dropped))
+        if dropped_key not in self.relations:
+            fire_event(DropMissingRelation(relation=dropped_key))
             return
-        consequences = self.relations[dropped].collect_consequences()
-        fire_event(DropCascade(dropped=dropped, consequences=consequences))
+        consequences = self.relations[dropped_key].collect_consequences()
+        fire_event(DropCascade(dropped=dropped_key, consequences=consequences))
         self._remove_refs(consequences)
 
     def drop(self, relation):
@@ -378,10 +366,10 @@ class RelationsCache:
         :param str schema: The schema of the relation to drop.
         :param str identifier: The identifier of the relation to drop.
         """
-        dropped = _make_key(relation)
-        fire_event(DropRelation(dropped=dropped))
+        dropped_key = _make_key(relation)
+        fire_event(DropRelation(dropped=dropped_key))
         with self.lock:
-            self._drop_cascade_relation(dropped)
+            self._drop_cascade_relation(dropped_key)
 
     def _rename_relation(self, old_key, new_relation):
         """Rename a relation named old_key to new_key, updating references.
@@ -453,7 +441,7 @@ class RelationsCache:
         new_key = _make_key(new)
         fire_event(RenameSchema(old_key=old_key, new_key=new_key))
 
-        fire_event(DumpBeforeRenameSchema(dump=self.dump_graph()))
+        fire_event(DumpBeforeRenameSchema(dump=Lazy.defer(lambda: self.dump_graph())))
 
         with self.lock:
             if self._check_rename_constraints(old_key, new_key):
@@ -461,7 +449,7 @@ class RelationsCache:
             else:
                 self._setdefault(_CachedRelation(new))
 
-        fire_event(DumpAfterRenameSchema(dump=self.dump_graph()))
+        fire_event(DumpAfterRenameSchema(dump=Lazy.defer(lambda: self.dump_graph())))
 
     def get_relations(
         self, database: Optional[str], schema: Optional[str]

core/dbt/adapters/reference_keys.py

@@ -0,0 +1,24 @@
+# this module exists to resolve circular imports with the events module
+
+from collections import namedtuple
+from typing import Optional
+
+
+_ReferenceKey = namedtuple('_ReferenceKey', 'database schema identifier')
+
+
+def lowercase(value: Optional[str]) -> Optional[str]:
+    if value is None:
+        return None
+    else:
+        return value.lower()
+
+
+def _make_key(relation) -> _ReferenceKey:
+    """Make _ReferenceKeys with lowercase values for the cache so we don't have
+    to keep track of quoting
+    """
+    # databases and schemas can both be None
+    return _ReferenceKey(lowercase(relation.database),
+                         lowercase(relation.schema),
+                         lowercase(relation.identifier))

core/dbt/adapters/sql/connections.py

@@ -75,7 +75,8 @@ class SQLConnectionManager(BaseConnectionManager):
 
             fire_event(
                 SQLQueryStatus(
-                    status=str(self.get_response(cursor)), elapsed=round((time.time() - pre), 2)
+                    status=str(self.get_response(cursor)),
+                    elapsed=round((time.time() - pre), 2)
                 )
             )
 

core/dbt/adapters/sql/impl.py

@@ -5,6 +5,7 @@ import dbt.clients.agate_helper
 from dbt.contracts.connection import Connection
 import dbt.exceptions
 from dbt.adapters.base import BaseAdapter, available
+from dbt.adapters.cache import _make_key
 from dbt.adapters.sql import SQLConnectionManager
 from dbt.events.functions import fire_event
 from dbt.events.types import ColTypeChange, SchemaCreation, SchemaDrop
@@ -122,7 +123,7 @@ class SQLAdapter(BaseAdapter):
                     ColTypeChange(
                         orig_type=target_column.data_type,
                         new_type=new_type,
-                        table=current,
+                        table=_make_key(current),
                     )
                 )
 
@@ -182,7 +183,7 @@ class SQLAdapter(BaseAdapter):
 
     def create_schema(self, relation: BaseRelation) -> None:
         relation = relation.without_identifier()
-        fire_event(SchemaCreation(relation=relation))
+        fire_event(SchemaCreation(relation=_make_key(relation)))
         kwargs = {
             'relation': relation,
         }
@@ -193,7 +194,7 @@ class SQLAdapter(BaseAdapter):
 
     def drop_schema(self, relation: BaseRelation) -> None:
         relation = relation.without_identifier()
-        fire_event(SchemaDrop(relation=relation))
+        fire_event(SchemaDrop(relation=_make_key(relation)))
         kwargs = {
             'relation': relation,
         }

core/dbt/clients/README.md

@@ -0,0 +1 @@
+# Clients README

core/dbt/clients/agate_helper.py

@@ -13,6 +13,18 @@ from dbt.exceptions import RuntimeException
 BOM = BOM_UTF8.decode('utf-8')  # '\ufeff'
 
 
+class Number(agate.data_types.Number):
+    # undo the change in https://github.com/wireservice/agate/pull/733
+    # i.e. do not cast True and False to numeric 1 and 0
+    def cast(self, d):
+        if type(d) == bool:
+            raise agate.exceptions.CastError(
+                'Do not cast True to 1 or False to 0.'
+            )
+        else:
+            return super().cast(d)
+
+
 class ISODateTime(agate.data_types.DateTime):
     def cast(self, d):
         # this is agate.data_types.DateTime.cast with the "clever" bits removed
@@ -41,7 +53,7 @@ def build_type_tester(
 ) -> agate.TypeTester:
 
     types = [
-        agate.data_types.Number(null_values=('null', '')),
+        Number(null_values=('null', '')),
         agate.data_types.Date(null_values=('null', ''),
                               date_format='%Y-%m-%d'),
         agate.data_types.DateTime(null_values=('null', ''),

core/dbt/clients/git.py

@@ -8,7 +8,10 @@ from dbt.events.types import (
     GitProgressUpdatingExistingDependency, GitProgressPullingNewDependency,
     GitNothingToDo, GitProgressUpdatedCheckoutRange, GitProgressCheckedOutAt
 )
-import dbt.exceptions
+from dbt.exceptions import (
+    CommandResultError, RuntimeException, bad_package_spec, raise_git_cloning_error,
+    raise_git_cloning_problem
+)
 from packaging import version
 
 
@@ -22,9 +25,9 @@ def _raise_git_cloning_error(repo, revision, error):
     if 'usage: git' in stderr:
         stderr = stderr.split('\nusage: git')[0]
     if re.match("fatal: destination path '(.+)' already exists", stderr):
-        raise error
+        raise_git_cloning_error(error)
 
-    dbt.exceptions.bad_package_spec(repo, revision, stderr)
+    bad_package_spec(repo, revision, stderr)
 
 
 def clone(repo, cwd, dirname=None, remove_git_dir=False, revision=None, subdirectory=None):
@@ -53,7 +56,7 @@ def clone(repo, cwd, dirname=None, remove_git_dir=False, revision=None, subdirec
         clone_cmd.append(dirname)
     try:
         result = run_cmd(cwd, clone_cmd, env={'LC_ALL': 'C'})
-    except dbt.exceptions.CommandResultError as exc:
+    except CommandResultError as exc:
         _raise_git_cloning_error(repo, revision, exc)
 
     if subdirectory:
@@ -61,7 +64,7 @@ def clone(repo, cwd, dirname=None, remove_git_dir=False, revision=None, subdirec
         clone_cmd_subdir = ['git', 'sparse-checkout', 'set', subdirectory]
         try:
             run_cmd(cwd_subdir, clone_cmd_subdir)
-        except dbt.exceptions.CommandResultError as exc:
+        except CommandResultError as exc:
             _raise_git_cloning_error(repo, revision, exc)
 
     if remove_git_dir:
@@ -105,9 +108,9 @@ def checkout(cwd, repo, revision=None):
         revision = 'HEAD'
     try:
         return _checkout(cwd, repo, revision)
-    except dbt.exceptions.CommandResultError as exc:
+    except CommandResultError as exc:
         stderr = exc.stderr.decode('utf-8').strip()
-    dbt.exceptions.bad_package_spec(repo, revision, stderr)
+    bad_package_spec(repo, revision, stderr)
 
 
 def get_current_sha(cwd):
@@ -131,14 +134,11 @@ def clone_and_checkout(repo, cwd, dirname=None, remove_git_dir=False,
             remove_git_dir=remove_git_dir,
             subdirectory=subdirectory,
         )
-    except dbt.exceptions.CommandResultError as exc:
+    except CommandResultError as exc:
         err = exc.stderr.decode('utf-8')
         exists = re.match("fatal: destination path '(.+)' already exists", err)
         if not exists:
-            print(
-                '\nSomething went wrong while cloning {}'.format(repo) +
-                '\nCheck the debug logs for more information')
-            raise
+            raise_git_cloning_problem(repo)
 
     directory = None
     start_sha = None
@@ -148,7 +148,7 @@ def clone_and_checkout(repo, cwd, dirname=None, remove_git_dir=False,
     else:
         matches = re.match("Cloning into '(.+)'", err.decode('utf-8'))
         if matches is None:
-            raise dbt.exceptions.RuntimeException(
+            raise RuntimeException(
                 f'Error cloning {repo} - never saw "Cloning into ..." from git'
             )
         directory = matches.group(1)

core/dbt/clients/registry.py

@@ -33,7 +33,12 @@ def _get(path, registry_base_url=None):
     resp = requests.get(url, timeout=30)
     fire_event(RegistryProgressGETResponse(url=url, resp_code=resp.status_code))
     resp.raise_for_status()
-    if resp is None:
+
+    # It is unexpected for the content of the response to be None so if it is, raising this error
+    # will cause this function to retry (if called within _get_with_retries) and hopefully get
+    # a response.  This seems to happen when there's an issue with the Hub.
+    # See https://github.com/dbt-labs/dbt-core/issues/4577
+    if resp.json() is None:
         raise requests.exceptions.ContentDecodingError(
             'Request error: The response is None', response=resp
         )

core/dbt/clients/system.py

@@ -485,7 +485,7 @@ def untar_package(
 ) -> None:
     tar_path = convert_path(tar_path)
     tar_dir_name = None
-    with tarfile.open(tar_path, 'r') as tarball:
+    with tarfile.open(tar_path, 'r:gz') as tarball:
         tarball.extractall(dest_dir)
         tar_dir_name = os.path.commonprefix(tarball.getnames())
     if rename_to:

core/dbt/compilation.py

@@ -3,6 +3,7 @@ from collections import defaultdict
 from typing import List, Dict, Any, Tuple, cast, Optional
 
 import networkx as nx  # type: ignore
+import pickle
 import sqlparse
 
 from dbt import flags
@@ -162,7 +163,8 @@ class Linker:
         for node_id in self.graph:
             data = manifest.expect(node_id).to_dict(omit_none=True)
             out_graph.add_node(node_id, **data)
-        nx.write_gpickle(out_graph, outfile)
+        with open(outfile, 'wb') as outfh:
+            pickle.dump(out_graph, outfh, protocol=pickle.HIGHEST_PROTOCOL)
 
 
 class Compiler:

core/dbt/config/README.md

@@ -0,0 +1 @@
+# Config README

core/dbt/config/project.py

@@ -45,7 +45,7 @@ INVALID_VERSION_ERROR = """\
 This version of dbt is not supported with the '{package}' package.
   Installed version of dbt: {installed}
   Required version of dbt for '{package}': {version_spec}
-Check the requirements for the '{package}' package, or run dbt again with \
+Check for a different version of the '{package}' package, or run dbt again with \
 --no-version-check
 """
 
@@ -54,7 +54,7 @@ IMPOSSIBLE_VERSION_ERROR = """\
 The package version requirement can never be satisfied for the '{package}
 package.
   Required versions of dbt for '{package}': {version_spec}
-Check the requirements for the '{package}' package, or run dbt again with \
+Check for a different version of the '{package}' package, or run dbt again with \
 --no-version-check
 """
 

core/dbt/config/runtime.py

@@ -1,7 +1,7 @@
 import itertools
 import os
 from copy import deepcopy
-from dataclasses import dataclass, fields
+from dataclasses import dataclass
 from pathlib import Path
 from typing import (
     Dict, Any, Optional, Mapping, Iterator, Iterable, Tuple, List, MutableSet,
@@ -13,20 +13,17 @@ from .project import Project
 from .renderer import DbtProjectYamlRenderer, ProfileRenderer
 from .utils import parse_cli_vars
 from dbt import flags
-from dbt import tracking
 from dbt.adapters.factory import get_relation_class_by_name, get_include_paths
 from dbt.helper_types import FQNPath, PathSet
+from dbt.config.profile import read_user_config
 from dbt.contracts.connection import AdapterRequiredConfig, Credentials
 from dbt.contracts.graph.manifest import ManifestMetadata
 from dbt.contracts.relation import ComponentName
-from dbt.events.types import ProfileLoadError, ProfileNotFound
-from dbt.events.functions import fire_event
 from dbt.ui import warning_tag
 
 from dbt.contracts.project import Configuration, UserConfig
 from dbt.exceptions import (
     RuntimeException,
-    DbtProfileError,
     DbtProjectError,
     validator_error_message,
     warn_or_error,
@@ -191,6 +188,7 @@ class RuntimeConfig(Project, Profile, AdapterRequiredConfig):
         profile_renderer: ProfileRenderer,
         profile_name: Optional[str],
     ) -> Profile:
+
         return Profile.render_from_args(
             args, profile_renderer, profile_name
         )
@@ -412,21 +410,12 @@ class UnsetCredentials(Credentials):
         return ()
 
 
-class UnsetConfig(UserConfig):
-    def __getattribute__(self, name):
-        if name in {f.name for f in fields(UserConfig)}:
-            raise AttributeError(
-                f"'UnsetConfig' object has no attribute {name}"
-            )
-
-    def __post_serialize__(self, dct):
-        return {}
-
-
+# This is used by UnsetProfileConfig, for commands which do
+# not require a profile, i.e. dbt deps and clean
 class UnsetProfile(Profile):
     def __init__(self):
         self.credentials = UnsetCredentials()
-        self.user_config = UnsetConfig()
+        self.user_config = UserConfig()  # This will be read in _get_rendered_profile
         self.profile_name = ''
         self.target_name = ''
         self.threads = -1
@@ -443,6 +432,8 @@ class UnsetProfile(Profile):
         return Profile.__getattribute__(self, name)
 
 
+# This class is used by the dbt deps and clean commands, because they don't
+# require a functioning profile.
 @dataclass
 class UnsetProfileConfig(RuntimeConfig):
     """This class acts a lot _like_ a RuntimeConfig, except if your profile is
@@ -525,7 +516,7 @@ class UnsetProfileConfig(RuntimeConfig):
             profile_env_vars=profile.profile_env_vars,
             profile_name='',
             target_name='',
-            user_config=UnsetConfig(),
+            user_config=UserConfig(),
             threads=getattr(args, 'threads', 1),
             credentials=UnsetCredentials(),
             args=args,
@@ -540,17 +531,12 @@ class UnsetProfileConfig(RuntimeConfig):
         profile_renderer: ProfileRenderer,
         profile_name: Optional[str],
     ) -> Profile:
-        try:
-            profile = Profile.render_from_args(
-                args, profile_renderer, profile_name
-            )
-        except (DbtProjectError, DbtProfileError) as exc:
-            fire_event(ProfileLoadError(exc=exc))
-            fire_event(ProfileNotFound(profile_name=profile_name))
-            # return the poisoned form
-            profile = UnsetProfile()
-            # disable anonymous usage statistics
-            tracking.disable_tracking()
+
+        profile = UnsetProfile()
+        # The profile (for warehouse connection) is not needed, but we want
+        # to get the UserConfig, which is also in profiles.yml
+        user_config = read_user_config(flags.PROFILES_DIR)
+        profile.user_config = user_config
         return profile
 
     @classmethod
@@ -565,9 +551,6 @@ class UnsetProfileConfig(RuntimeConfig):
         :raises ValidationException: If the cli variables are invalid.
         """
         project, profile = cls.collect_parts(args)
-        if not isinstance(profile, UnsetProfile):
-            # if it's a real profile, return a real config
-            cls = RuntimeConfig
 
         return cls.from_parts(
             project=project,

core/dbt/context/README.md

@@ -0,0 +1 @@
+# Contexts and Jinja rendering

core/dbt/context/base.py

@@ -488,9 +488,9 @@ class BaseContext(metaclass=ContextMeta):
             {% endmacro %}"
         """
         if info:
-            fire_event(MacroEventInfo(msg))
+            fire_event(MacroEventInfo(msg=msg))
         else:
-            fire_event(MacroEventDebug(msg))
+            fire_event(MacroEventDebug(msg=msg))
         return ''
 
     @contextproperty

core/dbt/context/providers.py

@@ -1186,10 +1186,12 @@ class ProviderContext(ManifestContext):
             # If this is compiling, do not save because it's irrelevant to parsing.
             if self.model and not hasattr(self.model, 'compiled'):
                 self.manifest.env_vars[var] = return_value
-                source_file = self.manifest.files[self.model.file_id]
-                # Schema files should never get here
-                if source_file.parse_file_type != 'schema':
-                    source_file.env_vars.append(var)
+                # hooks come from dbt_project.yml which doesn't have a real file_id
+                if self.model.file_id in self.manifest.files:
+                    source_file = self.manifest.files[self.model.file_id]
+                    # Schema files should never get here
+                    if source_file.parse_file_type != 'schema':
+                        source_file.env_vars.append(var)
             return return_value
         else:
             msg = f"Env var required but not provided: '{var}'"

core/dbt/contracts/README.md

@@ -0,0 +1 @@
+# Contracts README

core/dbt/contracts/graph/parsed.py

@@ -178,7 +178,26 @@ class ParsedNodeMandatory(
 
 
 @dataclass
-class ParsedNodeDefaults(ParsedNodeMandatory):
+class NodeInfoMixin():
+    _event_status: Dict[str, Any] = field(default_factory=dict)
+
+    @property
+    def node_info(self):
+        node_info = {
+            "node_path": getattr(self, 'path', None),
+            "node_name": getattr(self, 'name', None),
+            "unique_id": getattr(self, 'unique_id', None),
+            "resource_type": str(getattr(self, 'resource_type', '')),
+            "materialized": self.config.get('materialized'),
+            "node_status": str(self._event_status.get('node_status')),
+            "node_started_at": self._event_status.get("started_at"),
+            "node_finished_at": self._event_status.get("finished_at")
+        }
+        return node_info
+
+
+@dataclass
+class ParsedNodeDefaults(NodeInfoMixin, ParsedNodeMandatory):
     tags: List[str] = field(default_factory=list)
     refs: List[List[str]] = field(default_factory=list)
     sources: List[List[str]] = field(default_factory=list)
@@ -194,7 +213,6 @@ class ParsedNodeDefaults(ParsedNodeMandatory):
     unrendered_config: Dict[str, Any] = field(default_factory=dict)
     created_at: float = field(default_factory=lambda: time.time())
     config_call_dict: Dict[str, Any] = field(default_factory=dict)
-    _event_status: Dict[str, Any] = field(default_factory=dict)
 
     def write_node(self, target_path: str, subdirectory: str, payload: str):
         if (os.path.basename(self.path) ==
@@ -431,6 +449,10 @@ class ParsedSingularTestNode(ParsedNode):
     # refactor the various configs.
     config: TestConfig = field(default_factory=TestConfig)  # type: ignore
 
+    @property
+    def test_node_type(self):
+        return 'singular'
+
 
 @dataclass
 class ParsedGenericTestNode(ParsedNode, HasTestMetadata):
@@ -452,6 +474,10 @@ class ParsedGenericTestNode(ParsedNode, HasTestMetadata):
             True
         )
 
+    @property
+    def test_node_type(self):
+        return 'generic'
+
 
 @dataclass
 class IntermediateSnapshotNode(ParsedNode):
@@ -602,12 +628,11 @@ class UnpatchedSourceDefinition(UnparsedBaseNode, HasUniqueID, HasFqn):
 
 
 @dataclass
-class ParsedSourceDefinition(
+class ParsedSourceMandatory(
     UnparsedBaseNode,
     HasUniqueID,
     HasRelationMetadata,
     HasFqn,
-
 ):
     name: str
     source_name: str
@@ -615,6 +640,13 @@ class ParsedSourceDefinition(
     loader: str
     identifier: str
     resource_type: NodeType = field(metadata={'restrict': [NodeType.Source]})
+
+
+@dataclass
+class ParsedSourceDefinition(
+    NodeInfoMixin,
+    ParsedSourceMandatory
+):
     quoting: Quoting = field(default_factory=Quoting)
     loaded_at_field: Optional[str] = None
     freshness: Optional[FreshnessThreshold] = None
@@ -629,7 +661,6 @@ class ParsedSourceDefinition(
     unrendered_config: Dict[str, Any] = field(default_factory=dict)
     relation_name: Optional[str] = None
     created_at: float = field(default_factory=lambda: time.time())
-    _event_status: Dict[str, Any] = field(default_factory=dict)
 
     def __post_serialize__(self, dct):
         if '_event_status' in dct:

core/dbt/contracts/project.py

@@ -18,6 +18,18 @@ DEFAULT_SEND_ANONYMOUS_USAGE_STATS = True
 class Name(ValidatedStringMixin):
     ValidationRegex = r'^[^\d\W]\w*$'
 
+    @classmethod
+    def is_valid(cls, value: Any) -> bool:
+        if not isinstance(value, str):
+            return False
+
+        try:
+            cls.validate(value)
+        except ValidationError:
+            return False
+
+        return True
+
 
 register_pattern(Name, r'^[^\d\W]\w*$')
 
@@ -231,7 +243,7 @@ class UserConfig(ExtensibleDbtClassMixin, Replaceable, UserConfigContract):
     printer_width: Optional[int] = None
     write_json: Optional[bool] = None
     warn_error: Optional[bool] = None
-    log_format: Optional[bool] = None
+    log_format: Optional[str] = None
     debug: Optional[bool] = None
     version_check: Optional[bool] = None
     fail_fast: Optional[bool] = None

core/dbt/contracts/state.py

@@ -14,7 +14,8 @@ class PreviousState:
         manifest_path = self.path / 'manifest.json'
         if manifest_path.exists() and manifest_path.is_file():
             try:
-                self.manifest = WritableManifest.read(str(manifest_path))
+                # we want to bail with an error if schema versions don't match
+                self.manifest = WritableManifest.read_and_check_versions(str(manifest_path))
             except IncompatibleSchemaException as exc:
                 exc.add_filename(str(manifest_path))
                 raise
@@ -22,7 +23,8 @@ class PreviousState:
         results_path = self.path / 'run_results.json'
         if results_path.exists() and results_path.is_file():
             try:
-                self.results = RunResultsArtifact.read(str(results_path))
+                # we want to bail with an error if schema versions don't match
+                self.results = RunResultsArtifact.read_and_check_versions(str(results_path))
             except IncompatibleSchemaException as exc:
                 exc.add_filename(str(results_path))
                 raise

core/dbt/contracts/util.py

@@ -9,6 +9,7 @@ from dbt.clients.system import write_json, read_json
 from dbt.exceptions import (
     InternalException,
     RuntimeException,
+    IncompatibleSchemaException
 )
 from dbt.version import __version__
 from dbt.events.functions import get_invocation_id
@@ -158,6 +159,8 @@ def get_metadata_env() -> Dict[str, str]:
     }
 
 
+# This is used in the ManifestMetadata, RunResultsMetadata, RunOperationResultMetadata,
+# FreshnessMetadata, and CatalogMetadata classes
 @dataclasses.dataclass
 class BaseArtifactMetadata(dbtClassMixin):
     dbt_schema_version: str
@@ -177,6 +180,17 @@ class BaseArtifactMetadata(dbtClassMixin):
         return dct
 
 
+# This is used as a class decorator to set the schema_version in the
+# 'dbt_schema_version' class attribute. (It's copied into the metadata objects.)
+# Name attributes of SchemaVersion in classes with the 'schema_version' decorator:
+#   manifest
+#   run-results
+#   run-operation-result
+#   sources
+#   catalog
+#   remote-compile-result
+#   remote-execution-result
+#   remote-run-result
 def schema_version(name: str, version: int):
     def inner(cls: Type[VersionedSchema]):
         cls.dbt_schema_version = SchemaVersion(
@@ -187,6 +201,7 @@ def schema_version(name: str, version: int):
     return inner
 
 
+# This is used in the ArtifactMixin and RemoteResult classes
 @dataclasses.dataclass
 class VersionedSchema(dbtClassMixin):
     dbt_schema_version: ClassVar[SchemaVersion]
@@ -198,6 +213,30 @@ class VersionedSchema(dbtClassMixin):
             result['$id'] = str(cls.dbt_schema_version)
         return result
 
+    @classmethod
+    def read_and_check_versions(cls, path: str):
+        try:
+            data = read_json(path)
+        except (EnvironmentError, ValueError) as exc:
+            raise RuntimeException(
+                f'Could not read {cls.__name__} at "{path}" as JSON: {exc}'
+            ) from exc
+
+        # Check metadata version. There is a class variable 'dbt_schema_version', but
+        # that doesn't show up in artifacts, where it only exists in the 'metadata'
+        # dictionary.
+        if hasattr(cls, 'dbt_schema_version'):
+            if 'metadata' in data and 'dbt_schema_version' in data['metadata']:
+                previous_schema_version = data['metadata']['dbt_schema_version']
+                # cls.dbt_schema_version is a SchemaVersion object
+                if str(cls.dbt_schema_version) != previous_schema_version:
+                    raise IncompatibleSchemaException(
+                        expected=str(cls.dbt_schema_version),
+                        found=previous_schema_version
+                    )
+
+        return cls.from_dict(data)  # type: ignore
+
 
 T = TypeVar('T', bound='ArtifactMixin')
 
@@ -205,6 +244,8 @@ T = TypeVar('T', bound='ArtifactMixin')
 # metadata should really be a Generic[T_M] where T_M is a TypeVar bound to
 # BaseArtifactMetadata. Unfortunately this isn't possible due to a mypy issue:
 # https://github.com/python/mypy/issues/7520
+# This is used in the WritableManifest, RunResultsArtifact, RunOperationResultsArtifact,
+# and CatalogArtifact
 @dataclasses.dataclass(init=False)
 class ArtifactMixin(VersionedSchema, Writable, Readable):
     metadata: BaseArtifactMetadata

core/dbt/deprecations.py

@@ -36,9 +36,9 @@ class DBTDeprecation:
         if self.name not in active_deprecations:
             desc = self.description.format(**kwargs)
             msg = ui.line_wrap_message(
-                desc, prefix='* Deprecation Warning:\n\n'
+                desc, prefix='Deprecated functionality\n\n'
             )
-            dbt.exceptions.warn_or_error(msg)
+            dbt.exceptions.warn_or_error(msg, log_fmt=ui.warning_tag('{}'))
             self.track_deprecation_warn()
             active_deprecations.add(self.name)
 
@@ -62,7 +62,7 @@ class PackageInstallPathDeprecation(DBTDeprecation):
 
 class ConfigPathDeprecation(DBTDeprecation):
     _description = '''\
-    The `{deprecated_path}` config has been deprecated in favor of `{exp_path}`.
+    The `{deprecated_path}` config has been renamed to `{exp_path}`.
     Please update your `dbt_project.yml` configuration to reflect this change.
     '''
 

core/dbt/deps/README.md

@@ -0,0 +1 @@
+# Deps README

core/dbt/deps/registry.py

@@ -1,4 +1,5 @@
 import os
+import functools
 from typing import List
 
 from dbt import semver
@@ -14,6 +15,7 @@ from dbt.exceptions import (
     DependencyException,
     package_not_found,
 )
+from dbt.utils import _connection_exception_retry as connection_exception_retry
 
 
 class RegistryPackageMixin:
@@ -68,9 +70,28 @@ class RegistryPinnedPackage(RegistryPackageMixin, PinnedPackage):
         system.make_directory(os.path.dirname(tar_path))
 
         download_url = metadata.downloads.tarball
-        system.download_with_retries(download_url, tar_path)
         deps_path = project.packages_install_path
         package_name = self.get_project_name(project, renderer)
+
+        download_untar_fn = functools.partial(
+            self.download_and_untar,
+            download_url,
+            tar_path,
+            deps_path,
+            package_name
+        )
+        connection_exception_retry(download_untar_fn, 5)
+
+    def download_and_untar(self, download_url, tar_path, deps_path, package_name):
+        """
+        Sometimes the download of the files fails and we want to retry.  Sometimes the
+        download appears successful but the file did not make it through as expected
+        (generally due to a github incident).  Either way we want to retry downloading
+        and untarring to see if we can get a success.  Call this within
+        `_connection_exception_retry`
+        """
+
+        system.download(download_url, tar_path)
         system.untar_package(tar_path, deps_path, package_name)
 
 

core/dbt/events/README.md

@@ -1,12 +1,55 @@
 # Events Module
 
-The Events module is the implmentation for structured logging. These events represent both a programatic interface to dbt processes as well as human-readable messaging in one centralized place. The centralization allows for leveraging mypy to enforce interface invariants across all dbt events, and the distinct type layer allows for decoupling events and libraries such as loggers.
+The Events module is responsible for communicating internal dbt structures into a consumable interface. Right now, the events module is exclusively used for structured logging, but in the future could grow to include other user-facing components such as exceptions. These events represent both a programatic interface to dbt processes as well as human-readable messaging in one centralized place. The centralization allows for leveraging mypy to enforce interface invariants across all dbt events, and the distinct type layer allows for decoupling events and libraries such as loggers.
 
 # Using the Events Module
 The event module provides types that represent what is happening in dbt in `events.types`. These types are intended to represent an exhaustive list of all things happening within dbt that will need to be logged, streamed, or printed. To fire an event, `events.functions::fire_event` is the entry point to the module from everywhere in dbt.
 
+# Logging
+When events are processed via `fire_event`, nearly everything is logged. Whether or not the user has enabled the debug flag, all debug messages are still logged to the file. However, some events are particularly time consuming to construct because they return a huge amount of data. Today, the only messages in this category are cache events and are only logged if the `--log-cache-events` flag is on. This is important because these messages should not be created unless they are going to be logged, because they cause a noticable performance degredation. We achieve this by making the event class explicitly use lazy values for the expensive ones so they are not computed until the moment they are required. This is done with the data type `core/dbt/helper_types.py::Lazy` which includes usage documentation.
+
+Example:
+```
+@dataclass
+class DumpBeforeAddGraph(DebugLevel, Cache):
+    dump: Lazy[Dict[str, List[str]]]
+    code: str = "E031"
+
+    def message(self) -> str:
+        return f"before adding : {self.dump.force()}"
+```
+
+
 # Adding a New Event
-In `events.types` add a new class that represents the new event. This may be a simple class with no values, or it may be a dataclass with some values to construct downstream messaging. Only include the data necessary to construct this message within this class. You must extend all destinations (e.g. - if your log message belongs on the cli, extend `CliEventABC`) as well as the loglevel this event belongs to.
+In `events.types` add a new class that represents the new event. All events must be a dataclass with, at minimum, a code.  You may also include some other values to construct downstream messaging. Only include the data necessary to construct this message within this class. You must extend all destinations (e.g. - if your log message belongs on the cli, extend `Cli`) as well as the loglevel this event belongs to.  This system has been designed to take full advantage of mypy so running it will catch anything you may miss.
+
+## Required for Every Event
+
+- a string attribute `code`, that's unique across events
+- assign a log level by extending `DebugLevel`, `InfoLevel`, `WarnLevel`, or `ErrorLevel`
+- a message()
+- extend `File` and/or `Cli` based on where it should output
+
+Example
+```
+@dataclass
+class PartialParsingDeletedExposure(DebugLevel, Cli, File):
+    unique_id: str
+    code: str = "I049"
+
+    def message(self) -> str:
+        return f"Partial parsing: deleted exposure {self.unique_id}"
+
+```
+
+## Optional (based on your event)
+
+- Events associated with node status changes must be extended with `NodeInfo` which contains a node_info attribute
+
+
+All values other than `code` and `node_info` will be included in the `data` node of the json log output.
+
+Once your event has been added, add a dummy call to your new event at the bottom of `types.py` and also add your new Event to the list `sample_values` in `test/unit/test_events.py'.
 
 # Adapter Maintainers
 To integrate existing log messages from adapters, you likely have a line of code like this in your adapter already:

core/dbt/events/base_types.py

@@ -1,10 +1,10 @@
-from abc import ABCMeta, abstractmethod, abstractproperty
+from abc import ABCMeta, abstractproperty, abstractmethod
 from dataclasses import dataclass
-from datetime import datetime
-import json
+from dbt.events.serialization import EventSerialization
 import os
 import threading
-from typing import Any, Optional
+from typing import Any, Dict
+
 
 # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
 # These base types define the _required structure_ for the concrete event #
@@ -12,43 +12,9 @@ from typing import Any, Optional
 # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
 
 
-# in preparation for #3977
-class TestLevel():
-    def level_tag(self) -> str:
-        return "test"
-
-
-class DebugLevel():
-    def level_tag(self) -> str:
-        return "debug"
-
-
-class InfoLevel():
-    def level_tag(self) -> str:
-        return "info"
-
-
-class WarnLevel():
-    def level_tag(self) -> str:
-        return "warn"
-
-
-class ErrorLevel():
-    def level_tag(self) -> str:
-        return "error"
-
-
-@dataclass
-class Node():
-    node_path: str
-    node_name: str
-    unique_id: str
-    resource_type: str
-    materialized: str
-    node_status: str
-    node_started_at: datetime
-    node_finished_at: Optional[datetime]
-    type: str = 'node_status'
+class Cache():
+    # Events with this class will only be logged when the `--log-cache-events` flag is passed
+    pass
 
 
 @dataclass
@@ -64,14 +30,9 @@ class ShowException():
 
 
 # TODO add exhaustiveness checking for subclasses
-# can't use ABCs with @dataclass because of https://github.com/python/mypy/issues/5374
 # top-level superclass for all events
 class Event(metaclass=ABCMeta):
-    # fields that should be on all events with their default implementations
-    log_version: int = 1
-    ts: Optional[datetime] = None  # use getter for non-optional
-    pid: Optional[int] = None  # use getter for non-optional
-    node_info: Optional[Node]
+    # Do not define fields with defaults here
 
     # four digit string code that uniquely identifies this type of event
     # uniqueness and valid characters are enforced by tests
@@ -80,6 +41,12 @@ class Event(metaclass=ABCMeta):
     def code() -> str:
         raise Exception("code() not implemented for event")
 
+    # The 'to_dict' method is added by mashumaro via the EventSerialization.
+    # It should be in all subclasses that are to record actual events.
+    @abstractmethod
+    def to_dict(self):
+        raise Exception('to_dict not implemented for Event')
+
     # do not define this yourself. inherit it from one of the above level types.
     @abstractmethod
     def level_tag(self) -> str:
@@ -91,37 +58,9 @@ class Event(metaclass=ABCMeta):
     def message(self) -> str:
         raise Exception("msg not implemented for Event")
 
-    # override this method to convert non-json serializable fields to json.
-    # for override examples, see existing concrete types.
-    #
-    # there is no type-level mechanism to have mypy enforce json serializability, so we just try
-    # to serialize and raise an exception at runtime when that fails. This safety mechanism
-    # only works if we have attempted to serialize every concrete event type in our tests.
-    def fields_to_json(self, field_value: Any) -> Any:
-        try:
-            json.dumps(field_value, sort_keys=True)
-            return field_value
-        except TypeError:
-            val_type = type(field_value).__name__
-            event_type = type(self).__name__
-            return Exception(
-                f"type {val_type} is not serializable to json."
-                f" First make sure that the call sites for {event_type} match the type hints"
-                f" and if they do, you can override Event::fields_to_json in {event_type} in"
-                " types.py to define your own serialization function to any valid json type"
-            )
-
-    # exactly one time stamp per concrete event
-    def get_ts(self) -> datetime:
-        if not self.ts:
-            self.ts = datetime.now()
-        return self.ts
-
     # exactly one pid per concrete event
     def get_pid(self) -> int:
-        if not self.pid:
-            self.pid = os.getpid()
-        return self.pid
+        return os.getpid()
 
     # in theory threads can change so we don't cache them.
     def get_thread_name(self) -> str:
@@ -133,33 +72,49 @@ class Event(metaclass=ABCMeta):
         return get_invocation_id()
 
 
-@dataclass  # type: ignore
-class NodeInfo(Event, metaclass=ABCMeta):
-    report_node_data: Any  # Union[ParsedModelNode, ...] TODO: resolve circular imports
-
-    def get_node_info(self):
-        node_info = Node(
-            node_path=self.report_node_data.path,
-            node_name=self.report_node_data.name,
-            unique_id=self.report_node_data.unique_id,
-            resource_type=self.report_node_data.resource_type.value,
-            materialized=self.report_node_data.config.materialized,
-            node_status=str(self.report_node_data._event_status.get('node_status')),
-            node_started_at=self.report_node_data._event_status.get("started_at"),
-            node_finished_at=self.report_node_data._event_status.get("finished_at")
-        )
-        return node_info
+# in preparation for #3977
+@dataclass  # type: ignore[misc]
+class TestLevel(EventSerialization, Event):
+    def level_tag(self) -> str:
+        return "test"
 
 
-class File(Event, metaclass=ABCMeta):
-    # Solely the human readable message. Timestamps and formatting will be added by the logger.
-    def file_msg(self) -> str:
-        # returns the event msg unless overriden in the concrete class
-        return self.message()
+@dataclass  # type: ignore[misc]
+class DebugLevel(EventSerialization, Event):
+    def level_tag(self) -> str:
+        return "debug"
 
 
-class Cli(Event, metaclass=ABCMeta):
-    # Solely the human readable message. Timestamps and formatting will be added by the logger.
-    def cli_msg(self) -> str:
-        # returns the event msg unless overriden in the concrete class
-        return self.message()
+@dataclass  # type: ignore[misc]
+class InfoLevel(EventSerialization, Event):
+    def level_tag(self) -> str:
+        return "info"
+
+
+@dataclass  # type: ignore[misc]
+class WarnLevel(EventSerialization, Event):
+    def level_tag(self) -> str:
+        return "warn"
+
+
+@dataclass  # type: ignore[misc]
+class ErrorLevel(EventSerialization, Event):
+    def level_tag(self) -> str:
+        return "error"
+
+
+# prevents an event from going to the file
+class NoFile():
+    pass
+
+
+# prevents an event from going to stdout
+class NoStdOut():
+    pass
+
+
+# This class represents the node_info which is generated
+# by the NodeInfoMixin class in dbt.contracts.graph.parsed
+@dataclass
+class NodeInfo():
+    node_info: Dict[str, Any]

core/dbt/events/functions.py

@@ -1,31 +1,34 @@
-
+import colorama
 from colorama import Style
-from datetime import datetime
 import dbt.events.functions as this  # don't worry I hate it too.
-from dbt.events.base_types import Cli, Event, File, ShowException, NodeInfo
-from dbt.events.types import EventBufferFull, T_Event
+from dbt.events.base_types import NoStdOut, Event, NoFile, ShowException, Cache
+from dbt.events.types import EventBufferFull, T_Event, MainReportVersion, EmptyLine
 import dbt.flags as flags
 # TODO this will need to move eventually
 from dbt.logger import SECRET_ENV_PREFIX, make_log_dir_if_missing, GLOBAL_LOGGER
+from datetime import datetime
 import json
 import io
 from io import StringIO, TextIOWrapper
 import logbook
 import logging
 from logging import Logger
+import sys
 from logging.handlers import RotatingFileHandler
 import os
 import uuid
-from typing import Any, Callable, Dict, List, Optional, Union
-import dataclasses
+import threading
+from typing import Any, Dict, List, Optional, Union
 from collections import deque
 
+global LOG_VERSION
+LOG_VERSION = 2
 
-# create the global event history buffer with a max size of 100k records
+# create the global event history buffer with the default max size (10k)
 # python 3.7 doesn't support type hints on globals, but mypy requires them. hence the ignore.
-# TODO: make the maxlen something configurable from the command line via args(?)
+# TODO the flags module has not yet been resolved when this is created
 global EVENT_HISTORY
-EVENT_HISTORY = deque(maxlen=100000)  # type: ignore
+EVENT_HISTORY = deque(maxlen=flags.EVENT_BUFFER_SIZE)  # type: ignore
 
 # create the global file logger with no configuration
 global FILE_LOG
@@ -38,7 +41,7 @@ FILE_LOG.addHandler(null_handler)
 global STDOUT_LOG
 STDOUT_LOG = logging.getLogger('default_stdout')
 STDOUT_LOG.setLevel(logging.INFO)
-stdout_handler = logging.StreamHandler()
+stdout_handler = logging.StreamHandler(sys.stdout)
 stdout_handler.setLevel(logging.INFO)
 STDOUT_LOG.addHandler(stdout_handler)
 
@@ -46,8 +49,31 @@ format_color = True
 format_json = False
 invocation_id: Optional[str] = None
 
+# Colorama needs some help on windows because we're using logger.info
+# intead of print(). If the Windows env doesn't have a TERM var set,
+# then we should override the logging stream to use the colorama
+# converter. If the TERM var is set (as with Git Bash), then it's safe
+# to send escape characters and no log handler injection is needed.
+colorama_stdout = sys.stdout
+colorama_wrap = True
+
+colorama.init(wrap=colorama_wrap)
+
+if sys.platform == 'win32' and not os.getenv('TERM'):
+    colorama_wrap = False
+    colorama_stdout = colorama.AnsiToWin32(sys.stdout).stream
+
+elif sys.platform == 'win32':
+    colorama_wrap = False
+
+colorama.init(wrap=colorama_wrap)
+
 
 def setup_event_logger(log_path, level_override=None):
+    # flags have been resolved, and log_path is known
+    global EVENT_HISTORY
+    EVENT_HISTORY = deque(maxlen=flags.EVENT_BUFFER_SIZE)  # type: ignore
+
     make_log_dir_if_missing(log_path)
     this.format_json = flags.LOG_FORMAT == 'json'
     # USE_COLORS can be None if the app just started and the cli flags
@@ -64,7 +90,7 @@ def setup_event_logger(log_path, level_override=None):
     FORMAT = "%(message)s"
     stdout_passthrough_formatter = logging.Formatter(fmt=FORMAT)
 
-    stdout_handler = logging.StreamHandler()
+    stdout_handler = logging.StreamHandler(sys.stdout)
     stdout_handler.setFormatter(stdout_passthrough_formatter)
     stdout_handler.setLevel(level)
     # clear existing stdout TextIOWrapper stream handlers
@@ -80,7 +106,12 @@ def setup_event_logger(log_path, level_override=None):
 
     file_passthrough_formatter = logging.Formatter(fmt=FORMAT)
 
-    file_handler = RotatingFileHandler(filename=log_dest, encoding='utf8')
+    file_handler = RotatingFileHandler(
+        filename=log_dest,
+        encoding='utf8',
+        maxBytes=10 * 1024 * 1024,  # 10 mb
+        backupCount=5
+    )
     file_handler.setFormatter(file_passthrough_formatter)
     file_handler.setLevel(logging.DEBUG)  # always debug regardless of user input
     this.FILE_LOG.handlers.clear()
@@ -120,76 +151,100 @@ def scrub_secrets(msg: str, secrets: List[str]) -> str:
     return scrubbed
 
 
-# returns a dictionary representation of the event fields. You must specify which of the
-# available messages you would like to use (i.e. - e.message, e.cli_msg(), e.file_msg())
-# used for constructing json formatted events. includes secrets which must be scrubbed at
-# the usage site.
+# returns a dictionary representation of the event fields.
+# the message may contain secrets which must be scrubbed at the usage site.
 def event_to_serializable_dict(
-    e: T_Event, ts_fn: Callable[[datetime], str],
-    msg_fn: Callable[[T_Event], str]
+    e: T_Event,
 ) -> Dict[str, Any]:
-    data = dict()
-    node_info = dict()
-    if hasattr(e, '__dataclass_fields__'):
-        for field, value in dataclasses.asdict(e).items():  # type: ignore[attr-defined]
-            _json_value = e.fields_to_json(value)
 
-            if isinstance(e, NodeInfo):
-                node_info = dataclasses.asdict(e.get_node_info())
+    log_line = dict()
+    code: str
+    try:
+        log_line = e.to_dict()
+    except AttributeError as exc:
+        event_type = type(e).__name__
+        raise Exception(  # TODO this may hang async threads
+            f"type {event_type} is not serializable. {str(exc)}"
+        )
 
-            if not isinstance(_json_value, Exception):
-                data[field] = _json_value
-            else:
-                data[field] = f"JSON_SERIALIZE_FAILED: {type(value).__name__, 'NA'}"
+    # We get the code from the event object, so we don't need it in the data
+    if 'code' in log_line:
+        del log_line['code']
 
     event_dict = {
         'type': 'log_line',
-        'log_version': e.log_version,
-        'ts': ts_fn(e.get_ts()),
+        'log_version': LOG_VERSION,
+        'ts': get_ts_rfc3339(),
         'pid': e.get_pid(),
-        'msg': msg_fn(e),
+        'msg': e.message(),
         'level': e.level_tag(),
-        'data': data,
+        'data': log_line,
         'invocation_id': e.get_invocation_id(),
         'thread_name': e.get_thread_name(),
-        'node_info': node_info
+        'code': e.code
     }
 
     return event_dict
 
 
 # translates an Event to a completely formatted text-based log line
-# you have to specify which message you want. (i.e. - e.message, e.cli_msg(), e.file_msg())
 # type hinting everything as strings so we don't get any unintentional string conversions via str()
-def create_text_log_line(e: T_Event, msg_fn: Callable[[T_Event], str]) -> str:
+def create_info_text_log_line(e: T_Event) -> str:
     color_tag: str = '' if this.format_color else Style.RESET_ALL
-    ts: str = e.get_ts().strftime("%H:%M:%S")
-    scrubbed_msg: str = scrub_secrets(msg_fn(e), env_secrets())
+    ts: str = get_ts().strftime("%H:%M:%S")
+    scrubbed_msg: str = scrub_secrets(e.message(), env_secrets())
+    log_line: str = f"{color_tag}{ts}  {scrubbed_msg}"
+    return log_line
+
+
+def create_debug_text_log_line(e: T_Event) -> str:
+    log_line: str = ''
+    # Create a separator if this is the beginning of an invocation
+    if type(e) == MainReportVersion:
+        separator = 30 * '='
+        log_line = f'\n\n{separator} {get_ts()} | {get_invocation_id()} {separator}\n'
+    color_tag: str = '' if this.format_color else Style.RESET_ALL
+    ts: str = get_ts().strftime("%H:%M:%S.%f")
+    scrubbed_msg: str = scrub_secrets(e.message(), env_secrets())
     level: str = e.level_tag() if len(e.level_tag()) == 5 else f"{e.level_tag()} "
-    log_line: str = f"{color_tag}{ts} | [ {level} ] | {scrubbed_msg}"
+    thread = ''
+    if threading.current_thread().getName():
+        thread_name = threading.current_thread().getName()
+        thread_name = thread_name[:10]
+        thread_name = thread_name.ljust(10, ' ')
+        thread = f' [{thread_name}]:'
+    log_line = log_line + f"{color_tag}{ts} [{level}]{thread} {scrubbed_msg}"
     return log_line
 
 
 # translates an Event to a completely formatted json log line
-# you have to specify which message you want. (i.e. - e.message(), e.cli_msg(), e.file_msg())
-def create_json_log_line(e: T_Event, msg_fn: Callable[[T_Event], str]) -> str:
-    values = event_to_serializable_dict(e, lambda dt: dt.isoformat(), lambda x: msg_fn(x))
+def create_json_log_line(e: T_Event) -> Optional[str]:
+    if type(e) == EmptyLine:
+        return None  # will not be sent to logger
+    # using preformatted ts string instead of formatting it here to be extra careful about timezone
+    values = event_to_serializable_dict(e)
     raw_log_line = json.dumps(values, sort_keys=True)
     return scrub_secrets(raw_log_line, env_secrets())
 
 
-# calls create_text_log_line() or create_json_log_line() according to logger config
-def create_log_line(e: T_Event, msg_fn: Callable[[T_Event], str]) -> str:
-    return (
-        create_json_log_line(e, msg_fn)
-        if this.format_json else
-        create_text_log_line(e, msg_fn)
-    )
+# calls create_stdout_text_log_line() or create_json_log_line() according to logger config
+def create_log_line(
+    e: T_Event,
+    file_output=False
+) -> Optional[str]:
+    if this.format_json:
+        return create_json_log_line(e)  # json output, both console and file
+    elif file_output is True or flags.DEBUG:
+        return create_debug_text_log_line(e)  # default file output
+    else:
+        return create_info_text_log_line(e)  # console output
 
 
 # allows for resuse of this obnoxious if else tree.
 # do not use for exceptions, it doesn't pass along exc_info, stack_info, or extra
 def send_to_logger(l: Union[Logger, logbook.Logger], level_tag: str, log_line: str):
+    if not log_line:
+        return
     if level_tag == 'test':
         # TODO after implmenting #3977 send to new test level
         l.debug(log_line)
@@ -262,48 +317,54 @@ def send_exc_to_logger(
 # (i.e. - mutating the event history, printing to stdout, logging
 # to files, etc.)
 def fire_event(e: Event) -> None:
+    # skip logs when `--log-cache-events` is not passed
+    if isinstance(e, Cache) and not flags.LOG_CACHE_EVENTS:
+        return
+
     # if and only if the event history deque will be completely filled by this event
     # fire warning that old events are now being dropped
     global EVENT_HISTORY
-    if len(EVENT_HISTORY) == ((EVENT_HISTORY.maxlen or 100000) - 1):
+    if len(EVENT_HISTORY) == (flags.EVENT_BUFFER_SIZE - 1):
+        EVENT_HISTORY.append(e)
         fire_event(EventBufferFull())
-
-    EVENT_HISTORY.append(e)
+    else:
+        EVENT_HISTORY.append(e)
 
     # backwards compatibility for plugins that require old logger (dbt-rpc)
     if flags.ENABLE_LEGACY_LOGGER:
         # using Event::message because the legacy logger didn't differentiate messages by
         # destination
-        log_line = create_log_line(e, msg_fn=lambda x: x.message())
-
-        send_to_logger(GLOBAL_LOGGER, e.level_tag(), log_line)
+        log_line = create_log_line(e)
+        if log_line:
+            send_to_logger(GLOBAL_LOGGER, e.level_tag(), log_line)
         return  # exit the function to avoid using the current logger as well
 
     # always logs debug level regardless of user input
-    if isinstance(e, File):
-        log_line = create_log_line(e, msg_fn=lambda x: x.file_msg())
+    if not isinstance(e, NoFile):
+        log_line = create_log_line(e, file_output=True)
         # doesn't send exceptions to exception logger
-        send_to_logger(FILE_LOG, level_tag=e.level_tag(), log_line=log_line)
+        if log_line:
+            send_to_logger(FILE_LOG, level_tag=e.level_tag(), log_line=log_line)
 
-    if isinstance(e, Cli):
+    if not isinstance(e, NoStdOut):
         # explicitly checking the debug flag here so that potentially expensive-to-construct
         # log messages are not constructed if debug messages are never shown.
         if e.level_tag() == 'debug' and not flags.DEBUG:
             return  # eat the message in case it was one of the expensive ones
 
-        log_line = create_log_line(e, msg_fn=lambda x: x.cli_msg())
-        if not isinstance(e, ShowException):
-            send_to_logger(STDOUT_LOG, level_tag=e.level_tag(), log_line=log_line)
-        # CliEventABC and ShowException
-        else:
-            send_exc_to_logger(
-                STDOUT_LOG,
-                level_tag=e.level_tag(),
-                log_line=log_line,
-                exc_info=e.exc_info,
-                stack_info=e.stack_info,
-                extra=e.extra
-            )
+        log_line = create_log_line(e)
+        if log_line:
+            if not isinstance(e, ShowException):
+                send_to_logger(STDOUT_LOG, level_tag=e.level_tag(), log_line=log_line)
+            else:
+                send_exc_to_logger(
+                    STDOUT_LOG,
+                    level_tag=e.level_tag(),
+                    log_line=log_line,
+                    exc_info=e.exc_info,
+                    stack_info=e.stack_info,
+                    extra=e.extra
+                )
 
 
 def get_invocation_id() -> str:
@@ -318,3 +379,16 @@ def set_invocation_id() -> None:
     # commands in the dbt servers. It shouldn't be necessary for the CLI.
     global invocation_id
     invocation_id = str(uuid.uuid4())
+
+
+# exactly one time stamp per concrete event
+def get_ts() -> datetime:
+    ts = datetime.utcnow()
+    return ts
+
+
+# preformatted time stamp
+def get_ts_rfc3339() -> str:
+    ts = get_ts()
+    ts_rfc3339 = ts.strftime('%Y-%m-%dT%H:%M:%S.%fZ')
+    return ts_rfc3339

core/dbt/events/serialization.py

@@ -0,0 +1,56 @@
+from dbt.helper_types import Lazy
+from mashumaro import DataClassDictMixin
+from mashumaro.config import (
+    BaseConfig as MashBaseConfig
+)
+from mashumaro.types import SerializationStrategy
+from typing import Dict, List
+
+
+# The dbtClassMixin serialization class has a DateTime serialization strategy
+# class. If a datetime ends up in an event class, we could use a similar class
+# here to serialize it in our preferred format.
+
+class ExceptionSerialization(SerializationStrategy):
+    def serialize(self, value):
+        out = str(value)
+        return out
+
+    def deserialize(self, value):
+        return (Exception(value))
+
+
+class BaseExceptionSerialization(SerializationStrategy):
+    def serialize(self, value):
+        return str(value)
+
+    def deserialize(self, value):
+        return (BaseException(value))
+
+
+# This is an explicit deserializer for the type Lazy[Dict[str, List[str]]]
+# mashumaro does not support composing serialization strategies, so all
+# future uses of Lazy will need to register a unique serialization class like this one.
+class LazySerialization1(SerializationStrategy):
+    def serialize(self, value) -> Dict[str, List[str]]:
+        return value.force()
+
+    # we _can_ deserialize into a lazy value, but that defers running the deserialization
+    # function till the value is used which can raise errors at very unexpected times.
+    # It's best practice to do strict deserialization unless you're in a very special case.
+    def deserialize(self, value):
+        raise Exception("Don't deserialize into a Lazy value. Try just using the value itself.")
+
+
+# This class is the equivalent of dbtClassMixin that's used for serialization
+# in other parts of the code. That class did extra things which we didn't want
+# to use for events, so this class is a simpler version of dbtClassMixin.
+class EventSerialization(DataClassDictMixin):
+
+    # This is where we register serializtion strategies per type.
+    class Config(MashBaseConfig):
+        serialization_strategy = {
+            Exception: ExceptionSerialization(),
+            BaseException: ExceptionSerialization(),
+            Lazy[Dict[str, List[str]]]: LazySerialization1()
+        }

core/dbt/events/stubs.py

@@ -1,72 +0,0 @@
-from typing import (
-    Any,
-    List,
-    NamedTuple,
-    Optional,
-    Dict,
-)
-
-# N.B.:
-# These stubs were autogenerated by stubgen and then hacked
-# to pieces to ensure we had something other than "Any" types
-# where using external classes to instantiate event subclasses
-# in events/types.py.
-#
-# This goes away when we turn mypy on for everything.
-#
-# Don't trust them too much at all!
-
-
-class _ReferenceKey(NamedTuple):
-    database: Any
-    schema: Any
-    identifier: Any
-
-
-class _CachedRelation:
-    referenced_by: Any
-    inner: Any
-
-
-class BaseRelation:
-    path: Any
-    type: Optional[Any]
-    quote_character: str
-    include_policy: Any
-    quote_policy: Any
-    dbt_created: bool
-
-
-class InformationSchema(BaseRelation):
-    information_schema_view: Optional[str]
-
-
-class CompiledNode():
-    compiled_sql: Optional[str]
-    extra_ctes_injected: bool
-    extra_ctes: List[Any]
-    relation_name: Optional[str]
-
-
-class CompiledModelNode(CompiledNode):
-    resource_type: Any
-
-
-class ParsedModelNode():
-    resource_type: Any
-
-
-class ParsedHookNode():
-    resource_type: Any
-    index: Optional[int]
-
-
-class RunResult():
-    status: str
-    timing: List[Any]
-    thread_id: str
-    execution_time: float
-    adapter_response: Dict[str, Any]
-    message: Optional[str]
-    failures: Optional[int]
-    node: Any

core/dbt/events/test_types.py

@@ -5,7 +5,7 @@ from .types import (
     WarnLevel,
     ErrorLevel,
     ShowException,
-    Cli
+    NoFile
 )
 
 
@@ -13,7 +13,7 @@ from .types import (
 # Reuse the existing messages when adding logs to tests.
 
 @dataclass
-class IntegrationTestInfo(InfoLevel, Cli):
+class IntegrationTestInfo(InfoLevel, NoFile):
     msg: str
     code: str = "T001"
 
@@ -22,7 +22,7 @@ class IntegrationTestInfo(InfoLevel, Cli):
 
 
 @dataclass
-class IntegrationTestDebug(DebugLevel, Cli):
+class IntegrationTestDebug(DebugLevel, NoFile):
     msg: str
     code: str = "T002"
 
@@ -31,7 +31,7 @@ class IntegrationTestDebug(DebugLevel, Cli):
 
 
 @dataclass
-class IntegrationTestWarn(WarnLevel, Cli):
+class IntegrationTestWarn(WarnLevel, NoFile):
     msg: str
     code: str = "T003"
 
@@ -40,7 +40,7 @@ class IntegrationTestWarn(WarnLevel, Cli):
 
 
 @dataclass
-class IntegrationTestError(ErrorLevel, Cli):
+class IntegrationTestError(ErrorLevel, NoFile):
     msg: str
     code: str = "T004"
 
@@ -49,7 +49,7 @@ class IntegrationTestError(ErrorLevel, Cli):
 
 
 @dataclass
-class IntegrationTestException(ShowException, ErrorLevel, Cli):
+class IntegrationTestException(ShowException, ErrorLevel, NoFile):
     msg: str
     code: str = "T005"
 
@@ -58,7 +58,7 @@ class IntegrationTestException(ShowException, ErrorLevel, Cli):
 
 
 @dataclass
-class UnitTestInfo(InfoLevel, Cli):
+class UnitTestInfo(InfoLevel, NoFile):
     msg: str
     code: str = "T006"
 

core/dbt/exceptions.py

@@ -2,8 +2,7 @@ import builtins
 import functools
 from typing import NoReturn, Optional, Mapping, Any
 
-from dbt.logger import get_secret_env
-from dbt.events.functions import fire_event
+from dbt.events.functions import fire_event, scrub_secrets, env_secrets
 from dbt.events.types import GeneralWarningMsg, GeneralWarningException
 from dbt.node_types import NodeType
 from dbt import flags
@@ -54,7 +53,7 @@ class RuntimeException(RuntimeError, Exception):
     def __init__(self, msg, node=None):
         self.stack = []
         self.node = node
-        self.msg = msg
+        self.msg = scrub_secrets(msg, env_secrets())
 
     def add_node(self, node=None):
         if node is not None and node is not self.node:
@@ -401,8 +400,6 @@ class CommandError(RuntimeException):
         super().__init__(message)
         self.cwd = cwd
         self.cmd = cmd
-        for secret in get_secret_env():
-            self.cmd = str(self.cmd).replace(secret, "*****")
         self.args = (cwd, cmd, message)
 
     def __str__(self):
@@ -466,7 +463,21 @@ def raise_database_error(msg, node=None) -> NoReturn:
 
 
 def raise_dependency_error(msg) -> NoReturn:
-    raise DependencyException(msg)
+    raise DependencyException(scrub_secrets(msg, env_secrets()))
+
+
+def raise_git_cloning_error(error: CommandResultError) -> NoReturn:
+    error.cmd = scrub_secrets(str(error.cmd), env_secrets())
+    raise error
+
+
+def raise_git_cloning_problem(repo) -> NoReturn:
+    repo = scrub_secrets(repo, env_secrets())
+    msg = '''\
+    Something went wrong while cloning {}
+    Check the debug logs for more information
+    '''
+    raise RuntimeException(msg.format(repo))
 
 
 def disallow_secret_env_var(env_var_name) -> NoReturn:
@@ -692,9 +703,9 @@ def missing_materialization(model, adapter_type):
 
 
 def bad_package_spec(repo, spec, error_message):
-    raise InternalException(
-        "Error checking out spec='{}' for repo {}\n{}".format(
-            spec, repo, error_message))
+    msg = "Error checking out spec='{}' for repo {}\n{}".format(spec, repo, error_message)
+
+    raise InternalException(scrub_secrets(msg, env_secrets()))
 
 
 def raise_cache_inconsistent(message):
@@ -763,6 +774,10 @@ def system_error(operation_name):
 
 
 class ConnectionException(Exception):
+    """
+    There was a problem with the connection that returned a bad response,
+    timed out, or resulted in a file that is corrupt.
+    """
     pass
 
 
@@ -999,7 +1014,7 @@ def raise_duplicate_alias(
 
 def warn_or_error(msg, node=None, log_fmt=None):
     if flags.WARN_ERROR:
-        raise_compiler_error(msg, node)
+        raise_compiler_error(scrub_secrets(msg, env_secrets()), node)
     else:
         fire_event(GeneralWarningMsg(msg=msg, log_fmt=log_fmt))
 

core/dbt/flags.py

@@ -33,6 +33,8 @@ SEND_ANONYMOUS_USAGE_STATS = None
 PRINTER_WIDTH = 80
 WHICH = None
 INDIRECT_SELECTION = None
+LOG_CACHE_EVENTS = None
+EVENT_BUFFER_SIZE = 100000
 
 # Global CLI defaults. These flags are set from three places:
 # CLI args, environment variables, and user_config (profiles.yml).
@@ -51,7 +53,9 @@ flag_defaults = {
     "FAIL_FAST": False,
     "SEND_ANONYMOUS_USAGE_STATS": True,
     "PRINTER_WIDTH": 80,
-    "INDIRECT_SELECTION": 'eager'
+    "INDIRECT_SELECTION": 'eager',
+    "LOG_CACHE_EVENTS": False,
+    "EVENT_BUFFER_SIZE": 100000
 }
 
 
@@ -99,7 +103,7 @@ def set_from_args(args, user_config):
         USE_EXPERIMENTAL_PARSER, STATIC_PARSER, WRITE_JSON, PARTIAL_PARSE, \
         USE_COLORS, STORE_FAILURES, PROFILES_DIR, DEBUG, LOG_FORMAT, INDIRECT_SELECTION, \
         VERSION_CHECK, FAIL_FAST, SEND_ANONYMOUS_USAGE_STATS, PRINTER_WIDTH, \
-        WHICH
+        WHICH, LOG_CACHE_EVENTS, EVENT_BUFFER_SIZE
 
     STRICT_MODE = False  # backwards compatibility
     # cli args without user_config or env var option
@@ -122,6 +126,8 @@ def set_from_args(args, user_config):
     SEND_ANONYMOUS_USAGE_STATS = get_flag_value('SEND_ANONYMOUS_USAGE_STATS', args, user_config)
     PRINTER_WIDTH = get_flag_value('PRINTER_WIDTH', args, user_config)
     INDIRECT_SELECTION = get_flag_value('INDIRECT_SELECTION', args, user_config)
+    LOG_CACHE_EVENTS = get_flag_value('LOG_CACHE_EVENTS', args, user_config)
+    EVENT_BUFFER_SIZE = get_flag_value('EVENT_BUFFER_SIZE', args, user_config)
 
 
 def get_flag_value(flag, args, user_config):
@@ -134,7 +140,13 @@ def get_flag_value(flag, args, user_config):
         if env_value is not None and env_value != '':
             env_value = env_value.lower()
             # non Boolean values
-            if flag in ['LOG_FORMAT', 'PRINTER_WIDTH', 'PROFILES_DIR', 'INDIRECT_SELECTION']:
+            if flag in [
+                'LOG_FORMAT',
+                'PRINTER_WIDTH',
+                'PROFILES_DIR',
+                'INDIRECT_SELECTION',
+                'EVENT_BUFFER_SIZE'
+            ]:
                 flag_value = env_value
             else:
                 flag_value = env_set_bool(env_value)
@@ -142,7 +154,7 @@ def get_flag_value(flag, args, user_config):
             flag_value = getattr(user_config, lc_flag)
         else:
             flag_value = flag_defaults[flag]
-    if flag == 'PRINTER_WIDTH':  # printer_width must be an int or it hangs
+    if flag in ['PRINTER_WIDTH', 'EVENT_BUFFER_SIZE']:  # must be ints
         flag_value = int(flag_value)
     if flag == 'PROFILES_DIR':
         flag_value = os.path.abspath(flag_value)
@@ -165,5 +177,7 @@ def get_flag_dict():
         "fail_fast": FAIL_FAST,
         "send_anonymous_usage_stats": SEND_ANONYMOUS_USAGE_STATS,
         "printer_width": PRINTER_WIDTH,
-        "indirect_selection": INDIRECT_SELECTION
+        "indirect_selection": INDIRECT_SELECTION,
+        "log_cache_events": LOG_CACHE_EVENTS,
+        "event_buffer_size": EVENT_BUFFER_SIZE
     }

core/dbt/graph/README.md

@@ -0,0 +1 @@
+# Graph README

core/dbt/graph/graph.py

@@ -31,11 +31,13 @@ class Graph:
         """Returns all nodes having a path to `node` in `graph`"""
         if not self.graph.has_node(node):
             raise InternalException(f'Node {node} not found in the graph!')
-        with nx.utils.reversed(self.graph):
-            anc = nx.single_source_shortest_path_length(G=self.graph,
-                                                        source=node,
-                                                        cutoff=max_depth)\
-                .keys()
+        # This used to use nx.utils.reversed(self.graph), but that is deprecated,
+        # so changing to use self.graph.reverse(copy=False) as recommeneded
+        G = self.graph.reverse(copy=False) if self.graph.is_directed() else self.graph
+        anc = nx.single_source_shortest_path_length(G=G,
+                                                    source=node,
+                                                    cutoff=max_depth)\
+            .keys()
         return anc - {node}
 
     def descendants(

core/dbt/graph/selector.py

@@ -86,8 +86,9 @@ class NodeSelector(MethodManager):
         try:
             collected = self.select_included(nodes, spec)
         except InvalidSelectorException:
+            valid_selectors = ", ".join(self.SELECTOR_METHODS)
             fire_event(SelectorReportInvalidSelector(
-                selector_methods=self.SELECTOR_METHODS,
+                valid_selectors=valid_selectors,
                 spec_method=spec.method,
                 raw_spec=spec.raw
             ))

core/dbt/graph/selector_methods.py

@@ -1,7 +1,7 @@
 import abc
 from itertools import chain
 from pathlib import Path
-from typing import Set, List, Dict, Iterator, Tuple, Any, Union, Type, Optional
+from typing import Set, List, Dict, Iterator, Tuple, Any, Union, Type, Optional, Callable
 
 from dbt.dataclass_schema import StrEnum
 
@@ -478,42 +478,28 @@ class StateSelectorMethod(SelectorMethod):
             previous_macros = []
             return self.recursively_check_macros_modified(node, previous_macros)
 
-    def check_modified(self, old: Optional[SelectorTarget], new: SelectorTarget) -> bool:
+    # TODO check modifed_content and check_modified macro seems a bit redundent
+    def check_modified_content(self, old: Optional[SelectorTarget], new: SelectorTarget) -> bool:
         different_contents = not new.same_contents(old)  # type: ignore
         upstream_macro_change = self.check_macros_modified(new)
         return different_contents or upstream_macro_change
 
-    def check_modified_body(self, old: Optional[SelectorTarget], new: SelectorTarget) -> bool:
-        if hasattr(new, "same_body"):
-            return not new.same_body(old)  # type: ignore
-        else:
-            return False
-
-    def check_modified_configs(self, old: Optional[SelectorTarget], new: SelectorTarget) -> bool:
-        if hasattr(new, "same_config"):
-            return not new.same_config(old)  # type: ignore
-        else:
-            return False
-
-    def check_modified_persisted_descriptions(
-        self, old: Optional[SelectorTarget], new: SelectorTarget
-    ) -> bool:
-        if hasattr(new, "same_persisted_description"):
-            return not new.same_persisted_description(old)  # type: ignore
-        else:
-            return False
-
-    def check_modified_relation(
-        self, old: Optional[SelectorTarget], new: SelectorTarget
-    ) -> bool:
-        if hasattr(new, "same_database_representation"):
-            return not new.same_database_representation(old)  # type: ignore
-        else:
-            return False
-
     def check_modified_macros(self, _, new: SelectorTarget) -> bool:
         return self.check_macros_modified(new)
 
+    @staticmethod
+    def check_modified_factory(
+        compare_method: str
+    ) -> Callable[[Optional[SelectorTarget], SelectorTarget], bool]:
+        # get a function that compares two selector target based on compare method provided
+        def check_modified_things(old: Optional[SelectorTarget], new: SelectorTarget) -> bool:
+            if hasattr(new, compare_method):
+                # when old body does not exist or old and new are not the same
+                return not old or not getattr(new, compare_method)(old)  # type: ignore
+            else:
+                return False
+        return check_modified_things
+
     def check_new(self, old: Optional[SelectorTarget], new: SelectorTarget) -> bool:
         return old is None
 
@@ -527,14 +513,21 @@ class StateSelectorMethod(SelectorMethod):
 
         state_checks = {
             # it's new if there is no old version
-            'new': lambda old, _: old is None,
+            'new':
+                lambda old, _: old is None,
             # use methods defined above to compare properties of old + new
-            'modified': self.check_modified,
-            'modified.body': self.check_modified_body,
-            'modified.configs': self.check_modified_configs,
-            'modified.persisted_descriptions': self.check_modified_persisted_descriptions,
-            'modified.relation': self.check_modified_relation,
-            'modified.macros': self.check_modified_macros,
+            'modified':
+                self.check_modified_content,
+            'modified.body':
+                self.check_modified_factory('same_body'),
+            'modified.configs':
+                self.check_modified_factory('same_config'),
+            'modified.persisted_descriptions':
+                self.check_modified_factory('same_persisted_description'),
+            'modified.relation':
+                self.check_modified_factory('same_database_representation'),
+            'modified.macros':
+                self.check_modified_macros,
         }
         if selector in state_checks:
             checker = state_checks[selector]

core/dbt/helper_types.py

@@ -1,4 +1,8 @@
 # never name this package "types", or mypy will crash in ugly ways
+
+# necessary for annotating constructors
+from __future__ import annotations
+
 from dataclasses import dataclass
 from datetime import timedelta
 from pathlib import Path
@@ -9,6 +13,7 @@ from dbt.dataclass_schema import (
 )
 from hologram import FieldEncoder, JsonDict
 from mashumaro.types import SerializableType
+from typing import Callable, cast, Generic, Optional, TypeVar
 
 
 class Port(int, SerializableType):
@@ -93,3 +98,35 @@ dbtClassMixin.register_field_encoders({
 
 FQNPath = Tuple[str, ...]
 PathSet = AbstractSet[FQNPath]
+
+T = TypeVar('T')
+
+
+# A data type for representing lazily evaluated values.
+#
+# usage:
+# x = Lazy.defer(lambda: expensive_fn())
+# y = x.force()
+#
+# inspired by the purescript data type
+# https://pursuit.purescript.org/packages/purescript-lazy/5.0.0/docs/Data.Lazy
+@dataclass
+class Lazy(Generic[T]):
+    _f: Callable[[], T]
+    memo: Optional[T] = None
+
+    # constructor for lazy values
+    @classmethod
+    def defer(cls, f: Callable[[], T]) -> Lazy[T]:
+        return Lazy(f)
+
+    # workaround for open mypy issue:
+    # https://github.com/python/mypy/issues/6910
+    def _typed_eval_f(self) -> T:
+        return cast(Callable[[], T], getattr(self, "_f"))()
+
+    # evaluates the function if the value has not been memoized already
+    def force(self) -> T:
+        if self.memo is None:
+            self.memo = self._typed_eval_f()
+        return self.memo

core/dbt/include/README.md

@@ -0,0 +1 @@
+# Include README

core/dbt/include/global_project/docs/overview.md

@@ -35,7 +35,7 @@ Note that you can also right-click on models to interactively filter and explore
 
 ### More information
 
-- [What is dbt](https://docs.getdbt.com/docs/overview)?
+- [What is dbt](https://docs.getdbt.com/docs/introduction)?
 - Read the [dbt viewpoint](https://docs.getdbt.com/docs/viewpoint)
 - [Installation](https://docs.getdbt.com/docs/installation)
 - Join the [dbt Community](https://www.getdbt.com/community/) for questions and discussion

core/dbt/logger.py

@@ -424,7 +424,7 @@ class DelayedFileHandler(logbook.RotatingFileHandler, FormatterMixin):
             return
 
         make_log_dir_if_missing(log_dir)
-        log_path = os.path.join(log_dir, 'dbt.log.old')  # TODO hack for now
+        log_path = os.path.join(log_dir, 'dbt.log.legacy')  # TODO hack for now
         self._super_init(log_path)
         self._replay_buffered()
         self._log_path = log_path

core/dbt/main.py

@@ -36,7 +36,7 @@ from dbt.adapters.factory import reset_adapters, cleanup_connections
 
 import dbt.tracking
 
-from dbt.utils import ExitCodes
+from dbt.utils import ExitCodes, args_to_dict
 from dbt.config.profile import DEFAULT_PROFILES_DIR, read_user_config
 from dbt.exceptions import (
     InternalException,
@@ -140,7 +140,7 @@ def main(args=None):
             exit_code = e.code
 
         except BaseException as e:
-            fire_event(MainEncounteredError(e=e))
+            fire_event(MainEncounteredError(e=str(e)))
             fire_event(MainStackTrace(stack_trace=traceback.format_exc()))
             exit_code = ExitCodes.UnhandledError.value
 
@@ -205,7 +205,7 @@ def track_run(task):
         )
     except (NotImplementedException,
             FailedToConnectException) as e:
-        fire_event(MainEncounteredError(e=e))
+        fire_event(MainEncounteredError(e=str(e)))
         dbt.tracking.track_invocation_end(
             config=task.config, args=task.args, result_type="error"
         )
@@ -221,26 +221,24 @@ def track_run(task):
 def run_from_args(parsed):
     log_cache_events(getattr(parsed, 'log_cache_events', False))
 
-    # we can now use the logger for stdout
-    # set log_format in the logger
-    # if 'list' task: set stdout to WARN instead of INFO
-    level_override = parsed.cls.pre_init_hook(parsed)
-
-    fire_event(MainReportVersion(v=str(dbt.version.installed)))
-
     # this will convert DbtConfigErrors into RuntimeExceptions
     # task could be any one of the task objects
     task = parsed.cls.from_args(args=parsed)
-    fire_event(MainReportArgs(args=parsed))
 
+    # Set up logging
     log_path = None
     if task.config is not None:
         log_path = getattr(task.config, 'log_path', None)
-    # we can finally set the file logger up
     log_manager.set_path(log_path)
+    # if 'list' task: set stdout to WARN instead of INFO
+    level_override = parsed.cls.pre_init_hook(parsed)
     setup_event_logger(log_path or 'logs', level_override)
+
+    fire_event(MainReportVersion(v=str(dbt.version.installed)))
+    fire_event(MainReportArgs(args=args_to_dict(parsed)))
+
     if dbt.tracking.active_user is not None:  # mypy appeasement, always true
-        fire_event(MainTrackingUserState(dbt.tracking.active_user.state()))
+        fire_event(MainTrackingUserState(user_state=dbt.tracking.active_user.state()))
 
     results = None
 
@@ -1078,6 +1076,14 @@ def parse_args(args, cls=DBTArgumentParser):
         '''
     )
 
+    p.add_argument(
+        '--event-buffer-size',
+        dest='event_buffer_size',
+        help='''
+        Sets the max number of events to buffer in EVENT_HISTORY
+        '''
+    )
+
     subs = p.add_subparsers(title="Available sub-commands")
 
     base_subparser = _build_base_subparser()

core/dbt/parser/README.md

@@ -0,0 +1 @@
+# Parser README

core/dbt/parser/manifest.py

@@ -246,7 +246,7 @@ class ManifestLoader:
                     project_parser_files = self.partial_parser.get_parsing_files()
                     self.partially_parsing = True
                     self.manifest = self.saved_manifest
-                except Exception:
+                except Exception as exc:
                     # pp_files should still be the full set and manifest is new manifest,
                     # since get_parsing_files failed
                     fire_event(PartialParsingFullReparseBecauseOfError())
@@ -284,6 +284,9 @@ class ManifestLoader:
                         exc_info['full_reparse_reason'] = ReparseReason.exception
                         dbt.tracking.track_partial_parser(exc_info)
 
+                    if os.environ.get('DBT_PP_TEST'):
+                        raise exc
+
         if self.manifest._parsing_info is None:
             self.manifest._parsing_info = ParsingInfo()
 

core/dbt/parser/partial.py

@@ -315,7 +315,7 @@ class PartialParsing:
         if node.patch_path:
             file_id = node.patch_path
             # it might be changed...  then what?
-            if file_id not in self.file_diff['deleted']:
+            if file_id not in self.file_diff['deleted'] and file_id in self.saved_files:
                 # schema_files should already be updated
                 schema_file = self.saved_files[file_id]
                 dict_key = parse_file_type_to_key[source_file.parse_file_type]
@@ -375,7 +375,7 @@ class PartialParsing:
         for unique_id in unique_ids:
             if unique_id in self.saved_manifest.nodes:
                 node = self.saved_manifest.nodes[unique_id]
-                if node.resource_type == NodeType.Test:
+                if node.resource_type == NodeType.Test and node.test_node_type == 'generic':
                     # test nodes are handled separately. Must be removed from schema file
                     continue
                 file_id = node.file_id
@@ -435,7 +435,9 @@ class PartialParsing:
         self.check_for_special_deleted_macros(source_file)
         self.handle_macro_file_links(source_file, follow_references)
         file_id = source_file.file_id
-        self.deleted_manifest.files[file_id] = self.saved_files.pop(file_id)
+        # It's not clear when this file_id would not exist in saved_files
+        if file_id in self.saved_files:
+            self.deleted_manifest.files[file_id] = self.saved_files.pop(file_id)
 
     def check_for_special_deleted_macros(self, source_file):
         for unique_id in source_file.macros:
@@ -498,7 +500,9 @@ class PartialParsing:
         for unique_id in unique_ids:
             if unique_id in self.saved_manifest.nodes:
                 node = self.saved_manifest.nodes[unique_id]
-                if node.resource_type == NodeType.Test:
+                # Both generic tests from yaml files and singular tests have NodeType.Test
+                # so check for generic test.
+                if node.resource_type == NodeType.Test and node.test_node_type == 'generic':
                     schema_file_id = node.file_id
                     schema_file = self.saved_manifest.files[schema_file_id]
                     (key, name) = schema_file.get_key_and_name_for_test(node.unique_id)
@@ -670,8 +674,8 @@ class PartialParsing:
                     continue
                 elem = self.get_schema_element(new_yaml_dict[dict_key], name)
                 if elem:
-                    self.delete_schema_macro_patch(schema_file, macro)
-                    self.merge_patch(schema_file, dict_key, macro)
+                    self.delete_schema_macro_patch(schema_file, elem)
+                    self.merge_patch(schema_file, dict_key, elem)
 
         # exposures
         dict_key = 'exposures'

core/dbt/parser/schemas.py

@@ -960,10 +960,9 @@ class MacroPatchParser(NonSourceParser[UnparsedMacroUpdate, ParsedMacroPatch]):
         unique_id = f'macro.{patch.package_name}.{patch.name}'
         macro = self.manifest.macros.get(unique_id)
         if not macro:
-            warn_or_error(
-                f'WARNING: Found patch for macro "{patch.name}" '
-                f'which was not found'
-            )
+            msg = f'Found patch for macro "{patch.name}" ' \
+                  f'which was not found'
+            warn_or_error(msg, log_fmt=warning_tag('{}'))
             return
         if macro.patch_path:
             package_name, existing_file_path = macro.patch_path.split('://')

core/dbt/semver.py

@@ -1,5 +1,6 @@
 from dataclasses import dataclass
 import re
+import warnings
 from typing import List
 
 from packaging import version as packaging_version
@@ -145,10 +146,13 @@ class VersionSpecifier(VersionSpecification):
                     return 1
                 if b is None:
                     return -1
-            if packaging_version.parse(a) > packaging_version.parse(b):
-                return 1
-            elif packaging_version.parse(a) < packaging_version.parse(b):
-                return -1
+            # This suppresses the LegacyVersion deprecation warning
+            with warnings.catch_warnings():
+                warnings.simplefilter("ignore", category=DeprecationWarning)
+                if packaging_version.parse(a) > packaging_version.parse(b):
+                    return 1
+                elif packaging_version.parse(a) < packaging_version.parse(b):
+                    return -1
 
         equal = ((self.matcher == Matchers.GREATER_THAN_OR_EQUAL and
                   other.matcher == Matchers.LESS_THAN_OR_EQUAL) or

core/dbt/task/README.md

@@ -0,0 +1 @@
+# Task README

core/dbt/task/base.py

@@ -290,7 +290,7 @@ class BaseRunner(metaclass=ABCMeta):
             ctx.node._event_status['node_status'] = RunningStatus.Compiling
             fire_event(
                 NodeCompiling(
-                    report_node_data=ctx.node,
+                    node_info=ctx.node.node_info,
                     unique_id=ctx.node.unique_id,
                 )
             )
@@ -306,7 +306,7 @@ class BaseRunner(metaclass=ABCMeta):
                 ctx.node._event_status['node_status'] = RunningStatus.Executing
                 fire_event(
                     NodeExecuting(
-                        report_node_data=ctx.node,
+                        node_info=ctx.node.node_info,
                         unique_id=ctx.node.unique_id,
                     )
                 )
@@ -334,7 +334,7 @@ class BaseRunner(metaclass=ABCMeta):
             GenericExceptionOnRun(
                 build_path=self.node.build_path,
                 unique_id=self.node.unique_id,
-                exc=e
+                exc=str(e)  # TODO: unstring this when serialization is fixed
             )
         )
         fire_event(PrintDebugStackTrace())
@@ -448,7 +448,7 @@ class BaseRunner(metaclass=ABCMeta):
                         node_name=node_name,
                         index=self.node_index,
                         total=self.num_nodes,
-                        report_node_data=self.node
+                        node_info=self.node.node_info
                     )
                 )
 

core/dbt/task/clean.py

@@ -38,7 +38,7 @@ class CleanTask(BaseTask):
         """
         move_to_nearest_project_dir(self.args)
         if ('dbt_modules' in self.config.clean_targets and
-           self.config.packages_install_path != 'dbt_modules'):
+           self.config.packages_install_path not in self.config.clean_targets):
             deprecations.warn('install-packages-path')
         for path in self.config.clean_targets:
             fire_event(CheckCleanPath(path=path))

core/dbt/task/deps.py

@@ -10,7 +10,7 @@ from dbt.deps.resolver import resolve_packages
 from dbt.events.functions import fire_event
 from dbt.events.types import (
     DepsNoPackagesFound, DepsStartPackageInstall, DepsUpdateAvailable, DepsUTD,
-    DepsInstallInfo, DepsListSubdirectory, DepsNotifyUpdatesAvailable
+    DepsInstallInfo, DepsListSubdirectory, DepsNotifyUpdatesAvailable, EmptyLine
 )
 from dbt.clients import system
 
@@ -63,7 +63,7 @@ class DepsTask(BaseTask):
                 source_type = package.source_type()
                 version = package.get_version()
 
-                fire_event(DepsStartPackageInstall(package=package))
+                fire_event(DepsStartPackageInstall(package_name=package_name))
                 package.install(self.config, renderer)
                 fire_event(DepsInstallInfo(version_name=package.nice_version_name()))
                 if source_type == 'hub':
@@ -81,6 +81,7 @@ class DepsTask(BaseTask):
                     source_type=source_type,
                     version=version)
             if packages_to_upgrade:
+                fire_event(EmptyLine())
                 fire_event(DepsNotifyUpdatesAvailable(packages=packages_to_upgrade))
 
     @classmethod

core/dbt/task/freshness.py

@@ -41,7 +41,7 @@ class FreshnessRunner(BaseRunner):
                 description=description,
                 index=self.node_index,
                 total=self.num_nodes,
-                report_node_data=self.node
+                node_info=self.node.node_info
             )
         )
 
@@ -60,7 +60,7 @@ class FreshnessRunner(BaseRunner):
                     index=self.node_index,
                     total=self.num_nodes,
                     execution_time=result.execution_time,
-                    report_node_data=self.node
+                    node_info=self.node.node_info
                 )
             )
         elif result.status == FreshnessStatus.Error:
@@ -71,7 +71,7 @@ class FreshnessRunner(BaseRunner):
                     index=self.node_index,
                     total=self.num_nodes,
                     execution_time=result.execution_time,
-                    report_node_data=self.node
+                    node_info=self.node.node_info
                 )
             )
         elif result.status == FreshnessStatus.Warn:
@@ -82,7 +82,7 @@ class FreshnessRunner(BaseRunner):
                     index=self.node_index,
                     total=self.num_nodes,
                     execution_time=result.execution_time,
-                    report_node_data=self.node
+                    node_info=self.node.node_info
                 )
             )
         else:
@@ -93,7 +93,7 @@ class FreshnessRunner(BaseRunner):
                     index=self.node_index,
                     total=self.num_nodes,
                     execution_time=result.execution_time,
-                    report_node_data=self.node
+                    node_info=self.node.node_info
                 )
             )
 

core/dbt/task/init.py

@@ -14,6 +14,8 @@ from dbt import flags
 from dbt.version import _get_adapter_plugin_names
 from dbt.adapters.factory import load_plugin, get_include_paths
 
+from dbt.contracts.project import Name as ProjectName
+
 from dbt.events.functions import fire_event
 from dbt.events.types import (
     StarterProjectPath, ConfigFolderDirectory, NoSampleProfileFound, ProfileWrittenWithSample,
@@ -269,6 +271,16 @@ class InitTask(BaseTask):
         numeric_choice = click.prompt(prompt_msg, type=click.INT)
         return available_adapters[numeric_choice - 1]
 
+    def get_valid_project_name(self) -> str:
+        """Returns a valid project name, either from CLI arg or user prompt."""
+        name = self.args.project_name
+        while not ProjectName.is_valid(name):
+            if name:
+                click.echo(name + " is not a valid project name.")
+            name = click.prompt("Enter a name for your project (letters, digits, underscore)")
+
+        return name
+
     def run(self):
         """Entry point for the init task."""
         profiles_dir = flags.PROFILES_DIR
@@ -285,6 +297,8 @@ class InitTask(BaseTask):
             # just setup the user's profile.
             fire_event(SettingUpProfile())
             profile_name = self.get_profile_name_from_current_project()
+            if not self.check_if_can_write_profile(profile_name=profile_name):
+                return
             # If a profile_template.yml exists in the project root, that effectively
             # overrides the profile_template.yml for the given target.
             profile_template_path = Path("profile_template.yml")
@@ -296,8 +310,6 @@ class InitTask(BaseTask):
                     return
                 except Exception:
                     fire_event(InvalidProfileTemplateYAML())
-            if not self.check_if_can_write_profile(profile_name=profile_name):
-                return
             adapter = self.ask_for_adapter_choice()
             self.create_profile_from_target(
                 adapter, profile_name=profile_name
@@ -306,11 +318,7 @@ class InitTask(BaseTask):
 
         # When dbt init is run outside of an existing project,
         # create a new project and set up the user's profile.
-        project_name = self.args.project_name
-        if project_name is None:
-            # If project name is not provided,
-            # ask the user which project name they'd like to use.
-            project_name = click.prompt("What is the desired project name?")
+        project_name = self.get_valid_project_name()
         project_path = Path(project_name)
         if project_path.exists():
             fire_event(ProjectNameAlreadyExists(name=project_name))

core/dbt/task/printer.py

@@ -65,6 +65,8 @@ def print_run_status_line(results) -> None:
         stats[result_type] += 1
         stats['total'] += 1
 
+    with TextOnly():
+        fire_event(EmptyLine())
     fire_event(StatsLine(stats=stats))
 
 

core/dbt/task/run.py

@@ -11,7 +11,7 @@ from .printer import (
     print_run_end_messages,
     get_counts,
 )
-
+from datetime import datetime
 from dbt import tracking
 from dbt import utils
 from dbt.adapters.base import BaseRelation
@@ -21,7 +21,7 @@ from dbt.contracts.graph.compiled import CompileResultNode
 from dbt.contracts.graph.manifest import WritableManifest
 from dbt.contracts.graph.model_config import Hook
 from dbt.contracts.graph.parsed import ParsedHookNode
-from dbt.contracts.results import NodeStatus, RunResult, RunStatus
+from dbt.contracts.results import NodeStatus, RunResult, RunStatus, RunningStatus
 from dbt.exceptions import (
     CompilationException,
     InternalException,
@@ -178,7 +178,7 @@ class ModelRunner(CompileRunner):
                 description=self.describe_node(),
                 index=self.node_index,
                 total=self.num_nodes,
-                report_node_data=self.node
+                node_info=self.node.node_info
             )
         )
 
@@ -192,7 +192,7 @@ class ModelRunner(CompileRunner):
                     index=self.node_index,
                     total=self.num_nodes,
                     execution_time=result.execution_time,
-                    report_node_data=self.node
+                    node_info=self.node.node_info
                 )
             )
         else:
@@ -203,7 +203,7 @@ class ModelRunner(CompileRunner):
                     index=self.node_index,
                     total=self.num_nodes,
                     execution_time=result.execution_time,
-                    report_node_data=self.node
+                    node_info=self.node.node_info
                 )
             )
 
@@ -342,6 +342,8 @@ class RunTask(CompileTask):
         finishctx = TimestampNamed('node_finished_at')
 
         for idx, hook in enumerate(ordered_hooks, start=1):
+            hook._event_status['started_at'] = datetime.utcnow().isoformat()
+            hook._event_status['node_status'] = RunningStatus.Started
             sql = self.get_hook_sql(adapter, hook, idx, num_hooks,
                                     extra_context)
 
@@ -355,31 +357,36 @@ class RunTask(CompileTask):
                             statement=hook_text,
                             index=idx,
                             total=num_hooks,
-                            truncate=True,
-                            report_node_data=hook
+                            node_info=hook.node_info
                         )
                     )
 
-                status = 'OK'
-
                 with Timer() as timer:
                     if len(sql.strip()) > 0:
-                        status, _ = adapter.execute(sql, auto_begin=False,
-                                                    fetch=False)
-                self.ran_hooks.append(hook)
+                        response, _ = adapter.execute(sql, auto_begin=False, fetch=False)
+                        status = response._message
+                    else:
+                        status = 'OK'
 
+                self.ran_hooks.append(hook)
+                hook._event_status['finished_at'] = datetime.utcnow().isoformat()
                 with finishctx, DbtModelState({'node_status': 'passed'}):
+                    hook._event_status['node_status'] = RunStatus.Success
                     fire_event(
                         PrintHookEndLine(
                             statement=hook_text,
-                            status=str(status),
+                            status=status,
                             index=idx,
                             total=num_hooks,
                             execution_time=timer.elapsed,
-                            truncate=True,
-                            report_node_data=hook
+                            node_info=hook.node_info
                         )
                     )
+            # `_event_status` dict is only used for logging.  Make sure
+            # it gets deleted when we're done with it
+            del hook._event_status["started_at"]
+            del hook._event_status["finished_at"]
+            del hook._event_status["node_status"]
 
         self._total_executed += len(ordered_hooks)
 
@@ -392,7 +399,7 @@ class RunTask(CompileTask):
         try:
             self.run_hooks(adapter, hook_type, extra_context)
         except RuntimeException:
-            fire_event(DatabaseErrorRunning(hook_type))
+            fire_event(DatabaseErrorRunning(hook_type=hook_type.value))
             raise
 
     def print_results_line(self, results, execution_time):

core/dbt/task/runnable.py

@@ -6,7 +6,6 @@ from concurrent.futures import as_completed
 from datetime import datetime
 from multiprocessing.dummy import Pool as ThreadPool
 from typing import Optional, Dict, List, Set, Tuple, Iterable, AbstractSet
-from pathlib import PosixPath, WindowsPath
 
 from .printer import (
     print_run_result_error,
@@ -56,6 +55,7 @@ from dbt.parser.manifest import ManifestLoader
 import dbt.exceptions
 from dbt import flags
 import dbt.utils
+from dbt.ui import warning_tag
 
 RESULT_FILE_NAME = 'run_results.json'
 MANIFEST_FILE_NAME = 'manifest.json'
@@ -208,14 +208,14 @@ class GraphRunnableTask(ManifestTask):
         with RUNNING_STATE, uid_context:
             startctx = TimestampNamed('node_started_at')
             index = self.index_offset(runner.node_index)
-            runner.node._event_status['dbt_internal__started_at'] = datetime.utcnow().isoformat()
+            runner.node._event_status['started_at'] = datetime.utcnow().isoformat()
             runner.node._event_status['node_status'] = RunningStatus.Started
             extended_metadata = ModelMetadata(runner.node, index)
 
             with startctx, extended_metadata:
                 fire_event(
                     NodeStart(
-                        report_node_data=runner.node,
+                        node_info=runner.node.node_info,
                         unique_id=runner.node.unique_id,
                     )
                 )
@@ -224,22 +224,21 @@ class GraphRunnableTask(ManifestTask):
                 result = runner.run_with_hooks(self.manifest)
                 status = runner.get_result_status(result)
                 runner.node._event_status['node_status'] = result.status
-                runner.node._event_status['dbt_internal__finished_at'] = \
-                    datetime.utcnow().isoformat()
+                runner.node._event_status['finished_at'] = datetime.utcnow().isoformat()
             finally:
                 finishctx = TimestampNamed('finished_at')
                 with finishctx, DbtModelState(status):
                     fire_event(
                         NodeFinished(
-                            report_node_data=runner.node,
+                            node_info=runner.node.node_info,
                             unique_id=runner.node.unique_id,
-                            run_result=result
+                            run_result=result.to_dict(),
                         )
                     )
             # `_event_status` dict is only used for logging.  Make sure
             # it gets deleted when we're done with it
-            del runner.node._event_status["dbt_internal__started_at"]
-            del runner.node._event_status["dbt_internal__finished_at"]
+            del runner.node._event_status["started_at"]
+            del runner.node._event_status["finished_at"]
             del runner.node._event_status["node_status"]
 
         fail_fast = flags.FAIL_FAST
@@ -359,7 +358,7 @@ class GraphRunnableTask(ManifestTask):
         adapter = get_adapter(self.config)
 
         if not adapter.is_cancelable():
-            fire_event(QueryCancelationUnsupported(type=adapter.type))
+            fire_event(QueryCancelationUnsupported(type=adapter.type()))
         else:
             with adapter.connection_named('master'):
                 for conn_name in adapter.cancel_open_connections():
@@ -377,10 +376,8 @@ class GraphRunnableTask(ManifestTask):
         num_threads = self.config.threads
         target_name = self.config.target_name
 
-        text = "Concurrency: {} threads (target='{}')"
-        concurrency_line = text.format(num_threads, target_name)
         with NodeCount(self.num_nodes):
-            fire_event(ConcurrencyLine(concurrency_line=concurrency_line))
+            fire_event(ConcurrencyLine(num_threads=num_threads, target_name=target_name))
         with TextOnly():
             fire_event(EmptyLine())
 
@@ -461,8 +458,11 @@ class GraphRunnableTask(ManifestTask):
             )
 
         if len(self._flattened_nodes) == 0:
-            warn_or_error("\nWARNING: Nothing to do. Try checking your model "
-                          "configs and model specification args")
+            with TextOnly():
+                fire_event(EmptyLine())
+            msg = "Nothing to do. Try checking your model " \
+                  "configs and model specification args"
+            warn_or_error(msg, log_fmt=warning_tag('{}'))
             result = self.get_result(
                 results=[],
                 generated_at=datetime.utcnow(),
@@ -590,38 +590,8 @@ class GraphRunnableTask(ManifestTask):
             results=results,
             elapsed_time=elapsed_time,
             generated_at=generated_at,
-            args=self.args_to_dict(),
+            args=dbt.utils.args_to_dict(self.args),
         )
 
-    def args_to_dict(self):
-        var_args = vars(self.args).copy()
-        # update the args with the flags, which could also come from environment
-        # variables or user_config
-        flag_dict = flags.get_flag_dict()
-        var_args.update(flag_dict)
-        dict_args = {}
-        # remove args keys that clutter up the dictionary
-        for key in var_args:
-            if key == 'cls':
-                continue
-            if var_args[key] is None:
-                continue
-            # TODO: add more default_false_keys
-            default_false_keys = (
-                'debug', 'full_refresh', 'fail_fast', 'warn_error',
-                'single_threaded', 'log_cache_events',
-                'use_experimental_parser',
-            )
-            if key in default_false_keys and var_args[key] is False:
-                continue
-            if key == 'vars' and var_args[key] == '{}':
-                continue
-            # this was required for a test case
-            if (isinstance(var_args[key], PosixPath) or
-                    isinstance(var_args[key], WindowsPath)):
-                var_args[key] = str(var_args[key])
-            dict_args[key] = var_args[key]
-        return dict_args
-
     def task_end_messages(self, results):
         print_run_end_messages(results)

core/dbt/task/seed.py

@@ -11,7 +11,7 @@ from dbt.graph import ResourceTypeSelector
 from dbt.logger import TextOnly
 from dbt.events.functions import fire_event
 from dbt.events.types import (
-    SeedHeader, SeedHeaderSeperator, EmptyLine, PrintSeedErrorResultLine,
+    SeedHeader, SeedHeaderSeparator, EmptyLine, PrintSeedErrorResultLine,
     PrintSeedResultLine, PrintStartLine
 )
 from dbt.node_types import NodeType
@@ -28,7 +28,7 @@ class SeedRunner(ModelRunner):
                 description=self.describe_node(),
                 index=self.node_index,
                 total=self.num_nodes,
-                report_node_data=self.node
+                node_info=self.node.node_info
             )
         )
 
@@ -52,7 +52,7 @@ class SeedRunner(ModelRunner):
                     execution_time=result.execution_time,
                     schema=self.node.schema,
                     relation=model.alias,
-                    report_node_data=model
+                    node_info=model.node_info
                 )
             )
         else:
@@ -64,7 +64,7 @@ class SeedRunner(ModelRunner):
                     execution_time=result.execution_time,
                     schema=self.node.schema,
                     relation=model.alias,
-                    report_node_data=model
+                    node_info=model.node_info
                 )
             )
 
@@ -109,7 +109,7 @@ class SeedTask(RunTask):
         with TextOnly():
             fire_event(EmptyLine())
         fire_event(SeedHeader(header=header))
-        fire_event(SeedHeaderSeperator(len_header=len(header)))
+        fire_event(SeedHeaderSeparator(len_header=len(header)))
 
         rand_table.print_table(max_rows=10, max_columns=None)
         with TextOnly():

core/dbt/task/serve.py

@@ -6,7 +6,7 @@ from dbt.include.global_project import DOCS_INDEX_FILE_PATH
 from http.server import SimpleHTTPRequestHandler
 from socketserver import TCPServer
 from dbt.events.functions import fire_event
-from dbt.events.types import ServingDocsPort, ServingDocsAccessInfo, ServingDocsExitInfo
+from dbt.events.types import ServingDocsPort, ServingDocsAccessInfo, ServingDocsExitInfo, EmptyLine
 
 from dbt.task.base import ConfiguredTask
 
@@ -22,6 +22,8 @@ class ServeTask(ConfiguredTask):
 
         fire_event(ServingDocsPort(address=address, port=port))
         fire_event(ServingDocsAccessInfo(port=port))
+        fire_event(EmptyLine())
+        fire_event(EmptyLine())
         fire_event(ServingDocsExitInfo())
 
         # mypy doesn't think SimpleHTTPRequestHandler is ok here, but it is

core/dbt/task/snapshot.py

@@ -24,7 +24,7 @@ class SnapshotRunner(ModelRunner):
                     index=self.node_index,
                     total=self.num_nodes,
                     execution_time=result.execution_time,
-                    report_node_data=model
+                    node_info=model.node_info
                 )
             )
         else:
@@ -36,7 +36,7 @@ class SnapshotRunner(ModelRunner):
                     index=self.node_index,
                     total=self.num_nodes,
                     execution_time=result.execution_time,
-                    report_node_data=model
+                    node_info=model.node_info
                 )
             )
 

core/dbt/task/test.py

@@ -75,7 +75,7 @@ class TestRunner(CompileRunner):
                     index=self.node_index,
                     num_models=self.num_nodes,
                     execution_time=result.execution_time,
-                    report_node_data=model
+                    node_info=model.node_info
                 )
             )
         elif result.status == TestStatus.Pass:
@@ -85,7 +85,7 @@ class TestRunner(CompileRunner):
                     index=self.node_index,
                     num_models=self.num_nodes,
                     execution_time=result.execution_time,
-                    report_node_data=model
+                    node_info=model.node_info
                 )
             )
         elif result.status == TestStatus.Warn:
@@ -96,7 +96,7 @@ class TestRunner(CompileRunner):
                     num_models=self.num_nodes,
                     execution_time=result.execution_time,
                     failures=result.failures,
-                    report_node_data=model
+                    node_info=model.node_info
                 )
             )
         elif result.status == TestStatus.Fail:
@@ -107,7 +107,7 @@ class TestRunner(CompileRunner):
                     num_models=self.num_nodes,
                     execution_time=result.execution_time,
                     failures=result.failures,
-                    report_node_data=model
+                    node_info=model.node_info
                 )
             )
         else:
@@ -119,7 +119,7 @@ class TestRunner(CompileRunner):
                 description=self.describe_node(),
                 index=self.node_index,
                 total=self.num_nodes,
-                report_node_data=self.node
+                node_info=self.node.node_info
             )
         )
 

core/dbt/ui.py

@@ -66,6 +66,4 @@ def line_wrap_message(
 
 
 def warning_tag(msg: str) -> str:
-    # no longer needed, since new logging includes colorized log level
-    # return f'[{yellow("WARNING")}]: {msg}'
-    return msg
+    return f'[{yellow("WARNING")}]: {msg}'

core/dbt/utils.py

@@ -10,12 +10,15 @@ import jinja2
 import json
 import os
 import requests
+from tarfile import ReadError
 import time
+from pathlib import PosixPath, WindowsPath
 
 from contextlib import contextmanager
 from dbt.exceptions import ConnectionException
 from dbt.events.functions import fire_event
 from dbt.events.types import RetryExternalCall
+from dbt import flags
 from enum import Enum
 from typing_extensions import Protocol
 from typing import (
@@ -598,7 +601,9 @@ class MultiDict(Mapping[str, Any]):
 
 def _connection_exception_retry(fn, max_attempts: int, attempt: int = 0):
     """Attempts to run a function that makes an external call, if the call fails
-    on a connection error or timeout, it will be tried up to 5 more times.
+    on a connection error, timeout or decompression issue, it will be tried up to 5 more times.
+    See https://github.com/dbt-labs/dbt-core/issues/4579 for context on this decompression issues
+    specifically.
     """
     try:
         return fn()
@@ -606,6 +611,7 @@ def _connection_exception_retry(fn, max_attempts: int, attempt: int = 0):
         requests.exceptions.ConnectionError,
         requests.exceptions.Timeout,
         requests.exceptions.ContentDecodingError,
+        ReadError,
     ) as exc:
         if attempt <= max_attempts - 1:
             fire_event(RetryExternalCall(attempt=attempt, max=max_attempts))
@@ -613,3 +619,40 @@ def _connection_exception_retry(fn, max_attempts: int, attempt: int = 0):
             _connection_exception_retry(fn, max_attempts, attempt + 1)
         else:
             raise ConnectionException('External connection exception occurred: ' + str(exc))
+
+
+# This is used to serialize the args in the run_results and in the logs.
+# We do this separately because there are a few fields that don't serialize,
+# i.e. PosixPath, WindowsPath, and types. It also includes args from both
+# cli args and flags, which is more complete than just the cli args.
+# If new args are added that are false by default (particularly in the
+# global options) they should be added to the 'default_false_keys' list.
+def args_to_dict(args):
+    var_args = vars(args).copy()
+    # update the args with the flags, which could also come from environment
+    # variables or user_config
+    flag_dict = flags.get_flag_dict()
+    var_args.update(flag_dict)
+    dict_args = {}
+    # remove args keys that clutter up the dictionary
+    for key in var_args:
+        if key == 'cls':
+            continue
+        if var_args[key] is None:
+            continue
+        # TODO: add more default_false_keys
+        default_false_keys = (
+            'debug', 'full_refresh', 'fail_fast', 'warn_error',
+            'single_threaded', 'log_cache_events', 'store_failures',
+            'use_experimental_parser',
+        )
+        if key in default_false_keys and var_args[key] is False:
+            continue
+        if key == 'vars' and var_args[key] == '{}':
+            continue
+        # this was required for a test case
+        if (isinstance(var_args[key], PosixPath) or
+                isinstance(var_args[key], WindowsPath)):
+            var_args[key] = str(var_args[key])
+        dict_args[key] = var_args[key]
+    return dict_args

core/dbt/version.py

@@ -11,7 +11,7 @@ import dbt.exceptions
 import dbt.semver
 
 
-PYPI_VERSION_URL = 'https://pypi.org/pypi/dbt/json'
+PYPI_VERSION_URL = 'https://pypi.org/pypi/dbt-core/json'
 
 
 def get_latest_version():
@@ -96,5 +96,5 @@ def _get_dbt_plugins_info():
         yield plugin_name, mod.version
 
 
-__version__ = '1.0.0rc3'
+__version__ = '1.0.1'
 installed = get_installed_version()

core/scripts/create_adapter_plugins.py

@@ -284,12 +284,12 @@ def parse_args(argv=None):
     parser.add_argument('adapter')
     parser.add_argument('--title-case', '-t', default=None)
     parser.add_argument('--dependency', action='append')
-    parser.add_argument('--dbt-core-version', default='1.0.0rc3')
+    parser.add_argument('--dbt-core-version', default='1.0.1')
     parser.add_argument('--email')
     parser.add_argument('--author')
     parser.add_argument('--url')
     parser.add_argument('--sql', action='store_true')
-    parser.add_argument('--package-version', default='1.0.0rc3')
+    parser.add_argument('--package-version', default='1.0.1')
     parser.add_argument('--project-version', default='1.0')
     parser.add_argument(
         '--no-dependency', action='store_false', dest='set_dependency'

core/setup.py

@@ -2,9 +2,9 @@
 import os
 import sys
 
-if sys.version_info < (3, 7):
+if sys.version_info < (3, 7, 2):
     print('Error: dbt does not support this version of Python.')
-    print('Please upgrade to Python 3.7 or higher.')
+    print('Please upgrade to Python 3.7.2 or higher.')
     sys.exit(1)
 
 
@@ -25,7 +25,7 @@ with open(os.path.join(this_directory, 'README.md')) as f:
 
 
 package_name = "dbt-core"
-package_version = "1.0.0rc3"
+package_version = "1.0.1"
 description = """With dbt, data analysts and engineers can build analytics \
 the way engineers build applications."""
 
@@ -85,5 +85,5 @@ setup(
         'Programming Language :: Python :: 3.8',
         'Programming Language :: Python :: 3.9',
     ],
-    python_requires=">=3.7",
+    python_requires=">=3.7.2",
 )

docker-compose.yml

@@ -1,3 +1,8 @@
+##
+#  This compose file is used for local development and adapter testing only.
+#  See `/docker` for a generic and production-ready docker file
+##
+
 version: "3.5"
 services:
   database:

docker/Dockerfile

@@ -1,34 +1,134 @@
-ARG BASE_IMAGE=python:3.8-slim-bullseye
+##
+#  Generic dockerfile for dbt image building.  
+#  See README for operational details
+##
 
-FROM $BASE_IMAGE
-ARG BASE_REQUIREMENTS_SRC_PATH
-ARG WHEEL_REQUIREMENTS_SRC_PATH
-ARG DIST_PATH
+# Top level build args
+ARG build_for=linux/amd64
 
+##
+# base image (abstract)
+##
+FROM --platform=$build_for python:3.9.9-slim-bullseye as base
+
+# N.B. The refs updated automagically every release via bumpversion
+# N.B. dbt-postgres is currently found in the core codebase so a value of dbt-core@<some_version> is correct
+
+ARG dbt_core_ref=dbt-core@v1.0.0
+ARG dbt_postgres_ref=dbt-core@v1.0.0
+ARG dbt_redshift_ref=dbt-redshift@v1.0.0
+ARG dbt_bigquery_ref=dbt-bigquery@v1.0.0
+ARG dbt_snowflake_ref=dbt-snowflake@v1.0.0
+ARG dbt_spark_ref=dbt-spark@v1.0.0
+# special case args
+ARG dbt_spark_version=all
+ARG dbt_third_party
+
+# System setup
 RUN apt-get update \
   && apt-get dist-upgrade -y \
   && apt-get install -y --no-install-recommends \
-  git \
-  ssh-client \
-  software-properties-common \
-  make \
-  build-essential \
-  ca-certificates \
-  libpq-dev \
+    git \
+    ssh-client \
+    software-properties-common \
+    make \
+    build-essential \
+    ca-certificates \
+    libpq-dev \
   && apt-get clean \
-  && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+  && rm -rf \
+    /var/lib/apt/lists/* \
+    /tmp/* \
+    /var/tmp/*
 
-RUN echo BASE_REQUIREMENTS_SRC_PATH=$BASE_REQUIREMENTS_SRC_PATH
-RUN echo WHEEL_REQUIREMENTS_SRC_PATH=$WHEEL_REQUIREMENTS_SRC_PATH
-RUN echo DIST_PATH=$DIST_PATH
-COPY $BASE_REQUIREMENTS_SRC_PATH ./requirements.txt
-COPY $WHEEL_REQUIREMENTS_SRC_PATH ./wheel_requirements.txt
-COPY $DIST_PATH ./dist
-RUN pip install --upgrade pip setuptools
-RUN pip install --requirement ./requirements.txt
-RUN pip install --requirement ./wheel_requirements.txt
+# Env vars
 ENV PYTHONIOENCODING=utf-8
-ENV LANG C.UTF-8
-WORKDIR /usr/app
+ENV LANG=C.UTF-8
+
+# Update python
+RUN python -m pip install --upgrade pip setuptools wheel --no-cache-dir
+
+# Set docker basics
+WORKDIR /usr/app/dbt/
 VOLUME /usr/app
 ENTRYPOINT ["dbt"]
+
+##
+# dbt-core
+##
+FROM base as dbt-core
+RUN python -m pip install --no-cache-dir "git+https://github.com/dbt-labs/${dbt_core_ref}#egg=dbt-core&subdirectory=core"
+
+##
+# dbt-postgres
+##
+FROM base as dbt-postgres
+RUN python -m pip install --no-cache-dir "git+https://github.com/dbt-labs/${dbt_postgres_ref}#egg=dbt-postgres&subdirectory=plugins/postgres"
+
+
+##
+# dbt-redshift
+##
+FROM base as dbt-redshift
+RUN python -m pip install --no-cache-dir "git+https://github.com/dbt-labs/${dbt_redshift_ref}#egg=dbt-redshift"
+
+
+##
+# dbt-bigquery
+##
+FROM base as dbt-bigquery
+RUN python -m pip install --no-cache-dir "git+https://github.com/dbt-labs/${dbt_bigquery_ref}#egg=dbt-bigquery"
+
+
+##
+# dbt-snowflake
+##
+FROM base as dbt-snowflake
+RUN python -m pip install --no-cache-dir "git+https://github.com/dbt-labs/${dbt_snowflake_ref}#egg=dbt-snowflake"
+
+##
+# dbt-spark
+##
+FROM base as dbt-spark
+RUN apt-get update \
+  && apt-get dist-upgrade -y \
+  && apt-get install -y --no-install-recommends \
+    python-dev \
+    libsasl2-dev \
+    gcc \
+    unixodbc-dev \
+  && apt-get clean \
+  && rm -rf \
+    /var/lib/apt/lists/* \
+    /tmp/* \
+    /var/tmp/*
+RUN python -m pip install --no-cache-dir "git+https://github.com/dbt-labs/${dbt_spark_ref}#egg=dbt-spark[${dbt_spark_version}]"
+
+
+##
+# dbt-third-party
+##
+FROM dbt-core as dbt-third-party
+RUN python -m pip install --no-cache-dir "${dbt_third_party}"
+
+##
+# dbt-all
+##
+FROM base as dbt-all
+RUN apt-get update \
+  && apt-get dist-upgrade -y \
+  && apt-get install -y --no-install-recommends \
+    python-dev \
+    libsasl2-dev \
+    gcc \
+    unixodbc-dev \
+  && apt-get clean \
+  && rm -rf \
+    /var/lib/apt/lists/* \
+    /tmp/* \
+    /var/tmp/*
+  RUN python -m pip install --no-cache "git+https://github.com/dbt-labs/${dbt_redshift_ref}#egg=dbt-redshift"
+  RUN python -m pip install --no-cache "git+https://github.com/dbt-labs/${dbt_bigquery_ref}#egg=dbt-bigquery"
+  RUN python -m pip install --no-cache "git+https://github.com/dbt-labs/${dbt_snowflake_ref}#egg=dbt-snowflake"
+  RUN python -m pip install --no-cache "git+https://github.com/dbt-labs/${dbt_spark_ref}#egg=dbt-spark[${dbt_spark_version}]"
+  RUN python -m pip install --no-cache "git+https://github.com/dbt-labs/${dbt_postgres_ref}#egg=dbt-postgres&subdirectory=plugins/postgres"

docker/README.md

@@ -0,0 +1,106 @@
+# Docker for dbt
+This docker file is suitable for building dbt Docker images locally or using with CI/CD to automate populating a container registry.
+
+
+## Building an image:
+This Dockerfile can create images for the following targets, each named after the database they support:
+* `dbt-core` _(no db-adapter support)_
+* `dbt-postgres`
+* `dbt-redshift`
+* `dbt-bigquery`
+* `dbt-snowflake`
+* `dbt-spark`
+* `dbt-third-party` _(requires additional build-arg)_
+* `dbt-all` _(installs all of the above in a single image)_
+
+In order to build a new image, run the following docker command.
+```
+docker build --tag <your_image_name>  --target <target_name> <path/to/dockerfile>
+```
+By default the images will be populated with the most recent release of `dbt-core` and whatever database adapter you select.  If you need to use a different version you can specify it by git ref using the `--build-arg` flag:
+```
+docker build --tag <your_image_name> \
+  --target <target_name> \
+  --build-arg <arg_name>=<git_ref> \
+  <path/to/dockerfile>
+```
+valid arg names for versioning are:
+* `dbt_core_ref`
+* `dbt_postgres_ref`
+* `dbt_redshift_ref`
+* `dbt_bigquery_ref`
+* `dbt_snowflake_ref`
+* `dbt_spark_ref`
+
+> Note: Only overide a _single_ build arg for each build. Using multiple overides may lead to a non-functioning image.
+
+If you wish to build an image with a third-party adapter you can use the `dbt-third-party` target.  This target requires you provide a path to the adapter that can be processed by `pip` by using the `dbt_third_party` build arg:
+```
+docker build --tag <your_image_name> \
+  --target dbt-third-party \ 
+  --build-arg dbt_third_party=<pip_parsable_install_string> \ 
+  <path/to/dockerfile>
+```
+
+### Examples:
+To build an image named "my-dbt" that supports redshift using the latest releases:
+```
+cd dbt-core/docker
+docker build --tag my-dbt  --target dbt-redshift .
+```
+
+To build an image named "my-other-dbt" that supports bigquery using `dbt-core` version 0.21.latest and the bigquery adapter version 1.0.0b1:
+```
+cd dbt-core/docker
+docker build \
+  --tag my-other-dbt  \
+  --target dbt-bigquery \
+  --build-arg dbt_bigquery_ref=dbt-bigquery@v1.0.0b1 \
+  --build-arg dbt_core_ref=dbt-core@0.21.latest  \
+ .
+```
+
+To build an image named "my-third-party-dbt" that uses [Materilize third party adapter](https://github.com/MaterializeInc/materialize/tree/main/misc/dbt-materialize) and the latest release of `dbt-core`:
+```
+cd dbt-core/docker
+docker build --tag my-third-party-dbt \
+  --target dbt-third-party \
+  --build-arg dbt_third_party=dbt-materialize \
+  .
+```
+
+
+## Special cases
+There are a few special cases worth noting:
+* The `dbt-spark` database adapter comes in three different versions named `PyHive`, `ODBC`, and the default `all`.  If you wish to overide this you can use the `--build-arg` flag with the value of `dbt_spark_version=<version_name>`.  See the [docs](https://docs.getdbt.com/reference/warehouse-profiles/spark-profile) for more information.
+
+* The `dbt-postgres` database adapter is released as part of the `dbt-core` codebase.  If you wish to overide the version used, make sure you use the gitref for `dbt-core`: 
+```
+docker build --tag my_dbt \
+  --target dbt-postgres \
+  --build-arg dbt_postgres_ref=dbt-core@1.0.0b1 \
+  <path/to/dockerfile> \
+  ```
+
+* If you need to build against another architecture (linux/arm64 in this example) you can overide the `build_for` build arg:
+```
+docker build --tag my_dbt \
+  --target dbt-postgres \
+  --build-arg build_for=linux/arm64 \
+  <path/to/dockerfile> \
+  ```
+Supported architectures can be found in the python docker [dockerhub page](https://hub.docker.com/_/python).
+
+## Running an image in a container:
+The `ENTRYPOINT` for this Dockerfile is the command `dbt` so you can bind-mount your project to `/usr/app` and use dbt as normal:
+```
+docker run \
+--network=host
+--mount type=bind,source=path/to/project,target=/usr/app \
+--mount type=bind,source=path/to/profiles.yml,target=/root/.dbt/ \
+my-dbt \
+ls
+```
+> Notes: 
+> * Bind-mount sources _must_ be an absolute path
+> * You may need to make adjustments to the docker networking setting depending on the specifics of your data warehouse/database host.

docker/requirements/requirements.txt

@@ -1,44 +0,0 @@
-agate==1.6.3
-attrs==21.2.0
-Babel==2.9.1
-certifi==2021.10.8
-cffi==1.15.0
-charset-normalizer==2.0.8
-click==8.0.3
-colorama==0.4.4
-dbt-core==1.0.0rc3
-dbt-extractor==0.4.0
-dbt-postgres==1.0.0rc3
-future==0.18.2
-hologram==0.0.14
-idna==3.3
-importlib-metadata==4.8.2
-isodate==0.6.0
-Jinja2==2.11.3
-jsonschema==3.1.1
-leather==0.3.4
-Logbook==1.5.3
-MarkupSafe==2.0.1
-mashumaro==2.9
-minimal-snowplow-tracker==0.0.2
-msgpack==1.0.3
-networkx==2.6.3
-packaging==21.3
-parsedatetime==2.4
-psycopg2-binary==2.9.2
-pycparser==2.21
-pyparsing==3.0.6
-pyrsistent==0.18.0
-python-dateutil==2.8.2
-python-slugify==5.0.2
-pytimeparse==1.1.8
-pytz==2021.3
-PyYAML==6.0
-requests==2.26.0
-six==1.16.0
-sqlparse==0.4.2
-text-unidecode==1.3
-typing-extensions==3.10.0.2
-urllib3==1.26.7
-Werkzeug==2.0.2
-zipp==3.6.0

docker/test.sh

@@ -0,0 +1,136 @@
+# - VERY rudimentary test script to run latest + specific branch image builds and test them all by running `--version`
+# TODO: create a real test suite
+
+clear \
+&& echo "\n\n"\
+"###################################\n"\
+"##### Testing dbt-core latest #####\n"\
+"###################################\n"\
+&& docker build --tag dbt-core \
+  --target dbt-core \
+  docker \
+&& docker run dbt-core --version \
+\
+&& echo "\n\n"\
+"####################################\n"\
+"##### Testing dbt-core-1.0.0b1 #####\n"\
+"####################################\n"\
+&& docker build --tag dbt-core-1.0.0b1 \
+  --target dbt-core \
+  --build-arg dbt_core_ref=dbt-core@v1.0.0b1 \
+  docker \
+&& docker run dbt-core-1.0.0b1 --version \
+\
+&& echo "\n\n"\
+"#######################################\n"\
+"##### Testing dbt-postgres latest #####\n"\
+"#######################################\n"\
+&& docker build --tag dbt-postgres \
+  --target dbt-postgres \
+  docker \
+&& docker run dbt-postgres --version \
+\
+&& echo "\n\n"\
+"########################################\n"\
+"##### Testing dbt-postgres-1.0.0b1 #####\n"\
+"########################################\n"\
+&& docker build --tag dbt-postgres-1.0.0b1 \
+  --target dbt-postgres \
+  --build-arg dbt_postgres_ref=dbt-core@v1.0.0b1 \
+  docker \
+&& docker run dbt-postgres-1.0.0b1 --version \
+\
+&& echo "\n\n"\
+"#######################################\n"\
+"##### Testing dbt-redshift latest #####\n"\
+"#######################################\n"\
+&& docker build --tag dbt-redshift \
+  --target dbt-redshift \
+  docker \
+&& docker run dbt-redshift --version \
+\
+&& echo "\n\n"\
+"########################################\n"\
+"##### Testing dbt-redshift-1.0.0b1 #####\n"\
+"########################################\n"\
+&& docker build --tag dbt-redshift-1.0.0b1 \
+  --target dbt-redshift \
+  --build-arg dbt_redshift_ref=dbt-redshift@v1.0.0b1 \
+  docker \
+&& docker run dbt-redshift-1.0.0b1 --version \
+\
+&& echo "\n\n"\
+"#######################################\n"\
+"##### Testing dbt-bigquery latest #####\n"\
+"#######################################\n"\
+&& docker build --tag dbt-bigquery \
+  --target dbt-bigquery \
+  docker \
+&& docker run dbt-bigquery --version \
+\
+&& echo "\n\n"\
+"########################################\n"\
+"##### Testing dbt-bigquery-1.0.0b1 #####\n"\
+"########################################\n"\
+&& docker build --tag dbt-bigquery-1.0.0b1 \
+  --target dbt-bigquery \
+  --build-arg dbt_bigquery_ref=dbt-bigquery@v1.0.0b1 \
+  docker \
+&& docker run dbt-bigquery-1.0.0b1 --version \
+\
+&& echo "\n\n"\
+"########################################\n"\
+"##### Testing dbt-snowflake latest #####\n"\
+"########################################\n"\
+&& docker build --tag dbt-snowflake \
+  --target dbt-snowflake \
+  docker \
+&& docker run dbt-snowflake --version \
+\
+&& echo "\n\n"\
+"#########################################\n"\
+"##### Testing dbt-snowflake-1.0.0b1 #####\n"\
+"#########################################\n"\
+&& docker build --tag dbt-snowflake-1.0.0b1 \
+  --target dbt-snowflake\
+  --build-arg dbt_snowflake_ref=dbt-snowflake@v1.0.0b1 \
+  docker \
+&& docker run dbt-snowflake-1.0.0b1 --version \
+\
+&& echo "\n\n"\
+"####################################\n"\
+"##### Testing dbt-spark latest #####\n"\
+"####################################\n"\
+&& docker build --tag dbt-spark \
+  --target dbt-spark \
+  docker \
+&& docker run dbt-spark --version \
+\
+&& echo "\n\n"\
+"#####################################\n"\
+"##### Testing dbt-spark-1.0.0rc2 ####\n"\
+"#####################################\n"\
+&& docker build --tag dbt-spark-1.0.0rc2 \
+  --target dbt-spark \
+  --build-arg dbt_spark_ref=dbt-spark@v1.0.0rc2 \
+  docker \
+&& docker run dbt-spark-1.0.0rc2 --version \
+\
+&& echo "\n\n"\
+"###########################\n"\
+"##### Testing dbt-all #####\n"\
+"###########################\n"\
+&& docker build --tag dbt-all \
+  --target dbt-all \
+  docker \
+&& docker run dbt-all --version \
+\
+&& echo "\n\n"\
+"##########################################\n"\
+"##### Testing third party db adapter #####\n"\
+"##########################################\n"\
+&& docker build --tag dbt-materialize \
+  --target dbt-third-party \
+  --build-arg dbt_third_party="dbt-materialize" \
+  docker \
+&& docker run dbt-materialize --version

docs/arch/adr-002-structured-logging-arch.md

@@ -0,0 +1,34 @@
+# Structured Logging Arch
+
+## Context
+Consumers of dbt have been relying on log parsing well before this change. However, our logs were never optimized for programatic consumption, nor were logs treated like a formal interface between dbt and users. dbt's logging strategy was changed explicitly to address these two realities.
+
+### Options
+#### How to structure the data
+- Using a library like structlog to represent log data with structural types like dictionaries. This would allow us to easily add data to a log event's context at each call site and have structlog do all the string formatting and io work.
+- Creating our own nominal type layer that describes each event in source. This allows event fields to be enforced statically via mypy accross all call sites.
+
+#### How to output the data
+- Using structlog to output log lines regardless of if we used it to represent the data. The defaults for structlog are good, and it handles json vs text and formatting for us.
+- Using the std lib logger to log our messages more manually. Easy to use, but does far less for us.
+
+## Decision
+#### How to structure the data
+We decided to go with a custom nominal type layer even though this was going to be more work. This type layer centralizes our assumptions about what data each log event contains, and allows us to use mypy to enforce these centralized assumptions acrosss the codebase. This is all for the purpose for treating logs like a formal interface between dbt and users. Here are two concrete, practical examples of how this pattern is used:
+
+    1. On the abstract superclass of all events, there are abstract methods and fields that each concrete class must implement such as `level_tag()` and `code`. If you make a new concrete event type without those, mypy will fail and tell you that you need them, preventing lost log lines, and json log events without a computer-friendly code.
+
+    2. On each concrete event, the fields we need to construct the message are explicitly in the source of the class. At every call site if you construct an event without all the necessary data, mypy will fail and tell you which fields you are missing.
+
+Using mypy to enforce these assumptions is a step better than testing becacuse we do not need to write tests to run through every branch of dbt that it could take. Because it is checked statically on every file, mypy will give us these guarantees as long as it is configured to run everywhere.
+
+#### How to output the data
+We decided to use the std lib logger because it was far more difficult than we expected to get to structlog to work properly. Documentation was lacking, and reading the source code wasn't a quick way to learn. The std lib logger was used mostly out of a necessity, and because many of the pleasantries you get from using a log library we had already chosen to do explicitly with functions in our nominal typing layer. Swapping out the std lib logger in the future should be an easy task should we choose to do it.
+
+## Status
+Completed
+
+## Consequences 
+Adding a new log event is more cumbersome than it was previously: instead of writing the message at the log callsite, you must create a new concrete class in the event types. This is more opaque for new contributors. The json serialization approach we are using via `asdict` is fragile and unoptimized and should be replaced.
+
+All user-facing log messages now live in one file which makes the job of conforming them much simpler. Because they are all nominally typed separately, it opens up the possibility to have log documentation generated from the type hints as well as outputting our logs in multiple human languages if we want to translate our messages.

plugins/postgres/dbt/adapters/postgres/__version__.py

@@ -1 +1 @@
-version = '1.0.0rc3'
+version = '1.0.1'

plugins/postgres/setup.py

@@ -41,7 +41,7 @@ def _dbt_psycopg2_name():
 
 
 package_name = "dbt-postgres"
-package_version = "1.0.0rc3"
+package_version = "1.0.1"
 description = """The postgres adpter plugin for dbt (data build tool)"""
 
 this_directory = os.path.abspath(os.path.dirname(__file__))

pytest.ini

@@ -0,0 +1,3 @@
+[pytest]
+filterwarnings =
+    ignore:.*'soft_unicode' has been renamed to 'soft_str'*:DeprecationWarning

scripts/build-dist.sh

@@ -23,7 +23,4 @@ do
     cp -r "$DBT_PATH"/"$SUBPATH"/dist/* "$DBT_PATH"/dist/
 done
 
-cd "$DBT_PATH"
-$PYTHON_BIN setup.py sdist
-
 set +x

setup.py

@@ -1,84 +0,0 @@
-#!/usr/bin/env python
-import os
-import sys
-
-
-if 'sdist' not in sys.argv:
-    print('')
-    print('As of v1.0.0, `pip install dbt` is no longer supported.')
-    print('Instead, please use one of the following.')
-    print('')
-    print('**To use dbt with your specific database, platform, or query engine:**')
-    print('')
-    print('    pip install dbt-<adapter>')
-    print('')
-    print('    See full list: https://docs.getdbt.com/docs/available-adapters')
-    print('')
-    print('**For developers of integrations with dbt Core:**')
-    print('')
-    print('    pip install dbt-core')
-    print('')
-    print('    Be advised, dbt Core''s python API is not yet stable or documented')
-    print('    (https://docs.getdbt.com/docs/running-a-dbt-project/dbt-api)')
-    print('')
-    print('**For the previous behavior of `pip install dbt`:**')
-    print('')
-    print('    pip install dbt-core dbt-postgres dbt-redshift dbt-snowflake dbt-bigquery')
-    print('')
-    sys.exit(1)
-
-
-if sys.version_info < (3, 7):
-    print('Error: dbt does not support this version of Python.')
-    print('Please upgrade to Python 3.7 or higher.')
-    sys.exit(1)
-
-
-from setuptools import setup
-try:
-    from setuptools import find_namespace_packages
-except ImportError:
-    # the user has a downlevel version of setuptools.
-    print('Error: dbt requires setuptools v40.1.0 or higher.')
-    print('Please upgrade setuptools with "pip install --upgrade setuptools" '
-          'and try again')
-    sys.exit(1)
-
-this_directory = os.path.abspath(os.path.dirname(__file__))
-with open(os.path.join(this_directory, 'README.md')) as f:
-    long_description = f.read()
-
-
-package_name = "dbt"
-package_version = "1.0.0rc3"
-description = """With dbt, data analysts and engineers can build analytics \
-the way engineers build applications."""
-
-
-setup(
-    name=package_name,
-    version=package_version,
-
-    description=description,
-    long_description=long_description,
-    long_description_content_type='text/markdown',
-
-    author="dbt Labs",
-    author_email="info@dbtlabs.com",
-    url="https://github.com/dbt-labs/dbt-core",
-    zip_safe=False,
-    classifiers=[
-        'Development Status :: 7 - Inactive',
-
-        'License :: OSI Approved :: Apache Software License',
-
-        'Operating System :: Microsoft :: Windows',
-        'Operating System :: MacOS :: MacOS X',
-        'Operating System :: POSIX :: Linux',
-
-        'Programming Language :: Python :: 3.7',
-        'Programming Language :: Python :: 3.8',
-        'Programming Language :: Python :: 3.9',
-    ],
-    python_requires=">=3.7",
-)

test/integration/001_simple_copy_test/models-snowflake/incremental_overwrite.sql

@@ -1,6 +0,0 @@
-
-{{ config(materialized='incremental', unique_key='id') }}
-
-select 1 as id
-union all
-select 1 as id

test/integration/001_simple_copy_test/snowflake-seed-initial/seed.csv

@@ -1,101 +0,0 @@
-ID,FIRST_NAME,LAST_NAME,EMAIL,GENDER,IP_ADDRESS
-1,Jack,Hunter,jhunter0@pbs.org,Male,59.80.20.168
-2,Kathryn,Walker,kwalker1@ezinearticles.com,Female,194.121.179.35
-3,Gerald,Ryan,gryan2@com.com,Male,11.3.212.243
-4,Bonnie,Spencer,bspencer3@ameblo.jp,Female,216.32.196.175
-5,Harold,Taylor,htaylor4@people.com.cn,Male,253.10.246.136
-6,Jacqueline,Griffin,jgriffin5@t.co,Female,16.13.192.220
-7,Wanda,Arnold,warnold6@google.nl,Female,232.116.150.64
-8,Craig,Ortiz,cortiz7@sciencedaily.com,Male,199.126.106.13
-9,Gary,Day,gday8@nih.gov,Male,35.81.68.186
-10,Rose,Wright,rwright9@yahoo.co.jp,Female,236.82.178.100
-11,Raymond,Kelley,rkelleya@fc2.com,Male,213.65.166.67
-12,Gerald,Robinson,grobinsonb@disqus.com,Male,72.232.194.193
-13,Mildred,Martinez,mmartinezc@samsung.com,Female,198.29.112.5
-14,Dennis,Arnold,darnoldd@google.com,Male,86.96.3.250
-15,Judy,Gray,jgraye@opensource.org,Female,79.218.162.245
-16,Theresa,Garza,tgarzaf@epa.gov,Female,21.59.100.54
-17,Gerald,Robertson,grobertsong@csmonitor.com,Male,131.134.82.96
-18,Philip,Hernandez,phernandezh@adobe.com,Male,254.196.137.72
-19,Julia,Gonzalez,jgonzalezi@cam.ac.uk,Female,84.240.227.174
-20,Andrew,Davis,adavisj@patch.com,Male,9.255.67.25
-21,Kimberly,Harper,kharperk@foxnews.com,Female,198.208.120.253
-22,Mark,Martin,mmartinl@marketwatch.com,Male,233.138.182.153
-23,Cynthia,Ruiz,cruizm@google.fr,Female,18.178.187.201
-24,Samuel,Carroll,scarrolln@youtu.be,Male,128.113.96.122
-25,Jennifer,Larson,jlarsono@vinaora.com,Female,98.234.85.95
-26,Ashley,Perry,aperryp@rakuten.co.jp,Female,247.173.114.52
-27,Howard,Rodriguez,hrodriguezq@shutterfly.com,Male,231.188.95.26
-28,Amy,Brooks,abrooksr@theatlantic.com,Female,141.199.174.118
-29,Louise,Warren,lwarrens@adobe.com,Female,96.105.158.28
-30,Tina,Watson,twatsont@myspace.com,Female,251.142.118.177
-31,Janice,Kelley,jkelleyu@creativecommons.org,Female,239.167.34.233
-32,Terry,Mccoy,tmccoyv@bravesites.com,Male,117.201.183.203
-33,Jeffrey,Morgan,jmorganw@surveymonkey.com,Male,78.101.78.149
-34,Louis,Harvey,lharveyx@sina.com.cn,Male,51.50.0.167
-35,Philip,Miller,pmillery@samsung.com,Male,103.255.222.110
-36,Willie,Marshall,wmarshallz@ow.ly,Male,149.219.91.68
-37,Patrick,Lopez,plopez10@redcross.org,Male,250.136.229.89
-38,Adam,Jenkins,ajenkins11@harvard.edu,Male,7.36.112.81
-39,Benjamin,Cruz,bcruz12@linkedin.com,Male,32.38.98.15
-40,Ruby,Hawkins,rhawkins13@gmpg.org,Female,135.171.129.255
-41,Carlos,Barnes,cbarnes14@a8.net,Male,240.197.85.140
-42,Ruby,Griffin,rgriffin15@bravesites.com,Female,19.29.135.24
-43,Sean,Mason,smason16@icq.com,Male,159.219.155.249
-44,Anthony,Payne,apayne17@utexas.edu,Male,235.168.199.218
-45,Steve,Cruz,scruz18@pcworld.com,Male,238.201.81.198
-46,Anthony,Garcia,agarcia19@flavors.me,Male,25.85.10.18
-47,Doris,Lopez,dlopez1a@sphinn.com,Female,245.218.51.238
-48,Susan,Nichols,snichols1b@freewebs.com,Female,199.99.9.61
-49,Wanda,Ferguson,wferguson1c@yahoo.co.jp,Female,236.241.135.21
-50,Andrea,Pierce,apierce1d@google.co.uk,Female,132.40.10.209
-51,Lawrence,Phillips,lphillips1e@jugem.jp,Male,72.226.82.87
-52,Judy,Gilbert,jgilbert1f@multiply.com,Female,196.250.15.142
-53,Eric,Williams,ewilliams1g@joomla.org,Male,222.202.73.126
-54,Ralph,Romero,rromero1h@sogou.com,Male,123.184.125.212
-55,Jean,Wilson,jwilson1i@ocn.ne.jp,Female,176.106.32.194
-56,Lori,Reynolds,lreynolds1j@illinois.edu,Female,114.181.203.22
-57,Donald,Moreno,dmoreno1k@bbc.co.uk,Male,233.249.97.60
-58,Steven,Berry,sberry1l@eepurl.com,Male,186.193.50.50
-59,Theresa,Shaw,tshaw1m@people.com.cn,Female,120.37.71.222
-60,John,Stephens,jstephens1n@nationalgeographic.com,Male,191.87.127.115
-61,Richard,Jacobs,rjacobs1o@state.tx.us,Male,66.210.83.155
-62,Andrew,Lawson,alawson1p@over-blog.com,Male,54.98.36.94
-63,Peter,Morgan,pmorgan1q@rambler.ru,Male,14.77.29.106
-64,Nicole,Garrett,ngarrett1r@zimbio.com,Female,21.127.74.68
-65,Joshua,Kim,jkim1s@edublogs.org,Male,57.255.207.41
-66,Ralph,Roberts,rroberts1t@people.com.cn,Male,222.143.131.109
-67,George,Montgomery,gmontgomery1u@smugmug.com,Male,76.75.111.77
-68,Gerald,Alvarez,galvarez1v@flavors.me,Male,58.157.186.194
-69,Donald,Olson,dolson1w@whitehouse.gov,Male,69.65.74.135
-70,Carlos,Morgan,cmorgan1x@pbs.org,Male,96.20.140.87
-71,Aaron,Stanley,astanley1y@webnode.com,Male,163.119.217.44
-72,Virginia,Long,vlong1z@spiegel.de,Female,204.150.194.182
-73,Robert,Berry,rberry20@tripadvisor.com,Male,104.19.48.241
-74,Antonio,Brooks,abrooks21@unesco.org,Male,210.31.7.24
-75,Ruby,Garcia,rgarcia22@ovh.net,Female,233.218.162.214
-76,Jack,Hanson,jhanson23@blogtalkradio.com,Male,31.55.46.199
-77,Kathryn,Nelson,knelson24@walmart.com,Female,14.189.146.41
-78,Jason,Reed,jreed25@printfriendly.com,Male,141.189.89.255
-79,George,Coleman,gcoleman26@people.com.cn,Male,81.189.221.144
-80,Rose,King,rking27@ucoz.com,Female,212.123.168.231
-81,Johnny,Holmes,jholmes28@boston.com,Male,177.3.93.188
-82,Katherine,Gilbert,kgilbert29@altervista.org,Female,199.215.169.61
-83,Joshua,Thomas,jthomas2a@ustream.tv,Male,0.8.205.30
-84,Julie,Perry,jperry2b@opensource.org,Female,60.116.114.192
-85,Richard,Perry,rperry2c@oracle.com,Male,181.125.70.232
-86,Kenneth,Ruiz,kruiz2d@wikimedia.org,Male,189.105.137.109
-87,Jose,Morgan,jmorgan2e@webnode.com,Male,101.134.215.156
-88,Donald,Campbell,dcampbell2f@goo.ne.jp,Male,102.120.215.84
-89,Debra,Collins,dcollins2g@uol.com.br,Female,90.13.153.235
-90,Jesse,Johnson,jjohnson2h@stumbleupon.com,Male,225.178.125.53
-91,Elizabeth,Stone,estone2i@histats.com,Female,123.184.126.221
-92,Angela,Rogers,arogers2j@goodreads.com,Female,98.104.132.187
-93,Emily,Dixon,edixon2k@mlb.com,Female,39.190.75.57
-94,Albert,Scott,ascott2l@tinypic.com,Male,40.209.13.189
-95,Barbara,Peterson,bpeterson2m@ow.ly,Female,75.249.136.180
-96,Adam,Greene,agreene2n@fastcompany.com,Male,184.173.109.144
-97,Earl,Sanders,esanders2o@hc360.com,Male,247.34.90.117
-98,Angela,Brooks,abrooks2p@mtv.com,Female,10.63.249.126
-99,Harold,Foster,hfoster2q@privacy.gov.au,Male,139.214.40.244
-100,Carl,Meyer,cmeyer2r@disqus.com,Male,204.117.7.88

test/integration/001_simple_copy_test/snowflake-seed-update/seed.csv

@@ -1,201 +0,0 @@
-ID,FIRST_NAME,LAST_NAME,EMAIL,GENDER,IP_ADDRESS
-1,Jack,Hunter,jhunter0@pbs.org,Male,59.80.20.168
-2,Kathryn,Walker,kwalker1@ezinearticles.com,Female,194.121.179.35
-3,Gerald,Ryan,gryan2@com.com,Male,11.3.212.243
-4,Bonnie,Spencer,bspencer3@ameblo.jp,Female,216.32.196.175
-5,Harold,Taylor,htaylor4@people.com.cn,Male,253.10.246.136
-6,Jacqueline,Griffin,jgriffin5@t.co,Female,16.13.192.220
-7,Wanda,Arnold,warnold6@google.nl,Female,232.116.150.64
-8,Craig,Ortiz,cortiz7@sciencedaily.com,Male,199.126.106.13
-9,Gary,Day,gday8@nih.gov,Male,35.81.68.186
-10,Rose,Wright,rwright9@yahoo.co.jp,Female,236.82.178.100
-11,Raymond,Kelley,rkelleya@fc2.com,Male,213.65.166.67
-12,Gerald,Robinson,grobinsonb@disqus.com,Male,72.232.194.193
-13,Mildred,Martinez,mmartinezc@samsung.com,Female,198.29.112.5
-14,Dennis,Arnold,darnoldd@google.com,Male,86.96.3.250
-15,Judy,Gray,jgraye@opensource.org,Female,79.218.162.245
-16,Theresa,Garza,tgarzaf@epa.gov,Female,21.59.100.54
-17,Gerald,Robertson,grobertsong@csmonitor.com,Male,131.134.82.96
-18,Philip,Hernandez,phernandezh@adobe.com,Male,254.196.137.72
-19,Julia,Gonzalez,jgonzalezi@cam.ac.uk,Female,84.240.227.174
-20,Andrew,Davis,adavisj@patch.com,Male,9.255.67.25
-21,Kimberly,Harper,kharperk@foxnews.com,Female,198.208.120.253
-22,Mark,Martin,mmartinl@marketwatch.com,Male,233.138.182.153
-23,Cynthia,Ruiz,cruizm@google.fr,Female,18.178.187.201
-24,Samuel,Carroll,scarrolln@youtu.be,Male,128.113.96.122
-25,Jennifer,Larson,jlarsono@vinaora.com,Female,98.234.85.95
-26,Ashley,Perry,aperryp@rakuten.co.jp,Female,247.173.114.52
-27,Howard,Rodriguez,hrodriguezq@shutterfly.com,Male,231.188.95.26
-28,Amy,Brooks,abrooksr@theatlantic.com,Female,141.199.174.118
-29,Louise,Warren,lwarrens@adobe.com,Female,96.105.158.28
-30,Tina,Watson,twatsont@myspace.com,Female,251.142.118.177
-31,Janice,Kelley,jkelleyu@creativecommons.org,Female,239.167.34.233
-32,Terry,Mccoy,tmccoyv@bravesites.com,Male,117.201.183.203
-33,Jeffrey,Morgan,jmorganw@surveymonkey.com,Male,78.101.78.149
-34,Louis,Harvey,lharveyx@sina.com.cn,Male,51.50.0.167
-35,Philip,Miller,pmillery@samsung.com,Male,103.255.222.110
-36,Willie,Marshall,wmarshallz@ow.ly,Male,149.219.91.68
-37,Patrick,Lopez,plopez10@redcross.org,Male,250.136.229.89
-38,Adam,Jenkins,ajenkins11@harvard.edu,Male,7.36.112.81
-39,Benjamin,Cruz,bcruz12@linkedin.com,Male,32.38.98.15
-40,Ruby,Hawkins,rhawkins13@gmpg.org,Female,135.171.129.255
-41,Carlos,Barnes,cbarnes14@a8.net,Male,240.197.85.140
-42,Ruby,Griffin,rgriffin15@bravesites.com,Female,19.29.135.24
-43,Sean,Mason,smason16@icq.com,Male,159.219.155.249
-44,Anthony,Payne,apayne17@utexas.edu,Male,235.168.199.218
-45,Steve,Cruz,scruz18@pcworld.com,Male,238.201.81.198
-46,Anthony,Garcia,agarcia19@flavors.me,Male,25.85.10.18
-47,Doris,Lopez,dlopez1a@sphinn.com,Female,245.218.51.238
-48,Susan,Nichols,snichols1b@freewebs.com,Female,199.99.9.61
-49,Wanda,Ferguson,wferguson1c@yahoo.co.jp,Female,236.241.135.21
-50,Andrea,Pierce,apierce1d@google.co.uk,Female,132.40.10.209
-51,Lawrence,Phillips,lphillips1e@jugem.jp,Male,72.226.82.87
-52,Judy,Gilbert,jgilbert1f@multiply.com,Female,196.250.15.142
-53,Eric,Williams,ewilliams1g@joomla.org,Male,222.202.73.126
-54,Ralph,Romero,rromero1h@sogou.com,Male,123.184.125.212
-55,Jean,Wilson,jwilson1i@ocn.ne.jp,Female,176.106.32.194
-56,Lori,Reynolds,lreynolds1j@illinois.edu,Female,114.181.203.22
-57,Donald,Moreno,dmoreno1k@bbc.co.uk,Male,233.249.97.60
-58,Steven,Berry,sberry1l@eepurl.com,Male,186.193.50.50
-59,Theresa,Shaw,tshaw1m@people.com.cn,Female,120.37.71.222
-60,John,Stephens,jstephens1n@nationalgeographic.com,Male,191.87.127.115
-61,Richard,Jacobs,rjacobs1o@state.tx.us,Male,66.210.83.155
-62,Andrew,Lawson,alawson1p@over-blog.com,Male,54.98.36.94
-63,Peter,Morgan,pmorgan1q@rambler.ru,Male,14.77.29.106
-64,Nicole,Garrett,ngarrett1r@zimbio.com,Female,21.127.74.68
-65,Joshua,Kim,jkim1s@edublogs.org,Male,57.255.207.41
-66,Ralph,Roberts,rroberts1t@people.com.cn,Male,222.143.131.109
-67,George,Montgomery,gmontgomery1u@smugmug.com,Male,76.75.111.77
-68,Gerald,Alvarez,galvarez1v@flavors.me,Male,58.157.186.194
-69,Donald,Olson,dolson1w@whitehouse.gov,Male,69.65.74.135
-70,Carlos,Morgan,cmorgan1x@pbs.org,Male,96.20.140.87
-71,Aaron,Stanley,astanley1y@webnode.com,Male,163.119.217.44
-72,Virginia,Long,vlong1z@spiegel.de,Female,204.150.194.182
-73,Robert,Berry,rberry20@tripadvisor.com,Male,104.19.48.241
-74,Antonio,Brooks,abrooks21@unesco.org,Male,210.31.7.24
-75,Ruby,Garcia,rgarcia22@ovh.net,Female,233.218.162.214
-76,Jack,Hanson,jhanson23@blogtalkradio.com,Male,31.55.46.199
-77,Kathryn,Nelson,knelson24@walmart.com,Female,14.189.146.41
-78,Jason,Reed,jreed25@printfriendly.com,Male,141.189.89.255
-79,George,Coleman,gcoleman26@people.com.cn,Male,81.189.221.144
-80,Rose,King,rking27@ucoz.com,Female,212.123.168.231
-81,Johnny,Holmes,jholmes28@boston.com,Male,177.3.93.188
-82,Katherine,Gilbert,kgilbert29@altervista.org,Female,199.215.169.61
-83,Joshua,Thomas,jthomas2a@ustream.tv,Male,0.8.205.30
-84,Julie,Perry,jperry2b@opensource.org,Female,60.116.114.192
-85,Richard,Perry,rperry2c@oracle.com,Male,181.125.70.232
-86,Kenneth,Ruiz,kruiz2d@wikimedia.org,Male,189.105.137.109
-87,Jose,Morgan,jmorgan2e@webnode.com,Male,101.134.215.156
-88,Donald,Campbell,dcampbell2f@goo.ne.jp,Male,102.120.215.84
-89,Debra,Collins,dcollins2g@uol.com.br,Female,90.13.153.235
-90,Jesse,Johnson,jjohnson2h@stumbleupon.com,Male,225.178.125.53
-91,Elizabeth,Stone,estone2i@histats.com,Female,123.184.126.221
-92,Angela,Rogers,arogers2j@goodreads.com,Female,98.104.132.187
-93,Emily,Dixon,edixon2k@mlb.com,Female,39.190.75.57
-94,Albert,Scott,ascott2l@tinypic.com,Male,40.209.13.189
-95,Barbara,Peterson,bpeterson2m@ow.ly,Female,75.249.136.180
-96,Adam,Greene,agreene2n@fastcompany.com,Male,184.173.109.144
-97,Earl,Sanders,esanders2o@hc360.com,Male,247.34.90.117
-98,Angela,Brooks,abrooks2p@mtv.com,Female,10.63.249.126
-99,Harold,Foster,hfoster2q@privacy.gov.au,Male,139.214.40.244
-100,Carl,Meyer,cmeyer2r@disqus.com,Male,204.117.7.88
-101,Michael,Perez,mperez0@chronoengine.com,Male,106.239.70.175
-102,Shawn,Mccoy,smccoy1@reddit.com,Male,24.165.76.182
-103,Kathleen,Payne,kpayne2@cargocollective.com,Female,113.207.168.106
-104,Jimmy,Cooper,jcooper3@cargocollective.com,Male,198.24.63.114
-105,Katherine,Rice,krice4@typepad.com,Female,36.97.186.238
-106,Sarah,Ryan,sryan5@gnu.org,Female,119.117.152.40
-107,Martin,Mcdonald,mmcdonald6@opera.com,Male,8.76.38.115
-108,Frank,Robinson,frobinson7@wunderground.com,Male,186.14.64.194
-109,Jennifer,Franklin,jfranklin8@mail.ru,Female,91.216.3.131
-110,Henry,Welch,hwelch9@list-manage.com,Male,176.35.182.168
-111,Fred,Snyder,fsnydera@reddit.com,Male,217.106.196.54
-112,Amy,Dunn,adunnb@nba.com,Female,95.39.163.195
-113,Kathleen,Meyer,kmeyerc@cdc.gov,Female,164.142.188.214
-114,Steve,Ferguson,sfergusond@reverbnation.com,Male,138.22.204.251
-115,Teresa,Hill,thille@dion.ne.jp,Female,82.84.228.235
-116,Amanda,Harper,aharperf@mail.ru,Female,16.123.56.176
-117,Kimberly,Ray,krayg@xing.com,Female,48.66.48.12
-118,Johnny,Knight,jknighth@jalbum.net,Male,99.30.138.123
-119,Virginia,Freeman,vfreemani@tiny.cc,Female,225.172.182.63
-120,Anna,Austin,aaustinj@diigo.com,Female,62.111.227.148
-121,Willie,Hill,whillk@mail.ru,Male,0.86.232.249
-122,Sean,Harris,sharrisl@zdnet.com,Male,117.165.133.249
-123,Mildred,Adams,madamsm@usatoday.com,Female,163.44.97.46
-124,David,Graham,dgrahamn@zimbio.com,Male,78.13.246.202
-125,Victor,Hunter,vhuntero@ehow.com,Male,64.156.179.139
-126,Aaron,Ruiz,aruizp@weebly.com,Male,34.194.68.78
-127,Benjamin,Brooks,bbrooksq@jalbum.net,Male,20.192.189.107
-128,Lisa,Wilson,lwilsonr@japanpost.jp,Female,199.152.130.217
-129,Benjamin,King,bkings@comsenz.com,Male,29.189.189.213
-130,Christina,Williamson,cwilliamsont@boston.com,Female,194.101.52.60
-131,Jane,Gonzalez,jgonzalezu@networksolutions.com,Female,109.119.12.87
-132,Thomas,Owens,towensv@psu.edu,Male,84.168.213.153
-133,Katherine,Moore,kmoorew@naver.com,Female,183.150.65.24
-134,Jennifer,Stewart,jstewartx@yahoo.com,Female,38.41.244.58
-135,Sara,Tucker,stuckery@topsy.com,Female,181.130.59.184
-136,Harold,Ortiz,hortizz@vkontakte.ru,Male,198.231.63.137
-137,Shirley,James,sjames10@yelp.com,Female,83.27.160.104
-138,Dennis,Johnson,djohnson11@slate.com,Male,183.178.246.101
-139,Louise,Weaver,lweaver12@china.com.cn,Female,1.14.110.18
-140,Maria,Armstrong,marmstrong13@prweb.com,Female,181.142.1.249
-141,Gloria,Cruz,gcruz14@odnoklassniki.ru,Female,178.232.140.243
-142,Diana,Spencer,dspencer15@ifeng.com,Female,125.153.138.244
-143,Kelly,Nguyen,knguyen16@altervista.org,Female,170.13.201.119
-144,Jane,Rodriguez,jrodriguez17@biblegateway.com,Female,12.102.249.81
-145,Scott,Brown,sbrown18@geocities.jp,Male,108.174.99.192
-146,Norma,Cruz,ncruz19@si.edu,Female,201.112.156.197
-147,Marie,Peters,mpeters1a@mlb.com,Female,231.121.197.144
-148,Lillian,Carr,lcarr1b@typepad.com,Female,206.179.164.163
-149,Judy,Nichols,jnichols1c@t-online.de,Female,158.190.209.194
-150,Billy,Long,blong1d@yahoo.com,Male,175.20.23.160
-151,Howard,Reid,hreid1e@exblog.jp,Male,118.99.196.20
-152,Laura,Ferguson,lferguson1f@tuttocitta.it,Female,22.77.87.110
-153,Anne,Bailey,abailey1g@geocities.com,Female,58.144.159.245
-154,Rose,Morgan,rmorgan1h@ehow.com,Female,118.127.97.4
-155,Nicholas,Reyes,nreyes1i@google.ru,Male,50.135.10.252
-156,Joshua,Kennedy,jkennedy1j@house.gov,Male,154.6.163.209
-157,Paul,Watkins,pwatkins1k@upenn.edu,Male,177.236.120.87
-158,Kathryn,Kelly,kkelly1l@businessweek.com,Female,70.28.61.86
-159,Adam,Armstrong,aarmstrong1m@techcrunch.com,Male,133.235.24.202
-160,Norma,Wallace,nwallace1n@phoca.cz,Female,241.119.227.128
-161,Timothy,Reyes,treyes1o@google.cn,Male,86.28.23.26
-162,Elizabeth,Patterson,epatterson1p@sun.com,Female,139.97.159.149
-163,Edward,Gomez,egomez1q@google.fr,Male,158.103.108.255
-164,David,Cox,dcox1r@friendfeed.com,Male,206.80.80.58
-165,Brenda,Wood,bwood1s@over-blog.com,Female,217.207.44.179
-166,Adam,Walker,awalker1t@blogs.com,Male,253.211.54.93
-167,Michael,Hart,mhart1u@wix.com,Male,230.206.200.22
-168,Jesse,Ellis,jellis1v@google.co.uk,Male,213.254.162.52
-169,Janet,Powell,jpowell1w@un.org,Female,27.192.194.86
-170,Helen,Ford,hford1x@creativecommons.org,Female,52.160.102.168
-171,Gerald,Carpenter,gcarpenter1y@about.me,Male,36.30.194.218
-172,Kathryn,Oliver,koliver1z@army.mil,Female,202.63.103.69
-173,Alan,Berry,aberry20@gov.uk,Male,246.157.112.211
-174,Harry,Andrews,handrews21@ameblo.jp,Male,195.108.0.12
-175,Andrea,Hall,ahall22@hp.com,Female,149.162.163.28
-176,Barbara,Wells,bwells23@behance.net,Female,224.70.72.1
-177,Anne,Wells,awells24@apache.org,Female,180.168.81.153
-178,Harry,Harper,hharper25@rediff.com,Male,151.87.130.21
-179,Jack,Ray,jray26@wufoo.com,Male,220.109.38.178
-180,Phillip,Hamilton,phamilton27@joomla.org,Male,166.40.47.30
-181,Shirley,Hunter,shunter28@newsvine.com,Female,97.209.140.194
-182,Arthur,Daniels,adaniels29@reuters.com,Male,5.40.240.86
-183,Virginia,Rodriguez,vrodriguez2a@walmart.com,Female,96.80.164.184
-184,Christina,Ryan,cryan2b@hibu.com,Female,56.35.5.52
-185,Theresa,Mendoza,tmendoza2c@vinaora.com,Female,243.42.0.210
-186,Jason,Cole,jcole2d@ycombinator.com,Male,198.248.39.129
-187,Phillip,Bryant,pbryant2e@rediff.com,Male,140.39.116.251
-188,Adam,Torres,atorres2f@sun.com,Male,101.75.187.135
-189,Margaret,Johnston,mjohnston2g@ucsd.edu,Female,159.30.69.149
-190,Paul,Payne,ppayne2h@hhs.gov,Male,199.234.140.220
-191,Todd,Willis,twillis2i@businessweek.com,Male,191.59.136.214
-192,Willie,Oliver,woliver2j@noaa.gov,Male,44.212.35.197
-193,Frances,Robertson,frobertson2k@go.com,Female,31.117.65.136
-194,Gregory,Hawkins,ghawkins2l@joomla.org,Male,91.3.22.49
-195,Lisa,Perkins,lperkins2m@si.edu,Female,145.95.31.186
-196,Jacqueline,Anderson,janderson2n@cargocollective.com,Female,14.176.0.187
-197,Shirley,Diaz,sdiaz2o@ucla.edu,Female,207.12.95.46
-198,Nicole,Meyer,nmeyer2p@flickr.com,Female,231.79.115.13
-199,Mary,Gray,mgray2q@constantcontact.com,Female,210.116.64.253
-200,Jean,Mcdonald,jmcdonald2r@baidu.com,Female,122.239.235.117