Bump version: 0.14.3rc1 → 0.14.3

dbt 0.14.3

.bumpversion.cfg

@@ -1,9 +1,36 @@
 [bumpversion]
-current_version = 0.5.3
-commit = True
-tag = True
+current_version = 0.14.3
+parse = (?P<major>\d+)
+	\.(?P<minor>\d+)
+	\.(?P<patch>\d+)
+	((?P<prerelease>[a-z]+)(?P<num>\d+))?
+serialize = 
+	{major}.{minor}.{patch}{prerelease}{num}
+	{major}.{minor}.{patch}
+commit = False
+tag = False
+
+[bumpversion:part:prerelease]
+first_value = a
+values = 
+	a
+	b
+	rc
+
+[bumpversion:part:num]
+first_value = 1
 
 [bumpversion:file:setup.py]
 
-[bumpversion:file:dbt/version.py]
+[bumpversion:file:core/setup.py]
+
+[bumpversion:file:core/dbt/version.py]
+
+[bumpversion:file:plugins/postgres/setup.py]
+
+[bumpversion:file:plugins/redshift/setup.py]
+
+[bumpversion:file:plugins/snowflake/setup.py]
+
+[bumpversion:file:plugins/bigquery/setup.py]
 

.circleci/config.yml

@@ -0,0 +1,129 @@
+version: 2
+jobs:
+  unit:
+    docker: &test_and_postgres
+      - image: fishtownjacob/test-container
+      - image: postgres
+        name: database
+        environment: &pgenv
+          POSTGRES_USER: "root"
+          POSTGRES_PASSWORD: "password"
+          POSTGRES_DB: "dbt"
+    steps:
+      - checkout
+      - run: &setupdb
+          name: Setup postgres
+          command: bash test/setup_db.sh
+          environment:
+            PGHOST: database
+            PGUSER: root
+            PGPASSWORD: password
+            PGDATABASE: postgres
+      - run: tox -e flake8,unit-py27,unit-py36
+  integration-postgres-py36:
+    docker: *test_and_postgres
+    steps:
+      - checkout
+      - run: *setupdb
+      - run:
+          name: Run tests
+          command: tox -e integration-postgres-py36
+      - store_artifacts:
+          path: ./logs
+  integration-snowflake-py36:
+    docker: &test_only
+      - image: fishtownjacob/test-container
+    steps:
+      - checkout
+      - run:
+          name: Run tests
+          command: tox -e integration-snowflake-py36
+          no_output_timeout: 1h
+      - store_artifacts:
+          path: ./logs
+  integration-redshift-py36:
+    docker: *test_only
+    steps:
+      - checkout
+      - run:
+          name: Run tests
+          command: tox -e integration-redshift-py36
+      - store_artifacts:
+          path: ./logs
+  integration-bigquery-py36:
+    docker: *test_only
+    steps:
+      - checkout
+      - run:
+          name: Run tests
+          command: tox -e integration-bigquery-py36
+      - store_artifacts:
+          path: ./logs
+  integration-postgres-py27:
+    docker: *test_and_postgres
+    steps:
+      - checkout
+      - run: *setupdb
+      - run:
+          name: Run tests
+          command: tox -e integration-postgres-py27
+      - store_artifacts:
+          path: ./logs
+  integration-snowflake-py27:
+    docker: *test_only
+    steps:
+      - checkout
+      - run:
+          name: Run tests
+          command: tox -e integration-snowflake-py27
+          no_output_timeout: 1h
+      - store_artifacts:
+          path: ./logs
+  integration-redshift-py27:
+    docker: *test_only
+    steps:
+      - checkout
+      - run:
+          name: Run tests
+          command: tox -e integration-redshift-py27
+      - store_artifacts:
+          path: ./logs
+  integration-bigquery-py27:
+    docker: *test_only
+    steps:
+      - checkout
+      - run:
+          name: Run tests
+          command: tox -e integration-bigquery-py27
+      - store_artifacts:
+          path: ./logs
+
+workflows:
+  version: 2
+  test-everything:
+    jobs:
+      - unit
+      - integration-postgres-py36:
+          requires:
+            - unit
+      - integration-postgres-py27:
+          requires:
+            - unit
+      - integration-redshift-py27:
+          requires:
+            - integration-postgres-py27
+      - integration-bigquery-py27:
+          requires:
+            - integration-postgres-py27
+      - integration-snowflake-py27:
+          requires:
+            - integration-postgres-py27
+      - integration-redshift-py36:
+          requires:
+            - integration-postgres-py36
+      - integration-bigquery-py36:
+          requires:
+            - integration-postgres-py36
+      - integration-snowflake-py36:
+          requires:
+            - integration-postgres-py36

.dockerignore

@@ -0,0 +1 @@
+*

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,41 @@
+---
+name: Bug report
+about: Report a bug or an issue you've found with dbt
+title: ''
+labels: bug, triage
+assignees: ''
+
+---
+
+### Describe the bug
+A clear and concise description of what the bug is. What command did you run? What happened?
+
+### Steps To Reproduce
+In as much detail as possible, please provide steps to reproduce the issue. Sample data that triggers the issue, example model code, etc is all very helpful here.
+
+### Expected behavior
+A clear and concise description of what you expected to happen.
+
+### Screenshots and log output
+If applicable, add screenshots or log output to help explain your problem.
+
+### System information
+**Which database are you using dbt with?**
+- [ ] postgres
+- [ ] redshift
+- [ ] bigquery
+- [ ] snowflake
+- [ ] other (specify: ____________)
+
+
+**The output of `dbt --version`:**
+```
+<output goes here>
+```
+
+**The operating system you're using:**
+
+**The output of `python --version`:**
+
+### Additional context
+Add any other context about the problem here.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,20 @@
+---
+name: Feature request
+about: Suggest an idea for dbt
+title: ''
+labels: enhancement, triage
+assignees: ''
+
+---
+
+### Describe the feature
+A clear and concise description of what you want to happen.
+
+### Describe alternatives you've considered
+A clear and concise description of any alternative solutions or features you've considered.
+
+### Additional context
+Is this feature database-specific? Which database(s) is/are relevant? Please include any other relevant context here.
+
+### Who will this benefit?
+What kind of use case will this feature be useful for? Please be specific and provide examples, this will help us prioritize properly.

.gitignore

@@ -23,6 +23,7 @@ var/
 *.egg-info/
 .installed.cfg
 *.egg
+logs/
 
 # PyInstaller
 #  Usually these files are written by a python script from a template
@@ -44,6 +45,7 @@ nosetests.xml
 coverage.xml
 *,cover
 .hypothesis/
+test.env
 
 # Translations
 *.mo
@@ -60,3 +62,17 @@ target/
 
 #Ipython Notebook
 .ipynb_checkpoints
+
+#Emacs
+*~
+
+# Sublime Text
+*.sublime-*
+
+# Vim
+*.sw*
+
+.python-version
+
+# Vim
+*.sw*

CONTRIBUTING.md

@@ -0,0 +1,155 @@
+# Getting started with dbt
+
+## About this document
+
+This document is a guide intended for folks interested in contributing to dbt. It is not intended as a guide for end users of dbt (though if it is helpful, that's great!) and it assumes a certain level of familiarity with Python concepts such as virtualenvs, `pip`, python modules, filesystems, and so on. It also assumes you are using macOS or Linux and are comfortable with the command line. If you get stuck while reading this guide, drop us a line in the #development channel on [slack](slack.getdbt.com).
+
+## Getting the code
+
+### Installing git
+
+You will need `git` in order to download and modify the dbt source code. On macOS, the best way to download git is to just install Xcode.
+
+### External contributors
+
+If you are not a member of the `fishtown-analytics` GitHub organization, you can contribute to dbt by forking the dbt repository. For a detailed overview on forking, check out the [GitHub docs on forking](https://help.github.com/en/articles/fork-a-repo). In short, you will need to:
+
+1. fork the dbt repository
+2. clone your fork
+3. check out a new branch for your proposed changes
+4. push changes to your fork
+5. open a pull request against `fishtown-analytics/dbt` from your forked repository
+
+### Core contributors
+
+If you are a member of the `fishtown-analytics` GitHub organization, you will have push access to the dbt repo. Rather than 
+forking dbt to make your changes, just clone the repository and push directly to a branch.
+
+
+## Setting up an environment
+
+To begin developing code in dbt, you should set up the following:
+
+### virtualenv
+
+We strongly recommend using virtual environments when developing code in dbt. We recommend creating this virtualenv
+in the root of the dbt repository. To create a new virtualenv, run:
+```
+python3 -m venv env
+source env/bin/activate
+```
+
+This will create and activate a new Python virtual environment.
+
+### docker and docker-compose
+
+Docker and docker-compose are both used in testing. For macOS, the easiest thing to do is to [download docker for mac](https://store.docker.com/editions/community/docker-ce-desktop-mac). You'll need to make an account. On Linux, you can use one of the packages [here](https://docs.docker.com/install/#server). We recommend installing from docker.com instead of from your package manager. On Linux you also have to install docker-compose separately, follow [these instructions](https://docs.docker.com/compose/install/#install-compose).
+
+
+### Installing postgres locally (optional)
+
+For testing, and later in the examples in this document, you may want to have `psql` available so you can poke around in the database and see what happened. We recommend that you use [homebrew](https://brew.sh/) for that on macOS, and your package manager on Linux. You can install any version of the postgres client that you'd like. On macOS, with homebrew setup, you can run:
+
+```
+brew install postgresql
+```
+
+## Running dbt in development
+
+### Installation
+
+First make sure that you set up your `virtualenv` as described in section _Setting up an environment_. Next, install dbt (and it's dependencies) with:
+
+```
+pip install -r requirements.txt
+```
+
+When dbt is installed from source in this way, any changes you make to the dbt source code will be reflected immediately in your next `dbt` run.
+
+### Running dbt
+
+With your virtualenv activated, the `dbt` script should point back to the source code you've cloned on your machine. You can verify this by running `which dbt`. This command should show you a path to an executable in your virtualenv.
+
+Configure your [profile](https://docs.getdbt.com/docs/configure-your-profile) as necessary to connect to your target databases. It may be a good idea to add a new profile pointing to a local postgres instance, or a specific test sandbox within your data warehouse if appropriate.
+
+## Testing
+
+Getting the dbt integration tests set up in your local environment will be very helpful as you start to make changes to your local version of dbt. The section that follows outlines some helpful tips for setting up the test environment.
+
+### Tools
+
+A short list of tools used in dbt testing that will be helpful to your understanding:
+
+- [virtualenv](https://virtualenv.pypa.io/en/stable/) to manage dependencies
+- [tox](https://tox.readthedocs.io/en/latest/) to manage virtualenvs across python versions
+- [pytest](https://docs.pytest.org/en/latest/) to discover/run tests
+- [make](https://users.cs.duke.edu/~ola/courses/programming/Makefiles/Makefiles.html) - but don't worry too much, nobody _really_ understands how make works and our Makefile is super simple
+- [flake8](https://gitlab.com/pycqa/flake8) for code linting
+- [CircleCI](https://circleci.com/product/) and [Azure Pipelines](https://azure.microsoft.com/en-us/services/devops/pipelines/)
+
+A deep understanding of these tools in not required to effectively contribute to dbt, but we recommend checking out the attached documentation if you're interested in learning more about them.
+
+
+### Running tests via Docker
+
+dbt's unit and integration tests run in Docker. Because dbt works with a number of different databases, you will need to supply credentials for one or more of these databases in your test environment. Most organizations don't have access to each of a BigQuery, Redshift, Snowflake, and Postgres database, so it's likely that you will be unable to run every integration test locally. Fortunately, Fishtown Analytics provides a CI environment with access to sandboxed Redshift, Snowflake, BigQuery, and Postgres databases. See the section on _Submitting a Pull Request_ below for more information on this CI setup.
+
+
+#### Specifying your test credentials
+
+dbt uses test credentials specified in a `test.env` file in the root of the repository. This `test.env` file is git-ignored, but please be _extra_ careful to never check in credentials or other sensitive information when developing against dbt. To create your `test.env` file, copy the provided sample file, then supply your relevant credentials:
+
+```
+cp test.env.sample test.env
+atom test.env # supply your credentials
+```
+
+We recommend starting with dbt's Postgres tests. These tests cover most of the functionality in dbt, are the fastest to run, and are the easiest to set up. dbt's test suite runs Postgres in a Docker container, so no setup should be required to run these tests. If you additionally want to test Snowflake, Bigquery, or Redshift locally you'll need to get credentials and add them to the `test.env` file. 
+
+#### Running tests
+
+dbt's unit tests and Python linter can be run with:
+
+```
+make test-unit
+```
+
+To run the Postgres + Python 3.6 integration tests, you'll have to do one extra step of setting up the test database:
+
+```
+docker-compose up -d database
+PGHOST=localhost PGUSER=root PGPASSWORD=password PGDATABASE=postgres bash test/setup_db.sh
+```
+
+To run a quick test for Python3 integration tests on Postgres, you can run:
+
+```
+make test-quick
+```
+
+To run tests for a specific database, invoke `tox` directly with the required flags:
+```
+# Run Postgres py36 tests
+docker-compose run test tox -e integration-postgres-py36 -- -x
+
+# Run Snowflake py36 tests
+docker-compose run test tox -e integration-snowflake-py36 -- -x
+
+# Run BigQuery py36 tests
+docker-compose run test tox -e integration-bigquery-py36 -- -x
+
+# Run Redshift py36 tests
+docker-compose run test tox -e integration-redshift-py36 -- -x
+```
+
+See the `Makefile` contents for more some other examples of ways to run `tox`.
+
+### Submitting a Pull Request
+
+Fishtown Analytics provides a sandboxed Redshift, Snowflake, and BigQuery database for use in a CI environment.
+
+When pull requests are submitted to the `fishtown-analytics/dbt` repo, GitHub will trigger automated tests in CircleCI and Azure Pipelines. If the PR submitter is a member of the `fishtown-analytics` GitHub organization, then the credentials for these databases will be automatically supplied as environment variables in the CI test suite.
+
+**If the PR submitter is not a member of the `fishtown-analytics` organization, then these environment variables will not be automatically supplied in the CI environment**. Once a core maintainer has taken a look at the Pull Request, they will kick off the test suite with the required credentials.
+
+Once your tests are passing and your PR has been reviewed, a dbt maintainer will merge your changes into the active development branch! And that's it! Happy developing :tada:

Dockerfile

@@ -0,0 +1,53 @@
+FROM ubuntu:18.04
+
+ENV DEBIAN_FRONTEND noninteractive
+
+RUN apt-get update && \
+    apt-get install -y  --no-install-recommends \
+        netcat postgresql make build-essential libssl-dev zlib1g-dev \
+        libbz2-dev libreadline-dev libsqlite3-dev wget curl llvm libncurses5-dev \
+        xz-utils tk-dev libxml2-dev libxmlsec1-dev libffi-dev git ca-certificates \
+        curl git ssh && \
+    apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+
+RUN useradd -mU dbt_test_user
+RUN mkdir /usr/app && chown dbt_test_user /usr/app
+RUN mkdir /home/tox && chown dbt_test_user /home/tox
+USER dbt_test_user
+
+WORKDIR /usr/app
+VOLUME /usr/app
+
+RUN curl -L https://github.com/pyenv/pyenv-installer/raw/master/bin/pyenv-installer | bash
+
+ENV PYENV_ROOT="/home/dbt_test_user/.pyenv" \
+    PATH="/home/dbt_test_user/.pyenv/bin:/home/dbt_test_user/.pyenv/shims:$PATH"
+
+RUN pyenv update && \
+    echo "2.7.16 3.6.8 3.7.3" | xargs -P 4 -n 1 pyenv install && \
+    pyenv global $(pyenv versions --bare)
+
+RUN pyenv virtualenv 3.6.8 dbt36 && \
+    pyenv virtualenv 3.7.3 dbt37 && \
+    pyenv virtualenv 2.7.16 dbt27
+
+RUN cd /usr/app && \
+    python -m pip install -U pip && \
+    python -m pip install tox && \
+    pyenv local dbt37 && \
+    python -m pip install -U pip && \
+    python -m pip install tox && \
+    pyenv local --unset && \
+    pyenv local dbt36 && \
+    python -m pip install -U pip && \
+    python -m pip install tox && \
+    pyenv local --unset && \
+    pyenv local dbt27 && \
+    python -m pip install -U pip && \
+    python -m pip install tox && \
+    pyenv local --unset && \
+    pyenv rehash
+
+RUN pyenv local dbt36 dbt37 dbt27
+
+ENV PYTHONIOENCODING=utf-8

Makefile

@@ -0,0 +1,36 @@
+.PHONY: install test test-unit test-integration
+
+changed_tests := `git status --porcelain | grep '^\(M\| M\|A\| A\)' | awk '{ print $$2 }' | grep '\/test_[a-zA-Z_\-\.]\+.py'`
+
+install:
+	pip install -e .
+
+test:
+	@echo "Full test run starting..."
+	@time docker-compose run test tox
+
+test-unit:
+	@echo "Unit test run starting..."
+	@time docker-compose run test tox -e unit-py27,unit-py36,flake8
+
+test-integration:
+	@echo "Integration test run starting..."
+	@time docker-compose run test tox -e integration-postgres-py27,integration-postgres-py36,integration-redshift-py27,integration-redshift-py36,integration-snowflake-py27,integration-snowflake-py36,integration-bigquery-py27,integration-bigquery-py36
+
+test-quick:
+	@echo "Integration test run starting..."
+	@time docker-compose run test tox -e integration-postgres-py36 -- -x
+
+clean:
+	rm -f .coverage
+	rm -rf .eggs/
+	rm -rf .tox/
+	rm -rf build/
+	rm -rf dbt.egg-info/
+	rm -f dbt_project.yml
+	rm -rf dist/
+	rm -f htmlcov/*.{css,html,js,json,png}
+	rm -rf logs/
+	rm -rf target/
+	find . -type f -name '*.pyc' -delete
+	find . -type d -name '__pycache__' -depth -delete

README.md

@@ -1,24 +1,58 @@
-# dbt
-Tests: [![CircleCI](https://circleci.com/gh/analyst-collective/dbt/tree/master.svg?style=svg)](https://circleci.com/gh/analyst-collective/dbt/tree/master)
+<p align="center">
+  <img src="https://github.com/fishtown-analytics/dbt/blob/master/etc/dbt-horizontal.png?raw=true" alt="dbt logo"/>
+</p>
+<p align="center">
+  <a href="https://codeclimate.com/github/fishtown-analytics/dbt">
+    <img src="https://codeclimate.com/github/fishtown-analytics/dbt/badges/gpa.svg" alt="Code Climate"/>
+  </a>
+  <a href="https://circleci.com/gh/fishtown-analytics/dbt/tree/master">
+    <img src="https://circleci.com/gh/fishtown-analytics/dbt/tree/master.svg?style=svg"  alt="CircleCI" />
+  </a>
+  <a href="https://ci.appveyor.com/project/DrewBanin/dbt/branch/development">
+    <img src="https://ci.appveyor.com/api/projects/status/v01rwd3q91jnwp9m/branch/development?svg=true" alt="AppVeyor" />
+  </a>
+  <a href="https://slack.getdbt.com">
+    <img src="https://slack.getdbt.com/badge.svg" alt="Slack" />
+  </a>
+</p>
 
-dbt (data build tool) helps analysts write reliable, modular code using a workflow that closely mirrors software development.
+**[dbt](https://www.getdbt.com/)** (data build tool) enables data analysts and engineers to transform their data using the same practices that software engineers use to build applications.
 
----
+dbt is the T in ELT. Organize, cleanse, denormalize, filter, rename, and pre-aggregate the raw data in your warehouse so that it's ready for analysis.
 
-- [What is dbt]?
-- Read the [dbt viewpoint]
-- [Installation]
-- Join the [chat][slack-url] on Slack for live questions and support.
+![dbt architecture](https://github.com/fishtown-analytics/dbt/blob/master/etc/dbt-arch.png?raw=true)
 
+dbt can be used to [aggregate pageviews into sessions](https://github.com/fishtown-analytics/snowplow), calculate [ad spend ROI](https://github.com/fishtown-analytics/facebook-ads), or report on [email campaign performance](https://github.com/fishtown-analytics/mailchimp).
+
+## Understanding dbt
+
+Analysts using dbt can transform their data by simply writing select statements, while dbt handles turning these statements into tables and views in a data warehouse.
+
+These select statements, or "models", form a dbt project. Models frequently build on top of one another – dbt makes it easy to [manage relationships](https://docs.getdbt.com/docs/ref) between models, and [visualize these relationships](https://docs.getdbt.com/docs/documentation), as well as assure the quality of your transformations through [testing](https://docs.getdbt.com/docs/testing).
+
+![dbt dag](https://github.com/fishtown-analytics/dbt/blob/master/etc/dbt-dag.png?raw=true)
+
+## Getting started
+
+-   [Install dbt](https://docs.getdbt.com/docs/installation)
+-   Read the [documentation](https://docs.getdbt.com/).
+-   Productionize your dbt project with [dbt Cloud](https://www.getdbt.com)
+
+## Find out more
+
+-   Check out the [Introduction to dbt](https://dbt.readme.io/docs/introduction).
+-   Read the [dbt Viewpoint](https://dbt.readme.io/docs/viewpoint).
+
+## Join thousands of analysts in the dbt community
+
+-   Join the [chat](http://slack.getdbt.com/) on Slack.
+-   Find community posts on [dbt Discourse](https://discourse.getdbt.com).
+
+## Reporting bugs and contributing code
+
+-   Want to report a bug or request a feature? Let us know on [Slack](http://slack.getdbt.com/), or open [an issue](https://github.com/fishtown-analytics/dbt/issues/new).
+-   Want to help us build dbt? Check out the [Contributing Getting Started Guide](/CONTRIBUTING.md)
 
 ## Code of Conduct
 
-Everyone interacting in the dbt project's codebases, issue trackers, chat rooms, and mailing lists is expected to follow the [PyPA Code of Conduct].
-
-
-
-[PyPA Code of Conduct]: https://www.pypa.io/en/latest/code-of-conduct/
-[slack-url]: http://ac-slackin.herokuapp.com/
-[Installation]: http://dbt.readthedocs.io/en/master/guide/setup/
-[What is dbt]: http://dbt.readthedocs.io/en/master/about/overview/
-[dbt viewpoint]: http://dbt.readthedocs.io/en/master/about/viewpoint/
+Everyone interacting in the dbt project's codebases, issue trackers, chat rooms, and mailing lists is expected to follow the [PyPA Code of Conduct](https://www.pypa.io/en/latest/code-of-conduct/).

RELEASE.md

@@ -0,0 +1,92 @@
+### Release Procedure :shipit:
+
+#### Branching Strategy
+
+dbt has three types of branches:
+
+- **Trunks** track the latest release of a minor version of dbt. Historically, we used the `master` branch as the trunk. Each minor version release has a corresponding trunk. For example, the `0.11.x` series of releases has a branch called `0.11.latest`. This allows us to release new patch versions under `0.11` without necessarily needing to pull them into the latest version of dbt.
+- **Release Branches** track a specific, not yet complete release of dbt. These releases are codenamed since we don't always know what their semantic version will be. Example: `dev/lucretia-mott` became `0.11.1`.
+- **Feature Branches** track individual features and fixes. On completion they should be merged into a release branch.
+
+#### Git & PyPI
+
+1. Update CHANGELOG.md with the most recent changes
+2. If this is a release candidate, you want to create it off of your release branch. If it's an actual release, you must first merge to a master branch. Open a Pull Request in Github to merge it into the appropriate trunk (`X.X.latest`)
+3. Bump the version using `bumpversion`:
+  - Dry run first by running `bumpversion --new-version <desired-version> <part>` and checking the diff. If it looks correct, clean up the chanages and move on:
+  - Alpha releases: `bumpversion --commit --no-tag --new-version 0.10.2a1 num`
+  - Patch releases: `bumpversion --commit --no-tag --new-version 0.10.2 patch`
+  - Minor releases: `bumpversion --commit --no-tag --new-version 0.11.0 minor`
+  - Major releases: `bumpversion --commit --no-tag --new-version 1.0.0 major`
+4. (If this is a not a release candidate) Merge to `x.x.latest` and (optionally) `master`.
+5. Update the default branch to the next dev release branch.
+6. Build source distributions for all packages by running `./scripts/build-sdists.sh`. Note that this will clean out your `dist/` folder, so if you have important stuff in there, don't run it!!!
+7. Deploy to pypi
+  - `twine upload dist/*`
+8. Deploy to homebrew (see below)
+9. Deploy to conda-forge (see below)
+10. Git release notes (points to changelog)
+11. Post to slack (point to changelog)
+
+After releasing a new version, it's important to merge the changes back into the other outstanding release branches. This avoids merge conflicts moving forward.
+
+In some cases, where the branches have diverged wildly, it's ok to skip this step. But this means that the changes you just released won't be included in future releases.
+
+#### Homebrew Release Process
+
+1. Clone the `homebrew-dbt` repository:
+
+```
+git clone git@github.com:fishtown-analytics/homebrew-dbt.git
+```
+
+2. For ALL releases (prereleases and version releases), copy the relevant formula. To copy from the latest version release of dbt, do:
+
+```bash
+cp Formula/dbt.rb Formula/dbt@{NEW-VERSION}.rb
+```
+
+To copy from a different version, simply copy the corresponding file.
+
+3. Open the file, and edit the following:
+- the name of the ruby class: this is important, homebrew won't function properly if the class name is wrong. Check historical versions to figure out the right name.
+- under the `bottle` section, remove all of the hashes (lines starting with `sha256`)
+
+4. Create a **Python 3.7** virtualenv, activate it, and then install two packages: `homebrew-pypi-poet`, and the version of dbt you are preparing. I use:
+
+```
+pyenv virtualenv 3.7.0 homebrew-dbt-{VERSION}
+pyenv activate homebrew-dbt-{VERSION}
+pip install dbt=={VERSION} homebrew-pypi-poet
+```
+
+homebrew-pypi-poet is a program that generates a valid homebrew formula for an installed pip package. You want to use it to generate a diff against the existing formula. Then you want to apply the diff for the dependency packages only -- e.g. it will tell you that `google-api-core` has been updated and that you need to use the latest version.
+
+5. reinstall, test, and audit dbt. if the test or audit fails, fix the formula with step 1.
+
+```bash
+brew uninstall --force Formula/{YOUR-FILE}.rb
+brew install Formula/{YOUR-FILE}.rb
+brew test dbt
+brew audit --strict dbt
+```
+
+6. Ask Connor to bottle the change (only his laptop can do it!)
+
+#### Conda Forge Release Process
+
+1. Clone the fork of `conda-forge/dbt-feedstock` [here](https://github.com/fishtown-analytics/dbt-feedstock)
+```bash
+git clone git@github.com:fishtown-analytics/dbt-feedstock.git
+
+```
+2. Update the version and sha256 in `recipe/meta.yml`. To calculate the sha256, run:
+
+```bash
+wget https://github.com/fishtown-analytics/dbt/archive/v{version}.tar.gz
+openssl sha256 v{version}.tar.gz
+```
+
+3. Push the changes and create a PR against `conda-forge/dbt-feedstock`
+
+4. Confirm that all automated conda-forge tests are passing

azure-pipelines.yml

@@ -0,0 +1,125 @@
+# Python package
+# Create and test a Python package on multiple Python versions.
+# Add steps that analyze code, save the dist with the build record, publish to a PyPI-compatible index, and more:
+# https://docs.microsoft.com/azure/devops/pipelines/languages/python
+
+trigger:
+  branches:
+    include:
+    - master
+    - dev/*
+    - pr/*
+
+jobs:
+- job: UnitTest
+  pool:
+    vmImage: 'vs2017-win2016'
+  steps:
+  - task: UsePythonVersion@0
+    inputs:
+      versionSpec: '3.5'
+      architecture: 'x64'
+
+  - script: python -m pip install --upgrade pip && pip install tox
+    displayName: 'Install dependencies'
+
+  - script: python -m tox -e pywin-unit
+    displayName: Run unit tests
+
+- job: PostgresIntegrationTest
+  pool:
+    vmImage: 'vs2017-win2016'
+  dependsOn: UnitTest
+
+  steps:
+  - pwsh: |
+      choco install postgresql --params '/Password:password' --params-global --version 10.6
+      Set-Content "c:\program files\postgresql\10\data\pg_hba.conf" "host   all   all   ::1/128        trust"
+      Add-Content "c:\program files\postgresql\10\data\pg_hba.conf" "host   all   all   127.0.0.1/32   trust"
+      # the service name is "postgresql-x64-10", conveniently it's both the display name and the actual name
+      Restart-Service postgresql-x64-10
+
+      & "C:\program files\postgresql\10\bin\createdb.exe" -U postgres dbt
+      & "C:\program files\postgresql\10\bin\psql.exe"  -U postgres -c "CREATE ROLE root WITH PASSWORD 'password';"
+      & "C:\program files\postgresql\10\bin\psql.exe"  -U postgres -c "ALTER ROLE root WITH LOGIN;"
+      & "C:\program files\postgresql\10\bin\psql.exe"  -U postgres -c "GRANT CREATE, CONNECT ON DATABASE dbt TO root WITH GRANT OPTION;"
+      & "C:\program files\postgresql\10\bin\psql.exe"  -U postgres -c "CREATE ROLE noaccess WITH PASSWORD 'password' NOSUPERUSER;"
+      & "C:\program files\postgresql\10\bin\psql.exe"  -U postgres -c "ALTER ROLE noaccess WITH LOGIN;"
+      & "C:\program files\postgresql\10\bin\psql.exe"  -U postgres -c "GRANT CONNECT ON DATABASE dbt TO noaccess;"
+    displayName: Install postgresql and set up database
+
+  - task: UsePythonVersion@0
+    inputs:
+      versionSpec: '3.5'
+      architecture: 'x64'
+
+  - script: python -m pip install --upgrade pip && pip install tox
+    displayName: 'Install dependencies'
+
+  - script: python -m tox -e pywin-postgres
+    displayName: Run integration tests
+
+# These three are all similar except secure environment variables, which MUST be passed along to their tasks,
+# but there's probably a better way to do this!
+- job: SnowflakeIntegrationTest
+  pool:
+    vmImage: 'vs2017-win2016'
+  dependsOn: PostgresIntegrationTest
+
+  steps:
+  - task: UsePythonVersion@0
+    inputs:
+      versionSpec: '3.5'
+      architecture: 'x64'
+
+  - script: python -m pip install --upgrade pip && pip install tox
+    displayName: 'Install dependencies'
+
+  - script: python -m tox -e pywin-snowflake
+    env:
+      SNOWFLAKE_TEST_ACCOUNT: $(SNOWFLAKE_TEST_ACCOUNT)
+      SNOWFLAKE_TEST_PASSWORD: $(SNOWFLAKE_TEST_PASSWORD)
+      SNOWFLAKE_TEST_USER: $(SNOWFLAKE_TEST_USER)
+      SNOWFLAKE_TEST_WAREHOUSE: $(SNOWFLAKE_TEST_WAREHOUSE)
+    displayName: Run integration tests
+
+- job: BigQueryIntegrationTest
+  pool:
+    vmImage: 'vs2017-win2016'
+  dependsOn: PostgresIntegrationTest
+
+  steps:
+  - task: UsePythonVersion@0
+    inputs:
+      versionSpec: '3.5'
+      architecture: 'x64'
+
+  - script: python -m pip install --upgrade pip && pip install tox
+    displayName: 'Install dependencies'
+  - script: python -m tox -e pywin-bigquery
+    env:
+      BIGQUERY_SERVICE_ACCOUNT_JSON: $(BIGQUERY_SERVICE_ACCOUNT_JSON)
+    displayName: Run integration tests
+
+- job: RedshiftIntegrationTest
+  pool:
+    vmImage: 'vs2017-win2016'
+  dependsOn: PostgresIntegrationTest
+
+  steps:
+  - task: UsePythonVersion@0
+    inputs:
+      versionSpec: '3.5'
+      architecture: 'x64'
+
+  - script: python -m pip install --upgrade pip && pip install tox
+    displayName: 'Install dependencies'
+
+  - script: python -m tox -e pywin-redshift
+    env:
+      REDSHIFT_TEST_DBNAME: $(REDSHIFT_TEST_DBNAME)
+      REDSHIFT_TEST_PASS: $(REDSHIFT_TEST_PASS)
+      REDSHIFT_TEST_USER: $(REDSHIFT_TEST_USER)
+      REDSHIFT_TEST_PORT: $(REDSHIFT_TEST_PORT)
+      REDSHIFT_TEST_HOST: $(REDSHIFT_TEST_HOST)
+    displayName: Run integration tests

circle.yml

@@ -1,5 +0,0 @@
-dependencies:
-  pre:
-    - sudo add-apt-repository -y ppa:fkrull/deadsnakes
-    - sudo apt-get update
-    - sudo apt-get install python3.5 python3.5-dev

converter.py

@@ -0,0 +1,73 @@
+#!/usr/bin/env python
+import json
+import yaml
+import sys
+import argparse
+from datetime import datetime, timezone
+import dbt.clients.registry as registry
+
+
+def yaml_type(fname):
+    with open(fname) as f:
+        return yaml.load(f)
+
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--project", type=yaml_type, default="dbt_project.yml")
+    parser.add_argument("--namespace", required=True)
+    return parser.parse_args()
+
+
+def get_full_name(args):
+    return "{}/{}".format(args.namespace, args.project["name"])
+
+
+def init_project_in_packages(args, packages):
+    full_name = get_full_name(args)
+    if full_name not in packages:
+        packages[full_name] = {
+            "name": args.project["name"],
+            "namespace": args.namespace,
+            "latest": args.project["version"],
+            "assets": {},
+            "versions": {},
+        }
+    return packages[full_name]
+
+
+def add_version_to_package(args, project_json):
+    project_json["versions"][args.project["version"]] = {
+        "id": "{}/{}".format(get_full_name(args), args.project["version"]),
+        "name": args.project["name"],
+        "version": args.project["version"],
+        "description": "",
+        "published_at": datetime.now(timezone.utc).astimezone().isoformat(),
+        "packages": args.project.get("packages") or [],
+        "works_with": [],
+        "_source": {
+            "type": "github",
+            "url": "",
+            "readme": "",
+        },
+        "downloads": {
+            "tarball": "",
+            "format": "tgz",
+            "sha1": "",
+        },
+    }
+
+
+def main():
+    args = parse_args()
+    packages = registry.packages()
+    project_json = init_project_in_packages(args, packages)
+    if args.project["version"] in project_json["versions"]:
+        raise Exception("Version {} already in packages JSON"
+                        .format(args.project["version"]),
+                        file=sys.stderr)
+    add_version_to_package(args, project_json)
+    print(json.dumps(packages, indent=2))
+
+if __name__ == "__main__":
+    main()

core/MANIFEST.in

@@ -0,0 +1 @@
+recursive-include dbt/include *.py *.sql *.yml *.html *.md

core/dbt/__init__.py

@@ -0,0 +1 @@
+__path__ = __import__('pkgutil').extend_path(__path__, __name__)

core/dbt/adapters/__init__.py

@@ -0,0 +1 @@
+__path__ = __import__('pkgutil').extend_path(__path__, __name__)

core/dbt/adapters/base/__init__.py

@@ -0,0 +1,8 @@
+# these are all just exports, #noqa them so flake8 will be happy
+from dbt.adapters.base.meta import available  # noqa
+from dbt.adapters.base.relation import BaseRelation  # noqa
+from dbt.adapters.base.relation import Column  # noqa
+from dbt.adapters.base.connections import BaseConnectionManager  # noqa
+from dbt.adapters.base.connections import Credentials  # noqa
+from dbt.adapters.base.impl import BaseAdapter  # noqa
+from dbt.adapters.base.plugin import AdapterPlugin  # noqa

core/dbt/adapters/base/connections.py

@@ -0,0 +1,329 @@
+import abc
+import multiprocessing
+import os
+
+import six
+
+import dbt.exceptions
+import dbt.flags
+from dbt.api import APIObject
+from dbt.compat import abstractclassmethod, get_ident
+from dbt.contracts.connection import Connection
+from dbt.logger import GLOBAL_LOGGER as logger
+from dbt.utils import translate_aliases
+
+
+class Credentials(APIObject):
+    """Common base class for credentials. This is not valid to instantiate"""
+    SCHEMA = NotImplemented
+    # map credential aliases to their canonical names.
+    ALIASES = {}
+
+    def __init__(self, **kwargs):
+        renamed = self.translate_aliases(kwargs)
+        super(Credentials, self).__init__(**renamed)
+
+    @property
+    def type(self):
+        raise NotImplementedError(
+            'type not implemented for base credentials class'
+        )
+
+    def connection_info(self):
+        """Return an ordered iterator of key/value pairs for pretty-printing.
+        """
+        for key in self._connection_keys():
+            if key in self._contents:
+                yield key, self._contents[key]
+
+    def _connection_keys(self):
+        """The credential object keys that should be printed to users in
+        'dbt debug' output. This is specific to each adapter.
+        """
+        raise NotImplementedError
+
+    def incorporate(self, **kwargs):
+        # implementation note: we have to do this here, or
+        # incorporate(alias_name=...) will result in duplicate keys in the
+        # merged dict that APIObject.incorporate() creates.
+        renamed = self.translate_aliases(kwargs)
+        return super(Credentials, self).incorporate(**renamed)
+
+    def serialize(self, with_aliases=False):
+        serialized = super(Credentials, self).serialize()
+        if with_aliases:
+            serialized.update({
+                new_name: serialized[canonical_name]
+                for new_name, canonical_name in self.ALIASES.items()
+                if canonical_name in serialized
+            })
+        return serialized
+
+    @classmethod
+    def translate_aliases(cls, kwargs):
+        return translate_aliases(kwargs, cls.ALIASES)
+
+
+@six.add_metaclass(abc.ABCMeta)
+class BaseConnectionManager(object):
+    """Methods to implement:
+        - exception_handler
+        - cancel_open
+        - open
+        - begin
+        - commit
+        - clear_transaction
+        - execute
+
+    You must also set the 'TYPE' class attribute with a class-unique constant
+    string.
+    """
+    TYPE = NotImplemented
+
+    def __init__(self, profile):
+        self.profile = profile
+        self.thread_connections = {}
+        self.lock = multiprocessing.RLock()
+
+    @staticmethod
+    def get_thread_identifier():
+        # note that get_ident() may be re-used, but we should never experience
+        # that within a single process
+        return (os.getpid(), get_ident())
+
+    def get_thread_connection(self):
+        key = self.get_thread_identifier()
+        with self.lock:
+            if key not in self.thread_connections:
+                raise RuntimeError(
+                    'connection never acquired for thread {}, have {}'
+                    .format(key, list(self.thread_connections))
+                )
+            return self.thread_connections[key]
+
+    def get_if_exists(self):
+        key = self.get_thread_identifier()
+        with self.lock:
+            return self.thread_connections.get(key)
+
+    def clear_thread_connection(self):
+        key = self.get_thread_identifier()
+        with self.lock:
+            if key in self.thread_connections:
+                del self.thread_connections[key]
+
+    def clear_transaction(self):
+        """Clear any existing transactions."""
+        conn = self.get_thread_connection()
+        if conn is not None:
+            if conn.transaction_open:
+                self._rollback(conn)
+            self.begin()
+            self.commit()
+
+    @abc.abstractmethod
+    def exception_handler(self, sql):
+        """Create a context manager that handles exceptions caused by database
+        interactions.
+
+        :param str sql: The SQL string that the block inside the context
+            manager is executing.
+        :return: A context manager that handles exceptions raised by the
+            underlying database.
+        """
+        raise dbt.exceptions.NotImplementedException(
+            '`exception_handler` is not implemented for this adapter!')
+
+    def set_connection_name(self, name=None):
+        if name is None:
+            # if a name isn't specified, we'll re-use a single handle
+            # named 'master'
+            name = 'master'
+
+        conn = self.get_if_exists()
+        thread_id_key = self.get_thread_identifier()
+
+        if conn is None:
+            conn = Connection(
+                type=self.TYPE,
+                name=None,
+                state='init',
+                transaction_open=False,
+                handle=None,
+                credentials=self.profile.credentials
+            )
+            self.thread_connections[thread_id_key] = conn
+
+        if conn.name == name and conn.state == 'open':
+            return conn
+
+        logger.debug('Acquiring new {} connection "{}".'
+                     .format(self.TYPE, name))
+
+        if conn.state == 'open':
+            logger.debug(
+                'Re-using an available connection from the pool (formerly {}).'
+                .format(conn.name))
+        else:
+            logger.debug('Opening a new connection, currently in state {}'
+                         .format(conn.state))
+            self.open(conn)
+
+        conn.name = name
+        return conn
+
+    @abc.abstractmethod
+    def cancel_open(self):
+        """Cancel all open connections on the adapter. (passable)"""
+        raise dbt.exceptions.NotImplementedException(
+            '`cancel_open` is not implemented for this adapter!'
+        )
+
+    @abstractclassmethod
+    def open(cls, connection):
+        """Open a connection on the adapter.
+
+        This may mutate the given connection (in particular, its state and its
+        handle).
+
+        This should be thread-safe, or hold the lock if necessary. The given
+        connection should not be in either in_use or available.
+
+        :param Connection connection: A connection object to open.
+        :return: A connection with a handle attached and an 'open' state.
+        :rtype: Connection
+        """
+        raise dbt.exceptions.NotImplementedException(
+            '`open` is not implemented for this adapter!'
+        )
+
+    def release(self):
+        with self.lock:
+            conn = self.get_if_exists()
+            if conn is None:
+                return
+
+        try:
+            if conn.state == 'open':
+                if conn.transaction_open is True:
+                    self._rollback(conn)
+            else:
+                self.close(conn)
+        except Exception:
+            # if rollback or close failed, remove our busted connection
+            self.clear_thread_connection()
+            raise
+
+    def cleanup_all(self):
+        with self.lock:
+            for connection in self.thread_connections.values():
+                if connection.state not in {'closed', 'init'}:
+                    logger.debug("Connection '{}' was left open."
+                                 .format(connection.name))
+                else:
+                    logger.debug("Connection '{}' was properly closed."
+                                 .format(connection.name))
+                self.close(connection)
+
+            # garbage collect these connections
+            self.thread_connections.clear()
+
+    @abc.abstractmethod
+    def begin(self):
+        """Begin a transaction. (passable)
+
+        :param str name: The name of the connection to use.
+        """
+        raise dbt.exceptions.NotImplementedException(
+            '`begin` is not implemented for this adapter!'
+        )
+
+    @abc.abstractmethod
+    def commit(self):
+        """Commit a transaction. (passable)"""
+        raise dbt.exceptions.NotImplementedException(
+            '`commit` is not implemented for this adapter!'
+        )
+
+    @classmethod
+    def _rollback_handle(cls, connection):
+        """Perform the actual rollback operation."""
+        try:
+            connection.handle.rollback()
+        except Exception:
+            logger.debug(
+                'Failed to rollback {}'.format(connection.name),
+                exc_info=True
+            )
+
+    @classmethod
+    def _close_handle(cls, connection):
+        """Perform the actual close operation."""
+        # On windows, sometimes connection handles don't have a close() attr.
+        if hasattr(connection.handle, 'close'):
+            logger.debug('On {}: Close'.format(connection.name))
+            connection.handle.close()
+        else:
+            logger.debug('On {}: No close available on handle'
+                         .format(connection.name))
+
+    @classmethod
+    def _rollback(cls, connection):
+        """Roll back the given connection.
+        """
+        if dbt.flags.STRICT_MODE:
+            assert isinstance(connection, Connection)
+
+        if connection.transaction_open is False:
+            raise dbt.exceptions.InternalException(
+                'Tried to rollback transaction on connection "{}", but '
+                'it does not have one open!'.format(connection.name))
+
+        logger.debug('On {}: ROLLBACK'.format(connection.name))
+        cls._rollback_handle(connection)
+
+        connection.transaction_open = False
+
+        return connection
+
+    @classmethod
+    def close(cls, connection):
+        if dbt.flags.STRICT_MODE:
+            assert isinstance(connection, Connection)
+
+        # if the connection is in closed or init, there's nothing to do
+        if connection.state in {'closed', 'init'}:
+            return connection
+
+        if connection.transaction_open and connection.handle:
+            cls._rollback_handle(connection)
+        connection.transaction_open = False
+
+        cls._close_handle(connection)
+        connection.state = 'closed'
+
+        return connection
+
+    def commit_if_has_connection(self):
+        """If the named connection exists, commit the current transaction.
+
+        :param str name: The name of the connection to use.
+        """
+        connection = self.get_if_exists()
+        if connection:
+            self.commit()
+
+    @abc.abstractmethod
+    def execute(self, sql, auto_begin=False, fetch=False):
+        """Execute the given SQL.
+
+        :param str sql: The sql to execute.
+        :param bool auto_begin: If set, and dbt is not currently inside a
+            transaction, automatically begin one.
+        :param bool fetch: If set, fetch results.
+        :return: A tuple of the status and the results (empty if fetch=False).
+        :rtype: Tuple[str, agate.Table]
+        """
+        raise dbt.exceptions.NotImplementedException(
+            '`execute` is not implemented for this adapter!'
+        )

core/dbt/adapters/base/impl.py

@@ -0,0 +1,957 @@
+import abc
+from contextlib import contextmanager
+
+import agate
+import pytz
+import six
+
+import dbt.exceptions
+import dbt.flags
+import dbt.clients.agate_helper
+
+from dbt.compat import abstractclassmethod, classmethod
+from dbt.node_types import NodeType
+from dbt.loader import GraphLoader
+from dbt.logger import GLOBAL_LOGGER as logger
+from dbt.utils import filter_null_values
+
+
+from dbt.adapters.base.meta import AdapterMeta, available
+from dbt.adapters.base import BaseRelation
+from dbt.adapters.base import Column
+from dbt.adapters.cache import RelationsCache
+
+
+GET_CATALOG_MACRO_NAME = 'get_catalog'
+FRESHNESS_MACRO_NAME = 'collect_freshness'
+
+
+def _expect_row_value(key, row):
+    if key not in row.keys():
+        raise dbt.exceptions.InternalException(
+            'Got a row without "{}" column, columns: {}'
+            .format(key, row.keys())
+        )
+    return row[key]
+
+
+def _relations_filter_schemas(schemas):
+    def test(row):
+        referenced_schema = _expect_row_value('referenced_schema', row)
+        dependent_schema = _expect_row_value('dependent_schema', row)
+        # handle the null schema
+        if referenced_schema is not None:
+            referenced_schema = referenced_schema.lower()
+        if dependent_schema is not None:
+            dependent_schema = dependent_schema.lower()
+        return referenced_schema in schemas or dependent_schema in schemas
+    return test
+
+
+def _catalog_filter_schemas(manifest):
+    """Return a function that takes a row and decides if the row should be
+    included in the catalog output.
+    """
+    schemas = frozenset((d.lower(), s.lower())
+                        for d, s in manifest.get_used_schemas())
+
+    def test(row):
+        table_database = _expect_row_value('table_database', row)
+        table_schema = _expect_row_value('table_schema', row)
+        # the schema may be present but None, which is not an error and should
+        # be filtered out
+        if table_schema is None:
+            return False
+        return (table_database.lower(), table_schema.lower()) in schemas
+    return test
+
+
+def _utc(dt, source, field_name):
+    """If dt has a timezone, return a new datetime that's in UTC. Otherwise,
+    assume the datetime is already for UTC and add the timezone.
+    """
+    if dt is None:
+        raise dbt.exceptions.raise_database_error(
+            "Expected a non-null value when querying field '{}' of table "
+            " {} but received value 'null' instead".format(
+                field_name,
+                source))
+
+    elif not hasattr(dt, 'tzinfo'):
+        raise dbt.exceptions.raise_database_error(
+            "Expected a timestamp value when querying field '{}' of table "
+            "{} but received value of type '{}' instead".format(
+                field_name,
+                source,
+                type(dt).__name__))
+
+    elif dt.tzinfo:
+        return dt.astimezone(pytz.UTC)
+    else:
+        return dt.replace(tzinfo=pytz.UTC)
+
+
+class SchemaSearchMap(dict):
+    """A utility class to keep track of what information_schema tables to
+    search for what schemas
+    """
+    def add(self, relation):
+        key = relation.information_schema_only()
+        if key not in self:
+            self[key] = set()
+        self[key].add(relation.schema.lower())
+
+    def search(self):
+        for information_schema_name, schemas in self.items():
+            for schema in schemas:
+                yield information_schema_name, schema
+
+    def schemas_searched(self):
+        result = set()
+        for information_schema_name, schemas in self.items():
+            result.update(
+                (information_schema_name.database, schema)
+                for schema in schemas
+            )
+        return result
+
+    def flatten(self):
+        new = self.__class__()
+
+        database = None
+        # iterate once to look for a database name
+        seen = {r.database.lower() for r in self if r.database}
+        if len(seen) > 1:
+            dbt.exceptions.raise_compiler_error(str(seen))
+        elif len(seen) == 1:
+            database = list(seen)[0]
+
+        for information_schema_name, schema in self.search():
+            new.add(information_schema_name.incorporate(
+                path={'database': database, 'schema': schema},
+                quote_policy={'database': False},
+                include_policy={'database': False},
+            ))
+
+        return new
+
+
+@six.add_metaclass(AdapterMeta)
+class BaseAdapter(object):
+    """The BaseAdapter provides an abstract base class for adapters.
+
+    Adapters must implement the following methods and macros. Some of the
+    methods can be safely overridden as a noop, where it makes sense
+    (transactions on databases that don't support them, for instance). Those
+    methods are marked with a (passable) in their docstrings. Check docstrings
+    for type information, etc.
+
+    To implement a macro, implement "${adapter_type}__${macro_name}". in the
+    adapter's internal project.
+
+    Methods:
+        - exception_handler
+        - date_function
+        - list_schemas
+        - drop_relation
+        - truncate_relation
+        - rename_relation
+        - get_columns_in_relation
+        - expand_column_types
+        - list_relations_without_caching
+        - is_cancelable
+        - create_schema
+        - drop_schema
+        - quote
+        - convert_text_type
+        - convert_number_type
+        - convert_boolean_type
+        - convert_datetime_type
+        - convert_date_type
+        - convert_time_type
+
+    Macros:
+        - get_catalog
+    """
+    requires = {}
+
+    Relation = BaseRelation
+    Column = Column
+    # This should be an implementation of BaseConnectionManager
+    ConnectionManager = None
+
+    # A set of clobber config fields accepted by this adapter
+    # for use in materializations
+    AdapterSpecificConfigs = frozenset()
+
+    def __init__(self, config):
+        self.config = config
+        self.cache = RelationsCache()
+        self.connections = self.ConnectionManager(config)
+        self._internal_manifest_lazy = None
+
+    ###
+    # Methods that pass through to the connection manager
+    ###
+    def acquire_connection(self, name=None):
+        return self.connections.set_connection_name(name)
+
+    def release_connection(self):
+        return self.connections.release()
+
+    def cleanup_connections(self):
+        return self.connections.cleanup_all()
+
+    def clear_transaction(self):
+        self.connections.clear_transaction()
+
+    def commit_if_has_connection(self):
+        return self.connections.commit_if_has_connection()
+
+    def nice_connection_name(self):
+        conn = self.connections.get_thread_connection()
+        if conn is None or conn.name is None:
+            return '<None>'
+        return conn.name
+
+    @contextmanager
+    def connection_named(self, name):
+        try:
+            yield self.acquire_connection(name)
+        finally:
+            self.release_connection()
+
+    @available.parse(lambda *a, **k: ('', dbt.clients.agate_helper()))
+    def execute(self, sql, auto_begin=False, fetch=False):
+        """Execute the given SQL. This is a thin wrapper around
+        ConnectionManager.execute.
+
+        :param str sql: The sql to execute.
+        :param bool auto_begin: If set, and dbt is not currently inside a
+            transaction, automatically begin one.
+        :param bool fetch: If set, fetch results.
+        :return: A tuple of the status and the results (empty if fetch=False).
+        :rtype: Tuple[str, agate.Table]
+        """
+        return self.connections.execute(
+            sql=sql,
+            auto_begin=auto_begin,
+            fetch=fetch
+        )
+
+    ###
+    # Methods that should never be overridden
+    ###
+    @classmethod
+    def type(cls):
+        """Get the type of this adapter. Types must be class-unique and
+        consistent.
+
+        :return: The type name
+        :rtype: str
+        """
+        return cls.ConnectionManager.TYPE
+
+    @property
+    def _internal_manifest(self):
+        if self._internal_manifest_lazy is None:
+            manifest = GraphLoader.load_internal(self.config)
+            self._internal_manifest_lazy = manifest
+        return self._internal_manifest_lazy
+
+    def check_internal_manifest(self):
+        """Return the internal manifest (used for executing macros) if it's
+        been initialized, otherwise return None.
+        """
+        return self._internal_manifest_lazy
+
+    ###
+    # Caching methods
+    ###
+    def _schema_is_cached(self, database, schema):
+        """Check if the schema is cached, and by default logs if it is not."""
+
+        if dbt.flags.USE_CACHE is False:
+            return False
+        elif (database, schema) not in self.cache:
+            logger.debug(
+                'On "{}": cache miss for schema "{}.{}", this is inefficient'
+                .format(self.nice_connection_name(), database, schema)
+            )
+            return False
+        else:
+            return True
+
+    @classmethod
+    def _relations_filter_table(cls, table, schemas):
+        """Filter the table as appropriate for relations table entries.
+        Subclasses can override this to change filtering rules on a per-adapter
+        basis.
+        """
+        return table.where(_relations_filter_schemas(schemas))
+
+    def _get_cache_schemas(self, manifest, exec_only=False):
+        """Get a mapping of each node's "information_schema" relations to a
+        set of all schemas expected in that information_schema.
+
+        There may be keys that are technically duplicates on the database side,
+        for example all of '"foo", 'foo', '"FOO"' and 'FOO' could coexist as
+        databases, and values could overlap as appropriate. All values are
+        lowercase strings.
+        """
+        info_schema_name_map = SchemaSearchMap()
+        for node in manifest.nodes.values():
+            if exec_only and node.resource_type not in NodeType.executable():
+                continue
+            relation = self.Relation.create_from(self.config, node)
+            info_schema_name_map.add(relation)
+        # result is a map whose keys are information_schema Relations without
+        # identifiers that have appropriate database prefixes, and whose values
+        # are sets of lowercase schema names that are valid members of those
+        # schemas
+        return info_schema_name_map
+
+    def _relations_cache_for_schemas(self, manifest):
+        """Populate the relations cache for the given schemas. Returns an
+        iteratble of the schemas populated, as strings.
+        """
+        if not dbt.flags.USE_CACHE:
+            return
+
+        info_schema_name_map = self._get_cache_schemas(manifest,
+                                                       exec_only=True)
+        for db, schema in info_schema_name_map.search():
+            for relation in self.list_relations_without_caching(db, schema):
+                self.cache.add(relation)
+
+        # it's possible that there were no relations in some schemas. We want
+        # to insert the schemas we query into the cache's `.schemas` attribute
+        # so we can check it later
+        self.cache.update_schemas(info_schema_name_map.schemas_searched())
+
+    def set_relations_cache(self, manifest, clear=False):
+        """Run a query that gets a populated cache of the relations in the
+        database and set the cache on this adapter.
+        """
+        if not dbt.flags.USE_CACHE:
+            return
+
+        with self.cache.lock:
+            if clear:
+                self.cache.clear()
+            self._relations_cache_for_schemas(manifest)
+
+    def cache_new_relation(self, relation):
+        """Cache a new relation in dbt. It will show up in `list relations`."""
+        if relation is None:
+            name = self.nice_connection_name()
+            dbt.exceptions.raise_compiler_error(
+                'Attempted to cache a null relation for {}'.format(name)
+            )
+        if dbt.flags.USE_CACHE:
+            self.cache.add(relation)
+        # so jinja doesn't render things
+        return ''
+
+    ###
+    # Abstract methods for database-specific values, attributes, and types
+    ###
+    @abstractclassmethod
+    def date_function(cls):
+        """Get the date function used by this adapter's database.
+
+        :return: The date function
+        :rtype: str
+        """
+        raise dbt.exceptions.NotImplementedException(
+            '`date_function` is not implemented for this adapter!')
+
+    @abstractclassmethod
+    def is_cancelable(cls):
+        raise dbt.exceptions.NotImplementedException(
+            '`is_cancelable` is not implemented for this adapter!'
+        )
+
+    ###
+    # Abstract methods about schemas
+    ###
+    @abc.abstractmethod
+    def list_schemas(self, database):
+        """Get a list of existing schemas.
+
+        :param str database: The name of the database to list under.
+        :return: All schemas that currently exist in the database
+        :rtype: List[str]
+        """
+        raise dbt.exceptions.NotImplementedException(
+            '`list_schemas` is not implemented for this adapter!'
+        )
+
+    @available.parse(lambda *a, **k: False)
+    def check_schema_exists(self, database, schema):
+        """Check if a schema exists.
+
+        The default implementation of this is potentially unnecessarily slow,
+        and adapters should implement it if there is an optimized path (and
+        there probably is)
+        """
+        search = (
+            s.lower() for s in
+            self.list_schemas(database=database)
+        )
+        return schema.lower() in search
+
+    ###
+    # Abstract methods about relations
+    ###
+    @abc.abstractmethod
+    @available.parse_none
+    def drop_relation(self, relation):
+        """Drop the given relation.
+
+        *Implementors must call self.cache.drop() to preserve cache state!*
+
+        :param self.Relation relation: The relation to drop
+        """
+        raise dbt.exceptions.NotImplementedException(
+            '`drop_relation` is not implemented for this adapter!'
+        )
+
+    @abc.abstractmethod
+    @available.parse_none
+    def truncate_relation(self, relation):
+        """Truncate the given relation.
+
+        :param self.Relation relation: The relation to truncate
+        """
+        raise dbt.exceptions.NotImplementedException(
+            '`truncate_relation` is not implemented for this adapter!'
+        )
+
+    @abc.abstractmethod
+    @available.parse_none
+    def rename_relation(self, from_relation, to_relation):
+        """Rename the relation from from_relation to to_relation.
+
+        Implementors must call self.cache.rename() to preserve cache state.
+
+        :param self.Relation from_relation: The original relation name
+        :param self.Relation to_relation: The new relation name
+        """
+        raise dbt.exceptions.NotImplementedException(
+            '`rename_relation` is not implemented for this adapter!'
+        )
+
+    @abc.abstractmethod
+    @available.parse_list
+    def get_columns_in_relation(self, relation):
+        """Get a list of the columns in the given Relation.
+
+        :param self.Relation relation: The relation to query for.
+        :return: Information about all columns in the given relation.
+        :rtype: List[self.Column]
+        """
+        raise dbt.exceptions.NotImplementedException(
+            '`get_columns_in_relation` is not implemented for this adapter!'
+        )
+
+    @available.deprecated('get_columns_in_relation', lambda *a, **k: [])
+    def get_columns_in_table(self, schema, identifier):
+        """DEPRECATED: Get a list of the columns in the given table."""
+        relation = self.Relation.create(
+            database=self.config.credentials.database,
+            schema=schema,
+            identifier=identifier,
+            quote_policy=self.config.quoting
+        )
+        return self.get_columns_in_relation(relation)
+
+    @abc.abstractmethod
+    def expand_column_types(self, goal, current):
+        """Expand the current table's types to match the goal table. (passable)
+
+        :param self.Relation goal: A relation that currently exists in the
+            database with columns of the desired types.
+        :param self.Relation current: A relation that currently exists in the
+            database with columns of unspecified types.
+        """
+        raise dbt.exceptions.NotImplementedException(
+            '`expand_target_column_types` is not implemented for this adapter!'
+        )
+
+    @abc.abstractmethod
+    def list_relations_without_caching(self, information_schema, schema):
+        """List relations in the given schema, bypassing the cache.
+
+        This is used as the underlying behavior to fill the cache.
+
+        :param Relation information_schema: The information schema to list
+            relations from.
+        :param str schema: The name of the schema to list relations from.
+        :return: The relations in schema
+        :rtype: List[self.Relation]
+        """
+        raise dbt.exceptions.NotImplementedException(
+            '`list_relations_without_caching` is not implemented for this '
+            'adapter!'
+        )
+
+    ###
+    # Provided methods about relations
+    ###
+    @available.parse_list
+    def get_missing_columns(self, from_relation, to_relation):
+        """Returns a list of Columns in from_relation that are missing from
+        to_relation.
+
+        :param Relation from_relation: The relation that might have extra
+            columns
+        :param Relation to_relation: The realtion that might have columns
+            missing
+        :return: The columns in from_relation that are missing from to_relation
+        :rtype: List[self.Relation]
+        """
+        if not isinstance(from_relation, self.Relation):
+            dbt.exceptions.invalid_type_error(
+                method_name='get_missing_columns',
+                arg_name='from_relation',
+                got_value=from_relation,
+                expected_type=self.Relation)
+
+        if not isinstance(to_relation, self.Relation):
+            dbt.exceptions.invalid_type_error(
+                method_name='get_missing_columns',
+                arg_name='to_relation',
+                got_value=to_relation,
+                expected_type=self.Relation)
+
+        from_columns = {
+            col.name: col for col in
+            self.get_columns_in_relation(from_relation)
+        }
+
+        to_columns = {
+            col.name: col for col in
+            self.get_columns_in_relation(to_relation)
+        }
+
+        missing_columns = set(from_columns.keys()) - set(to_columns.keys())
+
+        return [
+            col for (col_name, col) in from_columns.items()
+            if col_name in missing_columns
+        ]
+
+    @available.parse_none
+    def valid_snapshot_target(self, relation):
+        """Ensure that the target relation is valid, by making sure it has the
+        expected columns.
+
+        :param Relation relation: The relation to check
+        :raises dbt.exceptions.CompilationException: If the columns are
+            incorrect.
+        """
+        if not isinstance(relation, self.Relation):
+            dbt.exceptions.invalid_type_error(
+                method_name='valid_snapshot_target',
+                arg_name='relation',
+                got_value=relation,
+                expected_type=self.Relation)
+
+        columns = self.get_columns_in_relation(relation)
+        names = set(c.name.lower() for c in columns)
+        expanded_keys = ('scd_id', 'valid_from', 'valid_to')
+        extra = []
+        missing = []
+        for legacy in expanded_keys:
+            desired = 'dbt_' + legacy
+            if desired not in names:
+                missing.append(desired)
+                if legacy in names:
+                    extra.append(legacy)
+
+        if missing:
+            if extra:
+                msg = (
+                    'Snapshot target has ("{}") but not ("{}") - is it an '
+                    'unmigrated previous version archive?'
+                    .format('", "'.join(extra), '", "'.join(missing))
+                )
+            else:
+                msg = (
+                    'Snapshot target is not a snapshot table (missing "{}")'
+                    .format('", "'.join(missing))
+                )
+            dbt.exceptions.raise_compiler_error(msg)
+
+    @available.parse_none
+    def expand_target_column_types(self, from_relation, to_relation):
+        if not isinstance(from_relation, self.Relation):
+            dbt.exceptions.invalid_type_error(
+                method_name='expand_target_column_types',
+                arg_name='from_relation',
+                got_value=from_relation,
+                expected_type=self.Relation)
+
+        if not isinstance(to_relation, self.Relation):
+            dbt.exceptions.invalid_type_error(
+                method_name='expand_target_column_types',
+                arg_name='to_relation',
+                got_value=to_relation,
+                expected_type=self.Relation)
+
+        self.expand_column_types(from_relation, to_relation)
+
+    def list_relations(self, database, schema):
+        if self._schema_is_cached(database, schema):
+            return self.cache.get_relations(database, schema)
+
+        information_schema = self.Relation.create(
+            database=database,
+            schema=schema,
+            model_name='',
+            quote_policy=self.config.quoting
+        ).information_schema()
+
+        # we can't build the relations cache because we don't have a
+        # manifest so we can't run any operations.
+        relations = self.list_relations_without_caching(
+            information_schema, schema
+        )
+
+        logger.debug('with database={}, schema={}, relations={}'
+                     .format(database, schema, relations))
+        return relations
+
+    def _make_match_kwargs(self, database, schema, identifier):
+        quoting = self.config.quoting
+        if identifier is not None and quoting['identifier'] is False:
+            identifier = identifier.lower()
+
+        if schema is not None and quoting['schema'] is False:
+            schema = schema.lower()
+
+        if database is not None and quoting['database'] is False:
+            database = database.lower()
+
+        return filter_null_values({
+            'database': database,
+            'identifier': identifier,
+            'schema': schema,
+        })
+
+    def _make_match(self, relations_list, database, schema, identifier):
+
+        matches = []
+
+        search = self._make_match_kwargs(database, schema, identifier)
+
+        for relation in relations_list:
+            if relation.matches(**search):
+                matches.append(relation)
+
+        return matches
+
+    @available.parse_none
+    def get_relation(self, database, schema, identifier):
+        relations_list = self.list_relations(database, schema)
+
+        matches = self._make_match(relations_list, database, schema,
+                                   identifier)
+
+        if len(matches) > 1:
+            kwargs = {
+                'identifier': identifier,
+                'schema': schema,
+                'database': database,
+            }
+            dbt.exceptions.get_relation_returned_multiple_results(
+                kwargs, matches
+            )
+
+        elif matches:
+            return matches[0]
+
+        return None
+
+    @available.deprecated('get_relation', lambda *a, **k: False)
+    def already_exists(self, schema, name):
+        """DEPRECATED: Return if a model already exists in the database"""
+        database = self.config.credentials.database
+        relation = self.get_relation(database, schema, name)
+        return relation is not None
+
+    ###
+    # ODBC FUNCTIONS -- these should not need to change for every adapter,
+    #                   although some adapters may override them
+    ###
+    @abc.abstractmethod
+    @available.parse_none
+    def create_schema(self, database, schema):
+        """Create the given schema if it does not exist.
+
+        :param str schema: The schema name to create.
+        """
+        raise dbt.exceptions.NotImplementedException(
+            '`create_schema` is not implemented for this adapter!'
+        )
+
+    @abc.abstractmethod
+    def drop_schema(self, database, schema):
+        """Drop the given schema (and everything in it) if it exists.
+
+        :param str schema: The schema name to drop.
+        """
+        raise dbt.exceptions.NotImplementedException(
+            '`drop_schema` is not implemented for this adapter!'
+        )
+
+    @available
+    @abstractclassmethod
+    def quote(cls, identifier):
+        """Quote the given identifier, as appropriate for the database.
+
+        :param str identifier: The identifier to quote
+        :return: The quoted identifier
+        :rtype: str
+        """
+        raise dbt.exceptions.NotImplementedException(
+            '`quote` is not implemented for this adapter!'
+        )
+
+    @available
+    def quote_as_configured(self, identifier, quote_key):
+        """Quote or do not quote the given identifer as configured in the
+        project config for the quote key.
+
+        The quote key should be one of 'database' (on bigquery, 'profile'),
+        'identifier', or 'schema', or it will be treated as if you set `True`.
+        """
+        default = self.Relation.DEFAULTS['quote_policy'].get(quote_key)
+        if self.config.quoting.get(quote_key, default):
+            return self.quote(identifier)
+        else:
+            return identifier
+
+    ###
+    # Conversions: These must be implemented by concrete implementations, for
+    # converting agate types into their sql equivalents.
+    ###
+    @abstractclassmethod
+    def convert_text_type(cls, agate_table, col_idx):
+        """Return the type in the database that best maps to the agate.Text
+        type for the given agate table and column index.
+
+        :param agate.Table agate_table: The table
+        :param int col_idx: The index into the agate table for the column.
+        :return: The name of the type in the database
+        :rtype: str
+        """
+        raise dbt.exceptions.NotImplementedException(
+            '`convert_text_type` is not implemented for this adapter!')
+
+    @abstractclassmethod
+    def convert_number_type(cls, agate_table, col_idx):
+        """Return the type in the database that best maps to the agate.Number
+        type for the given agate table and column index.
+
+        :param agate.Table agate_table: The table
+        :param int col_idx: The index into the agate table for the column.
+        :return: The name of the type in the database
+        :rtype: str
+        """
+        raise dbt.exceptions.NotImplementedException(
+            '`convert_number_type` is not implemented for this adapter!')
+
+    @abstractclassmethod
+    def convert_boolean_type(cls, agate_table, col_idx):
+        """Return the type in the database that best maps to the agate.Boolean
+        type for the given agate table and column index.
+
+        :param agate.Table agate_table: The table
+        :param int col_idx: The index into the agate table for the column.
+        :return: The name of the type in the database
+        :rtype: str
+        """
+        raise dbt.exceptions.NotImplementedException(
+            '`convert_boolean_type` is not implemented for this adapter!')
+
+    @abstractclassmethod
+    def convert_datetime_type(cls, agate_table, col_idx):
+        """Return the type in the database that best maps to the agate.DateTime
+        type for the given agate table and column index.
+
+        :param agate.Table agate_table: The table
+        :param int col_idx: The index into the agate table for the column.
+        :return: The name of the type in the database
+        :rtype: str
+        """
+        raise dbt.exceptions.NotImplementedException(
+            '`convert_datetime_type` is not implemented for this adapter!')
+
+    @abstractclassmethod
+    def convert_date_type(cls, agate_table, col_idx):
+        """Return the type in the database that best maps to the agate.Date
+        type for the given agate table and column index.
+
+        :param agate.Table agate_table: The table
+        :param int col_idx: The index into the agate table for the column.
+        :return: The name of the type in the database
+        :rtype: str
+        """
+        raise dbt.exceptions.NotImplementedException(
+            '`convert_date_type` is not implemented for this adapter!')
+
+    @abstractclassmethod
+    def convert_time_type(cls, agate_table, col_idx):
+        """Return the type in the database that best maps to the
+        agate.TimeDelta type for the given agate table and column index.
+
+        :param agate.Table agate_table: The table
+        :param int col_idx: The index into the agate table for the column.
+        :return: The name of the type in the database
+        :rtype: str
+        """
+        raise dbt.exceptions.NotImplementedException(
+            '`convert_time_type` is not implemented for this adapter!')
+
+    @available
+    @classmethod
+    def convert_type(cls, agate_table, col_idx):
+        return cls.convert_agate_type(agate_table, col_idx)
+
+    @classmethod
+    def convert_agate_type(cls, agate_table, col_idx):
+        agate_type = agate_table.column_types[col_idx]
+        conversions = [
+            (agate.Text, cls.convert_text_type),
+            (agate.Number, cls.convert_number_type),
+            (agate.Boolean, cls.convert_boolean_type),
+            (agate.DateTime, cls.convert_datetime_type),
+            (agate.Date, cls.convert_date_type),
+            (agate.TimeDelta, cls.convert_time_type),
+        ]
+        for agate_cls, func in conversions:
+            if isinstance(agate_type, agate_cls):
+                return func(agate_table, col_idx)
+
+    ###
+    # Operations involving the manifest
+    ###
+    def execute_macro(self, macro_name, manifest=None, project=None,
+                      context_override=None, kwargs=None, release=False):
+        """Look macro_name up in the manifest and execute its results.
+
+        :param str macro_name: The name of the macro to execute.
+        :param Optional[Manifest] manifest: The manifest to use for generating
+            the base macro execution context. If none is provided, use the
+            internal manifest.
+        :param Optional[str] project: The name of the project to search in, or
+            None for the first match.
+        :param Optional[dict] context_override: An optional dict to update()
+            the macro execution context.
+        :param Optional[dict] kwargs: An optional dict of keyword args used to
+            pass to the macro.
+        :param bool release: If True, release the connection after executing.
+
+        Return an an AttrDict with three attributes: 'table', 'data', and
+            'status'. 'table' is an agate.Table.
+        """
+        if kwargs is None:
+            kwargs = {}
+        if context_override is None:
+            context_override = {}
+
+        if manifest is None:
+            manifest = self._internal_manifest
+
+        macro = manifest.find_macro_by_name(macro_name, project)
+        if macro is None:
+            if project is None:
+                package_name = 'any package'
+            else:
+                package_name = 'the "{}" package'.format(project)
+
+            # The import of dbt.context.runtime below shadows 'dbt'
+            import dbt.exceptions
+            raise dbt.exceptions.RuntimeException(
+                'dbt could not find a macro with the name "{}" in {}'
+                .format(macro_name, package_name)
+            )
+        # This causes a reference cycle, as dbt.context.runtime.generate()
+        # ends up calling get_adapter, so the import has to be here.
+        import dbt.context.operation
+        macro_context = dbt.context.operation.generate(
+            macro,
+            self.config,
+            manifest
+        )
+        macro_context.update(context_override)
+
+        macro_function = macro.generator(macro_context)
+
+        try:
+            result = macro_function(**kwargs)
+        finally:
+            if release:
+                self.release_connection()
+        return result
+
+    @classmethod
+    def _catalog_filter_table(cls, table, manifest):
+        """Filter the table as appropriate for catalog entries. Subclasses can
+        override this to change filtering rules on a per-adapter basis.
+        """
+        return table.where(_catalog_filter_schemas(manifest))
+
+    def get_catalog(self, manifest):
+        """Get the catalog for this manifest by running the get catalog macro.
+        Returns an agate.Table of catalog information.
+        """
+        information_schemas = list(self._get_cache_schemas(manifest).keys())
+        # make it a list so macros can index into it.
+        kwargs = {'information_schemas': information_schemas}
+        table = self.execute_macro(GET_CATALOG_MACRO_NAME,
+                                   kwargs=kwargs,
+                                   release=True)
+
+        results = self._catalog_filter_table(table, manifest)
+        return results
+
+    def cancel_open_connections(self):
+        """Cancel all open connections."""
+        return self.connections.cancel_open()
+
+    def calculate_freshness(self, source, loaded_at_field, manifest=None):
+        """Calculate the freshness of sources in dbt, and return it"""
+        # in the future `source` will be a Relation instead of a string
+        kwargs = {
+            'source': source,
+            'loaded_at_field': loaded_at_field
+        }
+
+        # run the macro
+        table = self.execute_macro(
+            FRESHNESS_MACRO_NAME,
+            kwargs=kwargs,
+            release=True,
+            manifest=manifest
+        )
+        # now we have a 1-row table of the maximum `loaded_at_field` value and
+        # the current time according to the db.
+        if len(table) != 1 or len(table[0]) != 2:
+            dbt.exceptions.raise_compiler_error(
+                'Got an invalid result from "{}" macro: {}'.format(
+                    FRESHNESS_MACRO_NAME, [tuple(r) for r in table]
+                )
+            )
+
+        max_loaded_at = _utc(table[0][0], source, loaded_at_field)
+        snapshotted_at = _utc(table[0][1], source, loaded_at_field)
+
+        age = (snapshotted_at - max_loaded_at).total_seconds()
+        return {
+            'max_loaded_at': max_loaded_at,
+            'snapshotted_at': snapshotted_at,
+            'age': age,
+        }

core/dbt/adapters/base/meta.py

@@ -0,0 +1,117 @@
+import abc
+from functools import wraps
+from dbt.deprecations import warn, renamed_method
+
+
+def _always_none(*args, **kwargs):
+    return None
+
+
+def _always_list(*args, **kwargs):
+    return None
+
+
+def available(func):
+    """A decorator to indicate that a method on the adapter will be
+    exposed to the database wrapper, and will be available at parse and run
+    time.
+    """
+    func._is_available_ = True
+    return func
+
+
+def available_deprecated(supported_name, parse_replacement=None):
+    """A decorator that marks a function as available, but also prints a
+    deprecation warning. Use like
+
+    @available_deprecated('my_new_method')
+    def my_old_method(self, arg):
+        args = compatability_shim(arg)
+        return self.my_new_method(*args)
+
+    @available_deprecated('my_new_slow_method', lambda *a, **k: (0, ''))
+    def my_old_slow_method(self, arg):
+        args = compatibility_shim(arg)
+        return self.my_new_slow_method(*args)
+
+    To make `adapter.my_old_method` available but also print out a warning on
+    use directing users to `my_new_method`.
+
+    The optional parse_replacement, if provided, will provide a parse-time
+    replacement for the actual method (see `available_parse`).
+    """
+    def wrapper(func):
+        func_name = func.__name__
+        renamed_method(func_name, supported_name)
+
+        @wraps(func)
+        def inner(*args, **kwargs):
+            warn('adapter:{}'.format(func_name))
+            return func(*args, **kwargs)
+
+        if parse_replacement:
+            available = available_parse(parse_replacement)
+        return available(inner)
+    return wrapper
+
+
+def available_parse(parse_replacement):
+    """A decorator factory to indicate that a method on the adapter will be
+    exposed to the database wrapper, and will be stubbed out at parse time with
+    the given function.
+
+    @available_parse()
+    def my_method(self, a, b):
+        if something:
+            return None
+        return big_expensive_db_query()
+
+    @available_parse(lambda *args, **args: {})
+    def my_other_method(self, a, b):
+        x = {}
+        x.update(big_expensive_db_query())
+        return x
+    """
+    def inner(func):
+        func._parse_replacement_ = parse_replacement
+        available(func)
+        return func
+    return inner
+
+
+available.deprecated = available_deprecated
+available.parse = available_parse
+available.parse_none = available_parse(lambda *a, **k: None)
+available.parse_list = available_parse(lambda *a, **k: [])
+
+
+class AdapterMeta(abc.ABCMeta):
+    def __new__(mcls, name, bases, namespace, **kwargs):
+        cls = super(AdapterMeta, mcls).__new__(mcls, name, bases, namespace,
+                                               **kwargs)
+
+        # this is very much inspired by ABCMeta's own implementation
+
+        # dict mapping the method name to whether the model name should be
+        # injected into the arguments. All methods in here are exposed to the
+        # context.
+        available = set()
+        replacements = {}
+
+        # collect base class data first
+        for base in bases:
+            available.update(getattr(base, '_available_', set()))
+            replacements.update(getattr(base, '_parse_replacements_', set()))
+
+        # override with local data if it exists
+        for name, value in namespace.items():
+            if getattr(value, '_is_available_', False):
+                available.add(name)
+            parse_replacement = getattr(value, '_parse_replacement_', None)
+            if parse_replacement is not None:
+                replacements[name] = parse_replacement
+
+        cls._available_ = frozenset(available)
+        # should this be a namedtuple so it will be immutable like _available_?
+        cls._parse_replacements_ = replacements
+        return cls

core/dbt/adapters/base/plugin.py

@@ -0,0 +1,23 @@
+from dbt.config.project import Project
+
+
+class AdapterPlugin(object):
+    """Defines the basic requirements for a dbt adapter plugin.
+
+    :param type adapter: An adapter class, derived from BaseAdapter
+    :param type credentials: A credentials object, derived from Credentials
+    :param str project_name: The name of this adapter plugin's associated dbt
+        project.
+    :param str include_path: The path to this adapter plugin's root
+    :param Optional[List[str]] dependencies: A list of adapter names that this
+        adapter depends upon.
+    """
+    def __init__(self, adapter, credentials, include_path, dependencies=None):
+        self.adapter = adapter
+        self.credentials = credentials
+        self.include_path = include_path
+        project = Project.from_project_root(include_path, {})
+        self.project_name = project.project_name
+        if dependencies is None:
+            dependencies = []
+        self.dependencies = dependencies

core/dbt/adapters/base/relation.py

@@ -0,0 +1,406 @@
+from dbt.api import APIObject
+from dbt.utils import filter_null_values
+from dbt.node_types import NodeType
+
+import dbt.exceptions
+
+
+class BaseRelation(APIObject):
+
+    Table = "table"
+    View = "view"
+    CTE = "cte"
+    MaterializedView = "materializedview"
+    ExternalTable = "externaltable"
+
+    RelationTypes = [
+        Table,
+        View,
+        CTE,
+        MaterializedView,
+        ExternalTable
+    ]
+
+    DEFAULTS = {
+        'metadata': {
+            'type': 'BaseRelation'
+        },
+        'quote_character': '"',
+        'quote_policy': {
+            'database': True,
+            'schema': True,
+            'identifier': True,
+        },
+        'include_policy': {
+            'database': True,
+            'schema': True,
+            'identifier': True,
+        },
+        'dbt_created': False,
+    }
+
+    PATH_SCHEMA = {
+        'type': 'object',
+        'properties': {
+            'database': {'type': ['string', 'null']},
+            'schema': {'type': ['string', 'null']},
+            'identifier': {'type': ['string', 'null']},
+        },
+        'required': ['database', 'schema', 'identifier'],
+    }
+
+    POLICY_SCHEMA = {
+        'type': 'object',
+        'properties': {
+            'database': {'type': 'boolean'},
+            'schema': {'type': 'boolean'},
+            'identifier': {'type': 'boolean'},
+        },
+        'required': ['database', 'schema', 'identifier'],
+    }
+
+    SCHEMA = {
+        'type': 'object',
+        'properties': {
+            'metadata': {
+                'type': 'object',
+                'properties': {
+                    'type': {
+                        'type': 'string',
+                        'const': 'BaseRelation',
+                    },
+                },
+            },
+            'type': {
+                'enum': RelationTypes + [None],
+            },
+            'path': PATH_SCHEMA,
+            'include_policy': POLICY_SCHEMA,
+            'quote_policy': POLICY_SCHEMA,
+            'quote_character': {'type': 'string'},
+            'dbt_created': {'type': 'boolean'},
+        },
+        'required': ['metadata', 'type', 'path', 'include_policy',
+                     'quote_policy', 'quote_character', 'dbt_created']
+    }
+
+    PATH_ELEMENTS = ['database', 'schema', 'identifier']
+
+    def _is_exactish_match(self, field, value):
+        if self.dbt_created and self.quote_policy.get(field) is False:
+            return self.get_path_part(field).lower() == value.lower()
+        else:
+            return self.get_path_part(field) == value
+
+    def matches(self, database=None, schema=None, identifier=None):
+        search = filter_null_values({
+            'database': database,
+            'schema': schema,
+            'identifier': identifier
+        })
+
+        if not search:
+            # nothing was passed in
+            raise dbt.exceptions.RuntimeException(
+                "Tried to match relation, but no search path was passed!")
+
+        exact_match = True
+        approximate_match = True
+
+        for k, v in search.items():
+            if not self._is_exactish_match(k, v):
+                exact_match = False
+
+            if self.get_path_part(k).lower() != v.lower():
+                approximate_match = False
+
+        if approximate_match and not exact_match:
+            target = self.create(
+                database=database, schema=schema, identifier=identifier
+            )
+            dbt.exceptions.approximate_relation_match(target, self)
+
+        return exact_match
+
+    def get_path_part(self, part):
+        return self.path.get(part)
+
+    def should_quote(self, part):
+        return self.quote_policy.get(part)
+
+    def should_include(self, part):
+        return self.include_policy.get(part)
+
+    def quote(self, database=None, schema=None, identifier=None):
+        policy = filter_null_values({
+            'database': database,
+            'schema': schema,
+            'identifier': identifier
+        })
+
+        return self.incorporate(quote_policy=policy)
+
+    def include(self, database=None, schema=None, identifier=None):
+        policy = filter_null_values({
+            'database': database,
+            'schema': schema,
+            'identifier': identifier
+        })
+
+        return self.incorporate(include_policy=policy)
+
+    def information_schema(self, identifier=None):
+        include_db = self.database is not None
+        include_policy = filter_null_values({
+            'database': include_db,
+            'schema': True,
+            'identifier': identifier is not None
+        })
+        quote_policy = filter_null_values({
+            'database': self.quote_policy['database'],
+            'schema': False,
+            'identifier': False,
+        })
+
+        path_update = {
+            'schema': 'information_schema',
+            'identifier': identifier
+        }
+
+        return self.incorporate(
+            quote_policy=quote_policy,
+            include_policy=include_policy,
+            path=path_update,
+            table_name=identifier)
+
+    def information_schema_only(self):
+        return self.information_schema()
+
+    def information_schema_table(self, identifier):
+        return self.information_schema(identifier)
+
+    def render(self, use_table_name=True):
+        parts = []
+
+        for k in self.PATH_ELEMENTS:
+            if self.should_include(k):
+                path_part = self.get_path_part(k)
+
+                if path_part is None:
+                    continue
+                elif k == 'identifier':
+                    if use_table_name:
+                        path_part = self.table
+                    else:
+                        path_part = self.identifier
+
+                parts.append(
+                    self.quote_if(
+                        path_part,
+                        self.should_quote(k)))
+
+        if len(parts) == 0:
+            raise dbt.exceptions.RuntimeException(
+                "No path parts are included! Nothing to render.")
+
+        return '.'.join(parts)
+
+    def quote_if(self, identifier, should_quote):
+        if should_quote:
+            return self.quoted(identifier)
+
+        return identifier
+
+    def quoted(self, identifier):
+        return '{quote_char}{identifier}{quote_char}'.format(
+            quote_char=self.quote_character,
+            identifier=identifier)
+
+    @classmethod
+    def create_from_source(cls, source, **kwargs):
+        quote_policy = dbt.utils.deep_merge(
+            cls.DEFAULTS['quote_policy'],
+            source.quoting,
+            kwargs.get('quote_policy', {})
+        )
+        return cls.create(
+            database=source.database,
+            schema=source.schema,
+            identifier=source.identifier,
+            quote_policy=quote_policy,
+            **kwargs
+        )
+
+    @classmethod
+    def create_from_node(cls, config, node, table_name=None, quote_policy=None,
+                         **kwargs):
+        if quote_policy is None:
+            quote_policy = {}
+
+        quote_policy = dbt.utils.merge(config.quoting, quote_policy)
+
+        return cls.create(
+            database=node.get('database'),
+            schema=node.get('schema'),
+            identifier=node.get('alias'),
+            table_name=table_name,
+            quote_policy=quote_policy,
+            **kwargs)
+
+    @classmethod
+    def create_from(cls, config, node, **kwargs):
+        if node.resource_type == NodeType.Source:
+            return cls.create_from_source(node, **kwargs)
+        else:
+            return cls.create_from_node(config, node, **kwargs)
+
+    @classmethod
+    def create(cls, database=None, schema=None,
+               identifier=None, table_name=None,
+               type=None, **kwargs):
+        if table_name is None:
+            table_name = identifier
+
+        return cls(type=type,
+                   path={
+                       'database': database,
+                       'schema': schema,
+                       'identifier': identifier
+                   },
+                   table_name=table_name,
+                   **kwargs)
+
+    def __repr__(self):
+        return "<{} {}>".format(self.__class__.__name__, self.render())
+
+    def __hash__(self):
+        return hash(self.render())
+
+    def __str__(self):
+        return self.render()
+
+    @property
+    def path(self):
+        return self.get('path', {})
+
+    @property
+    def database(self):
+        return self.path.get('database')
+
+    @property
+    def schema(self):
+        return self.path.get('schema')
+
+    @property
+    def identifier(self):
+        return self.path.get('identifier')
+
+    # Here for compatibility with old Relation interface
+    @property
+    def name(self):
+        return self.identifier
+
+    # Here for compatibility with old Relation interface
+    @property
+    def table(self):
+        return self.table_name
+
+    @property
+    def is_table(self):
+        return self.type == self.Table
+
+    @property
+    def is_cte(self):
+        return self.type == self.CTE
+
+    @property
+    def is_view(self):
+        return self.type == self.View
+
+
+class Column(object):
+    TYPE_LABELS = {
+        'STRING': 'TEXT',
+        'TIMESTAMP': 'TIMESTAMP',
+        'FLOAT': 'FLOAT',
+        'INTEGER': 'INT'
+    }
+
+    def __init__(self, column, dtype, char_size=None, numeric_precision=None,
+                 numeric_scale=None):
+        self.column = column
+        self.dtype = dtype
+        self.char_size = char_size
+        self.numeric_precision = numeric_precision
+        self.numeric_scale = numeric_scale
+
+    @classmethod
+    def translate_type(cls, dtype):
+        return cls.TYPE_LABELS.get(dtype.upper(), dtype)
+
+    @classmethod
+    def create(cls, name, label_or_dtype):
+        column_type = cls.translate_type(label_or_dtype)
+        return cls(name, column_type)
+
+    @property
+    def name(self):
+        return self.column
+
+    @property
+    def quoted(self):
+        return '"{}"'.format(self.column)
+
+    @property
+    def data_type(self):
+        if self.is_string():
+            return Column.string_type(self.string_size())
+        elif self.is_numeric():
+            return Column.numeric_type(self.dtype, self.numeric_precision,
+                                       self.numeric_scale)
+        else:
+            return self.dtype
+
+    def is_string(self):
+        return self.dtype.lower() in ['text', 'character varying', 'character',
+                                      'varchar']
+
+    def is_numeric(self):
+        return self.dtype.lower() in ['numeric', 'number']
+
+    def string_size(self):
+        if not self.is_string():
+            raise RuntimeError("Called string_size() on non-string field!")
+
+        if self.dtype == 'text' or self.char_size is None:
+            # char_size should never be None. Handle it reasonably just in case
+            return 256
+        else:
+            return int(self.char_size)
+
+    def can_expand_to(self, other_column):
+        """returns True if this column can be expanded to the size of the
+        other column"""
+        if not self.is_string() or not other_column.is_string():
+            return False
+
+        return other_column.string_size() > self.string_size()
+
+    def literal(self, value):
+        return "{}::{}".format(value, self.data_type)
+
+    @classmethod
+    def string_type(cls, size):
+        return "character varying({})".format(size)
+
+    @classmethod
+    def numeric_type(cls, dtype, precision, scale):
+        # This could be decimal(...), numeric(...), number(...)
+        # Just use whatever was fed in here -- don't try to get too clever
+        if precision is None or scale is None:
+            return dtype
+        else:
+            return "{}({},{})".format(dtype, precision, scale)
+
+    def __repr__(self):
+        return "<Column {} ({})>".format(self.name, self.data_type)

core/dbt/adapters/cache.py

@@ -0,0 +1,476 @@
+from collections import namedtuple
+import threading
+from copy import deepcopy
+from dbt.logger import CACHE_LOGGER as logger
+import dbt.exceptions
+
+
+_ReferenceKey = namedtuple('_ReferenceKey', 'database schema identifier')
+
+
+def _lower(value):
+    """Postgres schemas can be None so we can't just call lower()."""
+    if value is None:
+        return None
+    return value.lower()
+
+
+def _make_key(relation):
+    """Make _ReferenceKeys with lowercase values for the cache so we don't have
+    to keep track of quoting
+    """
+    return _ReferenceKey(_lower(relation.database),
+                         _lower(relation.schema),
+                         _lower(relation.identifier))
+
+
+def dot_separated(key):
+    """Return the key in dot-separated string form.
+
+    :param key _ReferenceKey: The key to stringify.
+    """
+    return '.'.join(map(str, key))
+
+
+class _CachedRelation(object):
+    """Nothing about _CachedRelation is guaranteed to be thread-safe!
+
+    :attr str schema: The schema of this relation.
+    :attr str identifier: The identifier of this relation.
+    :attr Dict[_ReferenceKey, _CachedRelation] referenced_by: The relations
+        that refer to this relation.
+    :attr BaseRelation inner: The underlying dbt relation.
+    """
+    def __init__(self, inner):
+        self.referenced_by = {}
+        self.inner = inner
+
+    def __str__(self):
+        return (
+            '_CachedRelation(database={}, schema={}, identifier={}, inner={})'
+        ).format(self.database, self.schema, self.identifier, self.inner)
+
+    @property
+    def database(self):
+        return _lower(self.inner.database)
+
+    @property
+    def schema(self):
+        return _lower(self.inner.schema)
+
+    @property
+    def identifier(self):
+        return _lower(self.inner.identifier)
+
+    def __copy__(self):
+        new = self.__class__(self.inner)
+        new.__dict__.update(self.__dict__)
+        return new
+
+    def __deepcopy__(self, memo):
+        new = self.__class__(self.inner.incorporate())
+        new.__dict__.update(self.__dict__)
+        new.referenced_by = deepcopy(self.referenced_by, memo)
+
+    def is_referenced_by(self, key):
+        return key in self.referenced_by
+
+    def key(self):
+        """Get the _ReferenceKey that represents this relation
+
+        :return _ReferenceKey: A key for this relation.
+        """
+        return _make_key(self)
+
+    def add_reference(self, referrer):
+        """Add a reference from referrer to self, indicating that if this node
+        were drop...cascaded, the referrer would be dropped as well.
+
+        :param _CachedRelation referrer: The node that refers to this node.
+        """
+        self.referenced_by[referrer.key()] = referrer
+
+    def collect_consequences(self):
+        """Recursively collect a set of _ReferenceKeys that would
+        consequentially get dropped if this were dropped via
+        "drop ... cascade".
+
+        :return Set[_ReferenceKey]: All the relations that would be dropped
+        """
+        consequences = {self.key()}
+        for relation in self.referenced_by.values():
+            consequences.update(relation.collect_consequences())
+        return consequences
+
+    def release_references(self, keys):
+        """Non-recursively indicate that an iterable of _ReferenceKey no longer
+        exist. Unknown keys are ignored.
+
+        :param Iterable[_ReferenceKey] keys: The keys to drop.
+        """
+        keys = set(self.referenced_by) & set(keys)
+        for key in keys:
+            self.referenced_by.pop(key)
+
+    def rename(self, new_relation):
+        """Rename this cached relation to new_relation.
+        Note that this will change the output of key(), all refs must be
+        updated!
+
+        :param _CachedRelation new_relation: The new name to apply to the
+            relation
+        """
+        # Relations store this stuff inside their `path` dict. But they
+        # also store a table_name, and usually use it in their  .render(),
+        # so we need to update that as well. It doesn't appear that
+        # table_name is ever anything but the identifier (via .create())
+        self.inner = self.inner.incorporate(
+            path={
+                'database': new_relation.inner.database,
+                'schema': new_relation.inner.schema,
+                'identifier': new_relation.inner.identifier
+            },
+            table_name=new_relation.inner.identifier
+        )
+
+    def rename_key(self, old_key, new_key):
+        """Rename a reference that may or may not exist. Only handles the
+        reference itself, so this is the other half of what `rename` does.
+
+        If old_key is not in referenced_by, this is a no-op.
+
+        :param _ReferenceKey old_key: The old key to be renamed.
+        :param _ReferenceKey new_key: The new key to rename to.
+        :raises InternalError: If the new key already exists.
+        """
+        if new_key in self.referenced_by:
+            dbt.exceptions.raise_cache_inconsistent(
+                'in rename of "{}" -> "{}", new name is in the cache already'
+                .format(old_key, new_key)
+            )
+
+        if old_key not in self.referenced_by:
+            return
+        value = self.referenced_by.pop(old_key)
+        self.referenced_by[new_key] = value
+
+    def dump_graph_entry(self):
+        """Return a key/value pair representing this key and its referents.
+
+        return List[str]: The dot-separated form of all referent keys.
+        """
+        return [dot_separated(r) for r in self.referenced_by]
+
+
+def lazy_log(msg, func):
+    if logger.disabled:
+        return
+    logger.debug(msg.format(func()))
+
+
+class RelationsCache(object):
+    """A cache of the relations known to dbt. Keeps track of relationships
+    declared between tables and handles renames/drops as a real database would.
+
+    :attr Dict[_ReferenceKey, _CachedRelation] relations: The known relations.
+    :attr threading.RLock lock: The lock around relations, held during updates.
+        The adapters also hold this lock while filling the cache.
+    :attr Set[str] schemas: The set of known/cached schemas, all lowercased.
+    """
+    def __init__(self):
+        self.relations = {}
+        self.lock = threading.RLock()
+        self.schemas = set()
+
+    def add_schema(self, database, schema):
+        """Add a schema to the set of known schemas (case-insensitive)
+
+        :param str database: The database name to add.
+        :param str schema: The schema name to add.
+        """
+        self.schemas.add((_lower(database), _lower(schema)))
+
+    def remove_schema(self, database, schema):
+        """Remove a schema from the set of known schemas (case-insensitive)
+
+        If the schema does not exist, it will be ignored - it could just be a
+        temporary table.
+
+        :param str database: The database name to remove.
+        :param str schema: The schema name to remove.
+        """
+        self.schemas.discard((_lower(database), _lower(schema)))
+
+    def update_schemas(self, schemas):
+        """Add multiple schemas to the set of known schemas (case-insensitive)
+
+        :param Iterable[str] schemas: An iterable of the schema names to add.
+        """
+        self.schemas.update((_lower(d), _lower(s)) for (d, s) in schemas)
+
+    def __contains__(self, schema_id):
+        """A schema is 'in' the relations cache if it is in the set of cached
+        schemas.
+
+        :param Tuple[str, str] schema: The db name and schema name to look up.
+        """
+        db, schema = schema_id
+        return (_lower(db), _lower(schema)) in self.schemas
+
+    def dump_graph(self):
+        """Dump a key-only representation of the schema to a dictionary. Every
+        known relation is a key with a value of a list of keys it is referenced
+        by.
+        """
+        # we have to hold the lock for the entire dump, if other threads modify
+        # self.relations or any cache entry's referenced_by during iteration
+        # it's a runtime error!
+        with self.lock:
+            return {
+                dot_separated(k): v.dump_graph_entry()
+                for k, v in self.relations.items()
+            }
+
+    def _setdefault(self, relation):
+        """Add a relation to the cache, or return it if it already exists.
+
+        :param _CachedRelation relation: The relation to set or get.
+        :return _CachedRelation: The relation stored under the given relation's
+            key
+        """
+        self.add_schema(relation.database, relation.schema)
+        key = relation.key()
+        return self.relations.setdefault(key, relation)
+
+    def _add_link(self, referenced_key, dependent_key):
+        """Add a link between two relations to the database. Both the old and
+        new entries must alraedy exist in the database.
+
+        :param _ReferenceKey referenced_key: The key identifying the referenced
+            model (the one that if dropped will drop the dependent model).
+        :param _ReferenceKey dependent_key: The key identifying the dependent
+            model.
+        :raises InternalError: If either entry does not exist.
+        """
+        referenced = self.relations.get(referenced_key)
+        if referenced is None:
+            dbt.exceptions.raise_cache_inconsistent(
+                'in add_link, referenced link key {} not in cache!'
+                .format(referenced_key)
+            )
+
+        dependent = self.relations.get(dependent_key)
+        if dependent is None:
+            dbt.exceptions.raise_cache_inconsistent(
+                'in add_link, dependent link key {} not in cache!'
+                .format(dependent_key)
+            )
+
+        referenced.add_reference(dependent)
+
+    def add_link(self, referenced, dependent):
+        """Add a link between two relations to the database. Both the old and
+        new entries must already exist in the database.
+
+        The dependent model refers _to_ the referenced model. So, given
+        arguments of (jake_test, bar, jake_test, foo):
+        both values are in the schema jake_test and foo is a view that refers
+        to bar, so "drop bar cascade" will drop foo and all of foo's
+        dependents.
+
+        :param BaseRelation referenced: The referenced model.
+        :param BaseRelation dependent: The dependent model.
+        :raises InternalError: If either entry does not exist.
+        """
+        referenced = _make_key(referenced)
+        if (referenced.database, referenced.schema) not in self:
+            # if we have not cached the referenced schema at all, we must be
+            # referring to a table outside our control. There's no need to make
+            # a link - we will never drop the referenced relation during a run.
+            logger.debug(
+                '{dep!s} references {ref!s} but {ref.database}.{ref.schema} '
+                'is not in the cache, skipping assumed external relation'
+                .format(dep=dependent, ref=referenced)
+            )
+            return
+        dependent = _make_key(dependent)
+        logger.debug(
+            'adding link, {!s} references {!s}'.format(dependent, referenced)
+        )
+        with self.lock:
+            self._add_link(referenced, dependent)
+
+    def add(self, relation):
+        """Add the relation inner to the cache, under the schema schema and
+        identifier identifier
+
+        :param BaseRelation relation: The underlying relation.
+        """
+        cached = _CachedRelation(relation)
+        logger.debug('Adding relation: {!s}'.format(cached))
+
+        lazy_log('before adding: {!s}', self.dump_graph)
+
+        with self.lock:
+            self._setdefault(cached)
+
+        lazy_log('after adding: {!s}', self.dump_graph)
+
+    def _remove_refs(self, keys):
+        """Removes all references to all entries in keys. This does not
+        cascade!
+
+        :param Iterable[_ReferenceKey] keys: The keys to remove.
+        """
+        # remove direct refs
+        for key in keys:
+            del self.relations[key]
+        # then remove all entries from each child
+        for cached in self.relations.values():
+            cached.release_references(keys)
+
+    def _drop_cascade_relation(self, dropped):
+        """Drop the given relation and cascade it appropriately to all
+        dependent relations.
+
+        :param _CachedRelation dropped: An existing _CachedRelation to drop.
+        """
+        if dropped not in self.relations:
+            logger.debug('dropped a nonexistent relationship: {!s}'
+                         .format(dropped))
+            return
+        consequences = self.relations[dropped].collect_consequences()
+        logger.debug(
+            'drop {} is cascading to {}'.format(dropped, consequences)
+        )
+        self._remove_refs(consequences)
+
+    def drop(self, relation):
+        """Drop the named relation and cascade it appropriately to all
+        dependent relations.
+
+        Because dbt proactively does many `drop relation if exist ... cascade`
+        that are noops, nonexistent relation drops cause a debug log and no
+        other actions.
+
+        :param str schema: The schema of the relation to drop.
+        :param str identifier: The identifier of the relation to drop.
+        """
+        dropped = _make_key(relation)
+        logger.debug('Dropping relation: {!s}'.format(dropped))
+        with self.lock:
+            self._drop_cascade_relation(dropped)
+
+    def _rename_relation(self, old_key, new_relation):
+        """Rename a relation named old_key to new_key, updating references.
+        Return whether or not there was a key to rename.
+
+        :param _ReferenceKey old_key: The existing key, to rename from.
+        :param _CachedRelation new_key: The new relation, to rename to.
+        """
+        # On the database level, a rename updates all values that were
+        # previously referenced by old_name to be referenced by new_name.
+        # basically, the name changes but some underlying ID moves. Kind of
+        # like an object reference!
+        relation = self.relations.pop(old_key)
+        new_key = new_relation.key()
+
+        # relaton has to rename its innards, so it needs the _CachedRelation.
+        relation.rename(new_relation)
+        # update all the relations that refer to it
+        for cached in self.relations.values():
+            if cached.is_referenced_by(old_key):
+                logger.debug(
+                    'updated reference from {0} -> {2} to {1} -> {2}'
+                    .format(old_key, new_key, cached.key())
+                )
+                cached.rename_key(old_key, new_key)
+
+        self.relations[new_key] = relation
+        # also fixup the schemas!
+        self.remove_schema(old_key.database, old_key.schema)
+        self.add_schema(new_key.database, new_key.schema)
+
+        return True
+
+    def _check_rename_constraints(self, old_key, new_key):
+        """Check the rename constraints, and return whether or not the rename
+        can proceed.
+
+        If the new key is already present, that is an error.
+        If the old key is absent, we debug log and return False, assuming it's
+        a temp table being renamed.
+
+        :param _ReferenceKey old_key: The existing key, to rename from.
+        :param _ReferenceKey new_key: The new key, to rename to.
+        :return bool: If the old relation exists for renaming.
+        :raises InternalError: If the new key is already present.
+        """
+        if new_key in self.relations:
+            dbt.exceptions.raise_cache_inconsistent(
+                'in rename, new key {} already in cache: {}'
+                .format(new_key, list(self.relations.keys()))
+            )
+
+        if old_key not in self.relations:
+            logger.debug(
+                'old key {} not found in self.relations, assuming temporary'
+                .format(old_key)
+            )
+            return False
+        return True
+
+    def rename(self, old, new):
+        """Rename the old schema/identifier to the new schema/identifier and
+        update references.
+
+        If the new schema/identifier is already present, that is an error.
+        If the schema/identifier key is absent, we only debug log and return,
+        assuming it's a temp table being renamed.
+
+        :param BaseRelation old: The existing relation name information.
+        :param BaseRelation new: The new relation name information.
+        :raises InternalError: If the new key is already present.
+        """
+        old_key = _make_key(old)
+        new_key = _make_key(new)
+        logger.debug('Renaming relation {!s} to {!s}'.format(
+            old_key, new_key
+        ))
+
+        lazy_log('before rename: {!s}', self.dump_graph)
+
+        with self.lock:
+            if self._check_rename_constraints(old_key, new_key):
+                self._rename_relation(old_key, _CachedRelation(new))
+            else:
+                self._setdefault(_CachedRelation(new))
+
+        lazy_log('after rename: {!s}', self.dump_graph)
+
+    def get_relations(self, database, schema):
+        """Case-insensitively yield all relations matching the given schema.
+
+        :param str schema: The case-insensitive schema name to list from.
+        :return List[BaseRelation]: The list of relations with the given
+            schema
+        """
+        schema = _lower(schema)
+        with self.lock:
+            results = [
+                r.inner for r in self.relations.values()
+                if (r.schema == _lower(schema) and
+                    r.database == _lower(database))
+            ]
+
+        if None in results:
+            dbt.exceptions.raise_cache_inconsistent(
+                'in get_relations, a None relation was found in the cache!'
+            )
+        return results
+
+    def clear(self):
+        """Clear the cache"""
+        with self.lock:
+            self.relations.clear()
+            self.schemas.clear()

core/dbt/adapters/factory.py

@@ -0,0 +1,86 @@
+import dbt.exceptions
+from importlib import import_module
+from dbt.include.global_project import PACKAGES
+from dbt.logger import GLOBAL_LOGGER as logger
+
+import threading
+
+ADAPTER_TYPES = {}
+
+_ADAPTERS = {}
+_ADAPTER_LOCK = threading.Lock()
+
+
+def get_adapter_class_by_name(adapter_name):
+    with _ADAPTER_LOCK:
+        if adapter_name in ADAPTER_TYPES:
+            return ADAPTER_TYPES[adapter_name]
+
+    message = "Invalid adapter type {}! Must be one of {}"
+    adapter_names = ", ".join(ADAPTER_TYPES.keys())
+    formatted_message = message.format(adapter_name, adapter_names)
+    raise dbt.exceptions.RuntimeException(formatted_message)
+
+
+def get_relation_class_by_name(adapter_name):
+    adapter = get_adapter_class_by_name(adapter_name)
+    return adapter.Relation
+
+
+def load_plugin(adapter_name):
+    try:
+        mod = import_module('.' + adapter_name, 'dbt.adapters')
+    except ImportError as e:
+        logger.info("Error importing adapter: {}".format(e))
+        raise dbt.exceptions.RuntimeException(
+            "Could not find adapter type {}!".format(adapter_name)
+        )
+    plugin = mod.Plugin
+
+    if plugin.adapter.type() != adapter_name:
+        raise dbt.exceptions.RuntimeException(
+            'Expected to find adapter with type named {}, got adapter with '
+            'type {}'
+            .format(adapter_name, plugin.adapter.type())
+        )
+
+    with _ADAPTER_LOCK:
+        ADAPTER_TYPES[adapter_name] = plugin.adapter
+
+    PACKAGES[plugin.project_name] = plugin.include_path
+
+    for dep in plugin.dependencies:
+        load_plugin(dep)
+
+    return plugin.credentials
+
+
+def get_adapter(config):
+    adapter_name = config.credentials.type
+    if adapter_name in _ADAPTERS:
+        return _ADAPTERS[adapter_name]
+
+    with _ADAPTER_LOCK:
+        if adapter_name not in ADAPTER_TYPES:
+            raise dbt.exceptions.RuntimeException(
+                "Could not find adapter type {}!".format(adapter_name)
+            )
+
+        adapter_type = ADAPTER_TYPES[adapter_name]
+
+        # check again, in case something was setting it before
+        if adapter_name in _ADAPTERS:
+            return _ADAPTERS[adapter_name]
+
+        adapter = adapter_type(config)
+        _ADAPTERS[adapter_name] = adapter
+        return adapter
+
+
+def reset_adapters():
+    """Clear the adapters. This is useful for tests, which change configs.
+    """
+    with _ADAPTER_LOCK:
+        for adapter in _ADAPTERS.values():
+            adapter.cleanup_connections()
+        _ADAPTERS.clear()

core/dbt/adapters/sql/__init__.py

@@ -0,0 +1,3 @@
+# these are all just exports, #noqa them so flake8 will be happy
+from dbt.adapters.sql.connections import SQLConnectionManager  # noqa
+from dbt.adapters.sql.impl import SQLAdapter  # noqa

core/dbt/adapters/sql/connections.py

@@ -0,0 +1,141 @@
+import abc
+import time
+
+import dbt.clients.agate_helper
+import dbt.exceptions
+from dbt.contracts.connection import Connection
+from dbt.adapters.base import BaseConnectionManager
+from dbt.compat import abstractclassmethod
+from dbt.logger import GLOBAL_LOGGER as logger
+
+
+class SQLConnectionManager(BaseConnectionManager):
+    """The default connection manager with some common SQL methods implemented.
+
+    Methods to implement:
+        - exception_handler
+        - cancel
+        - get_status
+        - open
+    """
+    @abc.abstractmethod
+    def cancel(self, connection):
+        """Cancel the given connection.
+
+        :param Connection connection: The connection to cancel.
+        """
+        raise dbt.exceptions.NotImplementedException(
+            '`cancel` is not implemented for this adapter!'
+        )
+
+    def cancel_open(self):
+        names = []
+        this_connection = self.get_if_exists()
+        with self.lock:
+            for connection in self.thread_connections.values():
+                if connection is this_connection:
+                    continue
+
+                self.cancel(connection)
+                names.append(connection.name)
+        return names
+
+    def add_query(self, sql, auto_begin=True, bindings=None,
+                  abridge_sql_log=False):
+        connection = self.get_thread_connection()
+        if auto_begin and connection.transaction_open is False:
+            self.begin()
+
+        logger.debug('Using {} connection "{}".'
+                     .format(self.TYPE, connection.name))
+
+        with self.exception_handler(sql):
+            if abridge_sql_log:
+                logger.debug('On %s: %s....', connection.name, sql[0:512])
+            else:
+                logger.debug('On %s: %s', connection.name, sql)
+            pre = time.time()
+
+            cursor = connection.handle.cursor()
+            cursor.execute(sql, bindings)
+
+            logger.debug("SQL status: %s in %0.2f seconds",
+                         self.get_status(cursor), (time.time() - pre))
+
+            return connection, cursor
+
+    @abstractclassmethod
+    def get_status(cls, cursor):
+        """Get the status of the cursor.
+
+        :param cursor: A database handle to get status from
+        :return: The current status
+        :rtype: str
+        """
+        raise dbt.exceptions.NotImplementedException(
+            '`get_status` is not implemented for this adapter!'
+        )
+
+    @classmethod
+    def process_results(cls, column_names, rows):
+        return [dict(zip(column_names, row)) for row in rows]
+
+    @classmethod
+    def get_result_from_cursor(cls, cursor):
+        data = []
+        column_names = []
+
+        if cursor.description is not None:
+            column_names = [col[0] for col in cursor.description]
+            rows = cursor.fetchall()
+            data = cls.process_results(column_names, rows)
+
+        return dbt.clients.agate_helper.table_from_data(data, column_names)
+
+    def execute(self, sql, auto_begin=False, fetch=False):
+        _, cursor = self.add_query(sql, auto_begin)
+        status = self.get_status(cursor)
+        if fetch:
+            table = self.get_result_from_cursor(cursor)
+        else:
+            table = dbt.clients.agate_helper.empty_table()
+        return status, table
+
+    def add_begin_query(self):
+        return self.add_query('BEGIN', auto_begin=False)
+
+    def add_commit_query(self):
+        return self.add_query('COMMIT', auto_begin=False)
+
+    def begin(self):
+        connection = self.get_thread_connection()
+
+        if dbt.flags.STRICT_MODE:
+            assert isinstance(connection, Connection)
+
+        if connection.transaction_open is True:
+            raise dbt.exceptions.InternalException(
+                'Tried to begin a new transaction on connection "{}", but '
+                'it already had one open!'.format(connection.get('name')))
+
+        self.add_begin_query()
+
+        connection.transaction_open = True
+        return connection
+
+    def commit(self):
+        connection = self.get_thread_connection()
+        if dbt.flags.STRICT_MODE:
+            assert isinstance(connection, Connection)
+
+        if connection.transaction_open is False:
+            raise dbt.exceptions.InternalException(
+                'Tried to commit transaction on connection "{}", but '
+                'it does not have one open!'.format(connection.name))
+
+        logger.debug('On {}: COMMIT'.format(connection.name))
+        self.add_commit_query()
+
+        connection.transaction_open = False
+
+        return connection

core/dbt/adapters/sql/impl.py

@@ -0,0 +1,222 @@
+import agate
+
+import dbt.clients.agate_helper
+import dbt.exceptions
+import dbt.flags
+from dbt.adapters.base import BaseAdapter, available
+from dbt.logger import GLOBAL_LOGGER as logger
+
+
+LIST_RELATIONS_MACRO_NAME = 'list_relations_without_caching'
+GET_COLUMNS_IN_RELATION_MACRO_NAME = 'get_columns_in_relation'
+LIST_SCHEMAS_MACRO_NAME = 'list_schemas'
+CHECK_SCHEMA_EXISTS_MACRO_NAME = 'check_schema_exists'
+CREATE_SCHEMA_MACRO_NAME = 'create_schema'
+DROP_SCHEMA_MACRO_NAME = 'drop_schema'
+RENAME_RELATION_MACRO_NAME = 'rename_relation'
+TRUNCATE_RELATION_MACRO_NAME = 'truncate_relation'
+DROP_RELATION_MACRO_NAME = 'drop_relation'
+ALTER_COLUMN_TYPE_MACRO_NAME = 'alter_column_type'
+
+
+class SQLAdapter(BaseAdapter):
+    """The default adapter with the common agate conversions and some SQL
+    methods implemented. This adapter has a different much shorter list of
+    methods to implement, but some more macros that must be implemented.
+
+    To implement a macro, implement "${adapter_type}__${macro_name}". in the
+    adapter's internal project.
+
+    Methods to implement:
+        - date_function
+
+    Macros to implement:
+        - get_catalog
+        - list_relations_without_caching
+        - get_columns_in_relation
+    """
+    @available.parse(lambda *a, **k: (None, None))
+    def add_query(self, sql, auto_begin=True, bindings=None,
+                  abridge_sql_log=False):
+        """Add a query to the current transaction. A thin wrapper around
+        ConnectionManager.add_query.
+
+        :param str sql: The SQL query to add
+        :param bool auto_begin: If set and there is no transaction in progress,
+            begin a new one.
+        :param Optional[List[object]]: An optional list of bindings for the
+            query.
+        :param bool abridge_sql_log: If set, limit the raw sql logged to 512
+            characters
+        """
+        return self.connections.add_query(sql, auto_begin, bindings,
+                                          abridge_sql_log)
+
+    @classmethod
+    def convert_text_type(cls, agate_table, col_idx):
+        return "text"
+
+    @classmethod
+    def convert_number_type(cls, agate_table, col_idx):
+        decimals = agate_table.aggregate(agate.MaxPrecision(col_idx))
+        return "float8" if decimals else "integer"
+
+    @classmethod
+    def convert_boolean_type(cls, agate_table, col_idx):
+        return "boolean"
+
+    @classmethod
+    def convert_datetime_type(cls, agate_table, col_idx):
+        return "timestamp without time zone"
+
+    @classmethod
+    def convert_date_type(cls, agate_table, col_idx):
+        return "date"
+
+    @classmethod
+    def convert_time_type(cls, agate_table, col_idx):
+        return "time"
+
+    @classmethod
+    def is_cancelable(cls):
+        return True
+
+    def expand_column_types(self, goal, current):
+        reference_columns = {
+            c.name: c for c in
+            self.get_columns_in_relation(goal)
+        }
+
+        target_columns = {
+            c.name: c for c
+            in self.get_columns_in_relation(current)
+        }
+
+        for column_name, reference_column in reference_columns.items():
+            target_column = target_columns.get(column_name)
+
+            if target_column is not None and \
+               target_column.can_expand_to(reference_column):
+                col_string_size = reference_column.string_size()
+                new_type = self.Column.string_type(col_string_size)
+                logger.debug("Changing col type from %s to %s in table %s",
+                             target_column.data_type, new_type, current)
+
+                self.alter_column_type(current, column_name, new_type)
+
+    def alter_column_type(self, relation, column_name, new_column_type):
+        """
+        1. Create a new column (w/ temp name and correct type)
+        2. Copy data over to it
+        3. Drop the existing column (cascade!)
+        4. Rename the new column to existing column
+        """
+        kwargs = {
+            'relation': relation,
+            'column_name': column_name,
+            'new_column_type': new_column_type,
+        }
+        self.execute_macro(
+            ALTER_COLUMN_TYPE_MACRO_NAME,
+            kwargs=kwargs
+        )
+
+    def drop_relation(self, relation):
+        if dbt.flags.USE_CACHE:
+            self.cache.drop(relation)
+        if relation.type is None:
+            dbt.exceptions.raise_compiler_error(
+                'Tried to drop relation {}, but its type is null.'
+                .format(relation))
+
+        self.execute_macro(
+            DROP_RELATION_MACRO_NAME,
+            kwargs={'relation': relation}
+        )
+
+    def truncate_relation(self, relation):
+        self.execute_macro(
+            TRUNCATE_RELATION_MACRO_NAME,
+            kwargs={'relation': relation}
+        )
+
+    def rename_relation(self, from_relation, to_relation):
+        if dbt.flags.USE_CACHE:
+            self.cache.rename(from_relation, to_relation)
+
+        kwargs = {'from_relation': from_relation, 'to_relation': to_relation}
+        self.execute_macro(
+            RENAME_RELATION_MACRO_NAME,
+            kwargs=kwargs
+        )
+
+    def get_columns_in_relation(self, relation):
+        return self.execute_macro(
+            GET_COLUMNS_IN_RELATION_MACRO_NAME,
+            kwargs={'relation': relation}
+        )
+
+    def create_schema(self, database, schema):
+        logger.debug('Creating schema "%s"."%s".', database, schema)
+        kwargs = {
+            'database_name': self.quote_as_configured(database, 'database'),
+            'schema_name': self.quote_as_configured(schema, 'schema'),
+        }
+        self.execute_macro(CREATE_SCHEMA_MACRO_NAME, kwargs=kwargs)
+        self.commit_if_has_connection()
+
+    def drop_schema(self, database, schema):
+        logger.debug('Dropping schema "%s"."%s".', database, schema)
+        kwargs = {
+            'database_name': self.quote_as_configured(database, 'database'),
+            'schema_name': self.quote_as_configured(schema, 'schema'),
+        }
+        self.execute_macro(DROP_SCHEMA_MACRO_NAME,
+                           kwargs=kwargs)
+
+    def list_relations_without_caching(self, information_schema, schema):
+        kwargs = {'information_schema': information_schema, 'schema': schema}
+        results = self.execute_macro(
+            LIST_RELATIONS_MACRO_NAME,
+            kwargs=kwargs
+        )
+
+        relations = []
+        quote_policy = {
+            'database': True,
+            'schema': True,
+            'identifier': True
+        }
+        for _database, name, _schema, _type in results:
+            relations.append(self.Relation.create(
+                database=_database,
+                schema=_schema,
+                identifier=name,
+                quote_policy=quote_policy,
+                type=_type
+            ))
+        return relations
+
+    def quote(cls, identifier):
+        return '"{}"'.format(identifier)
+
+    def list_schemas(self, database):
+        results = self.execute_macro(
+            LIST_SCHEMAS_MACRO_NAME,
+            kwargs={'database': database}
+        )
+
+        return [row[0] for row in results]
+
+    def check_schema_exists(self, database, schema):
+        information_schema = self.Relation.create(
+            database=database, schema=schema,
+            quote_policy=self.config.quoting
+        ).information_schema()
+
+        kwargs = {'information_schema': information_schema, 'schema': schema}
+        results = self.execute_macro(
+            CHECK_SCHEMA_EXISTS_MACRO_NAME,
+            kwargs=kwargs
+        )
+        return results[0][0] > 0

core/dbt/api/__init__.py

@@ -0,0 +1,5 @@
+from dbt.api.object import APIObject
+
+__all__ = [
+    'APIObject'
+]

core/dbt/api/object.py

@@ -0,0 +1,125 @@
+import copy
+from collections import Mapping
+from jsonschema import Draft7Validator
+
+from dbt.exceptions import JSONValidationException
+from dbt.utils import deep_merge
+from dbt.clients.system import write_json
+
+
+class APIObject(Mapping):
+    """
+    A serializable / deserializable object intended for
+    use in a future dbt API.
+
+    To create a new object, you'll want to extend this
+    class, and then implement the SCHEMA property (a
+    valid JSON schema), the DEFAULTS property (default
+    settings for this object), and a static method that
+    calls this constructor.
+    """
+
+    SCHEMA = {
+        'type': 'object',
+        'properties': {}
+    }
+
+    DEFAULTS = {}
+
+    def __init__(self, **kwargs):
+        """
+        Create and validate an instance. Note that if you override this, you
+        will want to do so by modifying kwargs and only then calling
+        super(NewClass, self).__init__(**kwargs).
+        """
+        super(APIObject, self).__init__()
+        # note: deep_merge does a deep copy on its arguments.
+        self._contents = deep_merge(self.DEFAULTS, kwargs)
+        self.validate()
+
+    def __str__(self):
+        return '{}(**{})'.format(self.__class__.__name__, self._contents)
+
+    def __repr__(self):
+        return '{}(**{})'.format(self.__class__.__name__, self._contents)
+
+    def __eq__(self, other):
+        if not isinstance(other, self.__class__):
+            return False
+        return self.serialize() == other.serialize()
+
+    def incorporate(self, **kwargs):
+        """
+        Given a list of kwargs, incorporate these arguments
+        into a new copy of this instance, and return the new
+        instance after validating.
+        """
+        return type(self)(**deep_merge(self._contents, kwargs))
+
+    def serialize(self):
+        """
+        Return a dict representation of this object.
+        """
+        return copy.deepcopy(self._contents)
+
+    def write(self, path):
+        write_json(path, self.serialize())
+
+    @classmethod
+    def deserialize(cls, settings):
+        """
+        Convert a dict representation of this object into
+        an actual object for internal use.
+        """
+        return cls(**settings)
+
+    def validate(self):
+        """
+        Using the SCHEMA property, validate the attributes
+        of this instance. If any attributes are missing or
+        invalid, raise a ValidationException.
+        """
+        validator = Draft7Validator(self.SCHEMA)
+
+        errors = set()  # make errors a set to avoid duplicates
+
+        for error in validator.iter_errors(self.serialize()):
+            errors.add('.'.join(
+                list(map(str, error.path)) + [error.message]
+            ))
+
+        if errors:
+            raise JSONValidationException(type(self).__name__, errors)
+
+    # implement the Mapping protocol:
+    # https://docs.python.org/3/library/collections.abc.html
+    def __getitem__(self, key):
+        return self._contents[key]
+
+    def __iter__(self):
+        return self._contents.__iter__()
+
+    def __len__(self):
+        return self._contents.__len__()
+
+    # implement this because everyone always expects it.
+    def get(self, key, default=None):
+        try:
+            return self[key]
+        except KeyError:
+            return default
+
+    def set(self, key, value):
+        self._contents[key] = value
+
+    # most users of APIObject also expect the attributes to be available via
+    # dot-notation because the previous implementation assigned to __dict__.
+    # we should consider removing this if we fix all uses to have properties.
+    def __getattr__(self, name):
+        if name != '_contents' and name in self._contents:
+            return self._contents[name]
+        elif hasattr(self.__class__, name):
+            return getattr(self.__class__, name)
+        raise AttributeError((
+            "'{}' object has no attribute '{}'"
+        ).format(type(self).__name__, name))

core/dbt/clients/_jinja_blocks.py

@@ -0,0 +1,371 @@
+import re
+from collections import namedtuple
+
+import dbt.exceptions
+
+
+def regex(pat):
+    return re.compile(pat, re.DOTALL | re.MULTILINE)
+
+
+class BlockData(object):
+    """raw plaintext data from the top level of the file."""
+    def __init__(self, contents):
+        self.block_type_name = '__dbt__data'
+        self.contents = contents
+        self.full_block = contents
+
+
+class BlockTag(object):
+    def __init__(self, block_type_name, block_name, contents=None,
+                 full_block=None, **kw):
+        self.block_type_name = block_type_name
+        self.block_name = block_name
+        self.contents = contents
+        self.full_block = full_block
+
+    def __str__(self):
+        return 'BlockTag({!r}, {!r})'.format(self.block_type_name,
+                                             self.block_name)
+
+    def __repr__(self):
+        return str(self)
+
+    @property
+    def end_block_type_name(self):
+        return 'end{}'.format(self.block_type_name)
+
+    def end_pat(self):
+        # we don't want to use string formatting here because jinja uses most
+        # of the string formatting operators in its syntax...
+        pattern = ''.join((
+            r'(?P<endblock>((?:\s*\{\%\-|\{\%)\s*',
+            self.end_block_type_name,
+            r'\s*(?:\-\%\}\s*|\%\})))',
+        ))
+        return regex(pattern)
+
+
+Tag = namedtuple('Tag', 'block_type_name block_name start end')
+
+
+_NAME_PATTERN = r'[A-Za-z_][A-Za-z_0-9]*'
+
+COMMENT_START_PATTERN = regex(r'(?:(?P<comment_start>(\s*\{\#)))')
+COMMENT_END_PATTERN = regex(r'(.*?)(\s*\#\})')
+RAW_START_PATTERN = regex(
+    r'(?:\s*\{\%\-|\{\%)\s*(?P<raw_start>(raw))\s*(?:\-\%\}\s*|\%\})'
+)
+EXPR_START_PATTERN = regex(r'(?P<expr_start>(\{\{\s*))')
+EXPR_END_PATTERN = regex(r'(?P<expr_end>(\s*\}\}))')
+
+BLOCK_START_PATTERN = regex(''.join((
+    r'(?:\s*\{\%\-|\{\%)\s*',
+    r'(?P<block_type_name>({}))'.format(_NAME_PATTERN),
+    # some blocks have a 'block name'.
+    r'(?:\s+(?P<block_name>({})))?'.format(_NAME_PATTERN),
+)))
+
+
+RAW_BLOCK_PATTERN = regex(''.join((
+    r'(?:\s*\{\%\-|\{\%)\s*raw\s*(?:\-\%\}\s*|\%\})',
+    r'(?:.*)',
+    r'(?:\s*\{\%\-|\{\%)\s*endraw\s*(?:\-\%\}\s*|\%\})',
+)))
+
+TAG_CLOSE_PATTERN = regex(r'(?:(?P<tag_close>(\-\%\}\s*|\%\})))')
+
+# stolen from jinja's lexer. Note that we've consumed all prefix whitespace by
+# the time we want to use this.
+STRING_PATTERN = regex(
+    r"(?P<string>('([^'\\]*(?:\\.[^'\\]*)*)'|"
+    r'"([^"\\]*(?:\\.[^"\\]*)*)"))'
+)
+
+QUOTE_START_PATTERN = regex(r'''(?P<quote>(['"]))''')
+
+
+class TagIterator(object):
+    def __init__(self, data):
+        self.data = data
+        self.blocks = []
+        self._parenthesis_stack = []
+        self.pos = 0
+
+    def advance(self, new_position):
+        self.pos = new_position
+
+    def rewind(self, amount=1):
+        self.pos -= amount
+
+    def _search(self, pattern):
+        return pattern.search(self.data, self.pos)
+
+    def _match(self, pattern):
+        return pattern.match(self.data, self.pos)
+
+    def _first_match(self, *patterns, **kwargs):
+        matches = []
+        for pattern in patterns:
+            # default to 'search', but sometimes we want to 'match'.
+            if kwargs.get('method', 'search') == 'search':
+                match = self._search(pattern)
+            else:
+                match = self._match(pattern)
+            if match:
+                matches.append(match)
+        if not matches:
+            return None
+        # if there are multiple matches, pick the least greedy match
+        # TODO: do I need to account for m.start(), or is this ok?
+        return min(matches, key=lambda m: m.end())
+
+    def _expect_match(self, expected_name, *patterns, **kwargs):
+        match = self._first_match(*patterns, **kwargs)
+        if match is None:
+            msg = 'unexpected EOF, expected {}, got "{}"'.format(
+                expected_name, self.data[self.pos:]
+            )
+            dbt.exceptions.raise_compiler_error(msg)
+        return match
+
+    def handle_expr(self, match):
+        """Handle an expression. At this point we're at a string like:
+            {{ 1 + 2 }}
+            ^ right here
+
+        And the match contains "{{ "
+
+        We expect to find a `}}`, but we might find one in a string before
+        that. Imagine the case of `{{ 2 * "}}" }}`...
+
+        You're not allowed to have blocks or comments inside an expr so it is
+        pretty straightforward, I hope: only strings can get in the way.
+        """
+        self.advance(match.end())
+        while True:
+            match = self._expect_match('}}',
+                                       EXPR_END_PATTERN,
+                                       QUOTE_START_PATTERN)
+            if match.groupdict().get('expr_end') is not None:
+                break
+            else:
+                # it's a quote. we haven't advanced for this match yet, so
+                # just slurp up the whole string, no need to rewind.
+                match = self._expect_match('string', STRING_PATTERN)
+                self.advance(match.end())
+
+        self.advance(match.end())
+
+    def handle_comment(self, match):
+        self.advance(match.end())
+        match = self._expect_match('#}', COMMENT_END_PATTERN)
+        self.advance(match.end())
+
+    def _expect_block_close(self):
+        """Search for the tag close marker.
+        To the right of the type name, there are a few possiblities:
+           - a name (handled by the regex's 'block_name')
+           - any number of: `=`, `(`, `)`, strings, etc (arguments)
+           - nothing
+
+        followed eventually by a %}
+
+        So the only characters we actually have to worry about in this context
+        are quote and `%}` - nothing else can hide the %} and be valid jinja.
+        """
+        while True:
+            end_match = self._expect_match(
+                'tag close ("%}")',
+                QUOTE_START_PATTERN,
+                TAG_CLOSE_PATTERN
+            )
+            self.advance(end_match.end())
+            if end_match.groupdict().get('tag_close') is not None:
+                return
+            # must be a string. Rewind to its start and advance past it.
+            self.rewind()
+            string_match = self._expect_match('string', STRING_PATTERN)
+            self.advance(string_match.end())
+
+    def handle_raw(self):
+        # raw blocks are super special, they are a single complete regex
+        match = self._expect_match('{% raw %}...{% endraw %}',
+                                   RAW_BLOCK_PATTERN)
+        self.advance(match.end())
+        return match.end()
+
+    def handle_tag(self, match):
+        """The tag could be one of a few things:
+
+            {% mytag %}
+            {% mytag x = y %}
+            {% mytag x = "y" %}
+            {% mytag x.y() %}
+            {% mytag foo("a", "b", c="d") %}
+
+        But the key here is that it's always going to be `{% mytag`!
+        """
+        groups = match.groupdict()
+        # always a value
+        block_type_name = groups['block_type_name']
+        # might be None
+        block_name = groups.get('block_name')
+        start_pos = self.pos
+        if block_type_name == 'raw':
+            match = self._expect_match('{% raw %}...{% endraw %}',
+                                       RAW_BLOCK_PATTERN)
+            self.advance(match.end())
+        else:
+            self.advance(match.end())
+            self._expect_block_close()
+        return Tag(
+            block_type_name=block_type_name,
+            block_name=block_name,
+            start=start_pos,
+            end=self.pos
+        )
+
+    def find_tags(self):
+        while True:
+            match = self._first_match(
+                BLOCK_START_PATTERN,
+                COMMENT_START_PATTERN,
+                EXPR_START_PATTERN
+            )
+            if match is None:
+                break
+
+            self.advance(match.start())
+            # start = self.pos
+
+            groups = match.groupdict()
+            comment_start = groups.get('comment_start')
+            expr_start = groups.get('expr_start')
+            block_type_name = groups.get('block_type_name')
+
+            if comment_start is not None:
+                self.handle_comment(match)
+            elif expr_start is not None:
+                self.handle_expr(match)
+            elif block_type_name is not None:
+                yield self.handle_tag(match)
+            else:
+                raise dbt.exceptions.InternalException(
+                    'Invalid regex match in next_block, expected block start, '
+                    'expr start, or comment start'
+                )
+
+    def __iter__(self):
+        return self.find_tags()
+
+
+duplicate_tags = (
+    'Got nested tags: {outer.block_type_name} (started at {outer.start}) did '
+    'not have a matching {{% end{outer.block_type_name} %}} before a '
+    'subsequent {inner.block_type_name} was found (started at {inner.start})'
+)
+
+
+_CONTROL_FLOW_TAGS = {
+    'if': 'endif',
+    'for': 'endfor',
+}
+
+_CONTROL_FLOW_END_TAGS = {
+    v: k
+    for k, v in _CONTROL_FLOW_TAGS.items()
+}
+
+
+class BlockIterator(object):
+    def __init__(self, data):
+        self.tag_parser = TagIterator(data)
+        self.current = None
+        self.stack = []
+        self.last_position = 0
+
+    @property
+    def current_end(self):
+        if self.current is None:
+            return 0
+        else:
+            return self.current.end
+
+    @property
+    def data(self):
+        return self.tag_parser.data
+
+    def is_current_end(self, tag):
+        return (
+            tag.block_type_name.startswith('end') and
+            self.current is not None and
+            tag.block_type_name[3:] == self.current.block_type_name
+        )
+
+    def find_blocks(self, allowed_blocks=None, collect_raw_data=True):
+        """Find all top-level blocks in the data."""
+        if allowed_blocks is None:
+            allowed_blocks = {'snapshot', 'macro', 'materialization', 'docs'}
+
+        for tag in self.tag_parser.find_tags():
+            if tag.block_type_name in _CONTROL_FLOW_TAGS:
+                self.stack.append(tag.block_type_name)
+            elif tag.block_type_name in _CONTROL_FLOW_END_TAGS:
+                found = None
+                if self.stack:
+                    found = self.stack.pop()
+                else:
+                    expected = _CONTROL_FLOW_END_TAGS[tag.block_type_name]
+                    dbt.exceptions.raise_compiler_error((
+                        'Got an unexpected control flow end tag, got {} but '
+                        'never saw a preceeding {} (@ {})'
+                    ).format(tag.block_type_name, expected, tag.start))
+                expected = _CONTROL_FLOW_TAGS[found]
+                if expected != tag.block_type_name:
+                    dbt.exceptions.raise_compiler_error((
+                        'Got an unexpected control flow end tag, got {} but '
+                        'expected {} next (@ {})'
+                    ).format(tag.block_type_name, expected, tag.start))
+
+            if tag.block_type_name in allowed_blocks:
+                if self.stack:
+                    dbt.exceptions.raise_compiler_error((
+                        'Got a block definition inside control flow at {}. '
+                        'All dbt block definitions must be at the top level'
+                    ).format(tag.start))
+                if self.current is not None:
+                    dbt.exceptions.raise_compiler_error(
+                        duplicate_tags.format(outer=self.current, inner=tag)
+                    )
+                if collect_raw_data:
+                    raw_data = self.data[self.last_position:tag.start]
+                    self.last_position = tag.start
+                    if raw_data:
+                        yield BlockData(raw_data)
+                self.current = tag
+
+            elif self.is_current_end(tag):
+                self.last_position = tag.end
+                yield BlockTag(
+                    block_type_name=self.current.block_type_name,
+                    block_name=self.current.block_name,
+                    contents=self.data[self.current.end:tag.start],
+                    full_block=self.data[self.current.start:tag.end]
+                )
+                self.current = None
+
+        if self.current:
+            dbt.exceptions.raise_compiler_error((
+                'Reached EOF without finding a close block for '
+                '{0.block_type_name} (from {0.end})'
+            ).format(self.current))
+
+        if collect_raw_data:
+            raw_data = self.data[self.last_position:]
+            if raw_data:
+                yield BlockData(raw_data)
+
+    def lex_for_blocks(self, allowed_blocks=None, collect_raw_data=True):
+        return list(self.find_blocks(allowed_blocks=allowed_blocks,
+                                     collect_raw_data=collect_raw_data))

core/dbt/clients/agate_helper.py

@@ -0,0 +1,65 @@
+import dbt.compat
+
+import agate
+import json
+
+DEFAULT_TYPE_TESTER = agate.TypeTester(types=[
+    agate.data_types.Number(null_values=('null', '')),
+    agate.data_types.TimeDelta(null_values=('null', '')),
+    agate.data_types.Date(null_values=('null', '')),
+    agate.data_types.DateTime(null_values=('null', '')),
+    agate.data_types.Boolean(true_values=('true',),
+                             false_values=('false',),
+                             null_values=('null', '')),
+    agate.data_types.Text(null_values=('null', ''))
+])
+
+
+def table_from_data(data, column_names):
+    "Convert list of dictionaries into an Agate table"
+
+    # The agate table is generated from a list of dicts, so the column order
+    # from `data` is not preserved. We can use `select` to reorder the columns
+    #
+    # If there is no data, create an empty table with the specified columns
+
+    if len(data) == 0:
+        return agate.Table([], column_names=column_names)
+    else:
+        table = agate.Table.from_object(data, column_types=DEFAULT_TYPE_TESTER)
+        return table.select(column_names)
+
+
+def table_from_data_flat(data, column_names):
+    "Convert list of dictionaries into an Agate table"
+
+    rows = []
+    for _row in data:
+        row = []
+        for value in list(_row.values()):
+            if isinstance(value, (dict, list, tuple)):
+                row.append(json.dumps(value))
+            else:
+                row.append(value)
+        rows.append(row)
+
+    return agate.Table(rows, column_names)
+
+
+def empty_table():
+    "Returns an empty Agate table. To be used in place of None"
+
+    return agate.Table(rows=[])
+
+
+def as_matrix(table):
+    "Return an agate table as a matrix of data sans columns"
+
+    return [r.values() for r in table.rows.values()]
+
+
+def from_csv(abspath):
+    with dbt.compat.open_seed_file(abspath) as fp:
+        if fp.read(len(dbt.compat.BOM_UTF8)) != dbt.compat.BOM_UTF8:
+            fp.seek(0)
+        return agate.Table.from_csv(fp, column_types=DEFAULT_TYPE_TESTER)

core/dbt/clients/gcloud.py

@@ -0,0 +1,26 @@
+from dbt.logger import GLOBAL_LOGGER as logger
+import dbt.exceptions
+from dbt.clients.system import run_cmd
+
+NOT_INSTALLED_MSG = """
+dbt requires the gcloud SDK to be installed to authenticate with BigQuery.
+Please download and install the SDK, or use a Service Account instead.
+
+https://cloud.google.com/sdk/
+"""
+
+
+def gcloud_installed():
+    try:
+        run_cmd('.', ['gcloud', '--version'])
+        return True
+    except OSError as e:
+        logger.debug(e)
+        return False
+
+
+def setup_default_credentials():
+    if gcloud_installed():
+        run_cmd('.', ["gcloud", "auth", "application-default", "login"])
+    else:
+        raise dbt.exceptions.RuntimeException(NOT_INSTALLED_MSG)

core/dbt/clients/git.py

@@ -0,0 +1,101 @@
+import re
+import os.path
+
+from dbt.clients.system import run_cmd, rmdir
+from dbt.logger import GLOBAL_LOGGER as logger
+import dbt.exceptions
+
+
+def clone(repo, cwd, dirname=None, remove_git_dir=False):
+    clone_cmd = ['git', 'clone', '--depth', '1', repo]
+
+    if dirname is not None:
+        clone_cmd.append(dirname)
+
+    result = run_cmd(cwd, clone_cmd, env={'LC_ALL': 'C'})
+
+    if remove_git_dir:
+        rmdir(os.path.join(dirname, '.git'))
+
+    return result
+
+
+def list_tags(cwd):
+    out, err = run_cmd(cwd, ['git', 'tag', '--list'], env={'LC_ALL': 'C'})
+    tags = out.decode('utf-8').strip().split("\n")
+    return tags
+
+
+def _checkout(cwd, repo, branch):
+    logger.debug('  Checking out branch {}.'.format(branch))
+
+    run_cmd(cwd, ['git', 'remote', 'set-branches', 'origin', branch])
+    run_cmd(cwd, ['git', 'fetch', '--tags', '--depth', '1', 'origin', branch])
+
+    tags = list_tags(cwd)
+
+    # Prefer tags to branches if one exists
+    if branch in tags:
+        spec = 'tags/{}'.format(branch)
+    else:
+        spec = 'origin/{}'.format(branch)
+
+    out, err = run_cmd(cwd, ['git', 'reset', '--hard', spec],
+                       env={'LC_ALL': 'C'})
+    return out, err
+
+
+def checkout(cwd, repo, branch=None):
+    if branch is None:
+        branch = 'master'
+    try:
+        return _checkout(cwd, repo, branch)
+    except dbt.exceptions.CommandResultError as exc:
+        stderr = exc.stderr.decode('utf-8').strip()
+    dbt.exceptions.bad_package_spec(repo, branch, stderr)
+
+
+def get_current_sha(cwd):
+    out, err = run_cmd(cwd, ['git', 'rev-parse', 'HEAD'], env={'LC_ALL': 'C'})
+
+    return out.decode('utf-8')
+
+
+def remove_remote(cwd):
+    return run_cmd(cwd, ['git', 'remote', 'rm', 'origin'], env={'LC_ALL': 'C'})
+
+
+def clone_and_checkout(repo, cwd, dirname=None, remove_git_dir=False,
+                       branch=None):
+    exists = None
+    try:
+        _, err = clone(repo, cwd, dirname=dirname,
+                       remove_git_dir=remove_git_dir)
+    except dbt.exceptions.CommandResultError as exc:
+        err = exc.stderr.decode('utf-8')
+        exists = re.match("fatal: destination path '(.+)' already exists", err)
+        if not exists:  # something else is wrong, raise it
+            raise
+
+    directory = None
+    start_sha = None
+    if exists:
+        directory = exists.group(1)
+        logger.debug('Updating existing dependency %s.', directory)
+    else:
+        matches = re.match("Cloning into '(.+)'", err.decode('utf-8'))
+        directory = matches.group(1)
+        logger.debug('Pulling new dependency %s.', directory)
+    full_path = os.path.join(cwd, directory)
+    start_sha = get_current_sha(full_path)
+    checkout(full_path, repo, branch)
+    end_sha = get_current_sha(full_path)
+    if exists:
+        if start_sha == end_sha:
+            logger.debug('  Already at %s, nothing to do.', start_sha[:7])
+        else:
+            logger.debug('  Updated checkout from %s to %s.',
+                         start_sha[:7], end_sha[:7])
+    else:
+        logger.debug('  Checked out at %s.', end_sha[:7])
+    return directory

core/dbt/clients/jinja.py

@@ -0,0 +1,328 @@
+import codecs
+import linecache
+import os
+import tempfile
+
+import jinja2
+import jinja2._compat
+import jinja2.ext
+import jinja2.nodes
+import jinja2.parser
+import jinja2.sandbox
+
+import dbt.compat
+import dbt.exceptions
+import dbt.utils
+
+from dbt.clients._jinja_blocks import BlockIterator
+
+from dbt.logger import GLOBAL_LOGGER as logger  # noqa
+
+
+def _linecache_inject(source, write):
+    if write:
+        # this is the only reliable way to accomplish this. Obviously, it's
+        # really darn noisy and will fill your temporary directory
+        tmp_file = tempfile.NamedTemporaryFile(
+            prefix='dbt-macro-compiled-',
+            suffix='.py',
+            delete=False,
+            mode='w+',
+            encoding='utf-8',
+        )
+        tmp_file.write(source)
+        filename = tmp_file.name
+    else:
+        filename = codecs.encode(os.urandom(12), 'hex').decode('ascii')
+
+    # encode, though I don't think this matters
+    filename = jinja2._compat.encode_filename(filename)
+    # put ourselves in the cache
+    linecache.cache[filename] = (
+        len(source),
+        None,
+        [line + '\n' for line in source.splitlines()],
+        filename
+    )
+    return filename
+
+
+class MacroFuzzParser(jinja2.parser.Parser):
+    def parse_macro(self):
+        node = jinja2.nodes.Macro(lineno=next(self.stream).lineno)
+
+        # modified to fuzz macros defined in the same file. this way
+        # dbt can understand the stack of macros being called.
+        #  - @cmcarthur
+        node.name = dbt.utils.get_dbt_macro_name(
+            self.parse_assign_target(name_only=True).name)
+
+        self.parse_signature(node)
+        node.body = self.parse_statements(('name:endmacro',),
+                                          drop_needle=True)
+        return node
+
+
+class MacroFuzzEnvironment(jinja2.sandbox.SandboxedEnvironment):
+    def _parse(self, source, name, filename):
+        return MacroFuzzParser(
+            self, source, name,
+            jinja2._compat.encode_filename(filename)
+        ).parse()
+
+    def _compile(self, source, filename):
+        """Override jinja's compilation to stash the rendered source inside
+        the python linecache for debugging when the appropriate environment
+        variable is set.
+
+        If the value is 'write', also write the files to disk.
+        WARNING: This can write a ton of data if you aren't careful.
+        """
+        macro_compile = os.environ.get('DBT_MACRO_DEBUGGING')
+        if filename == '<template>' and macro_compile:
+            write = macro_compile == 'write'
+            filename = _linecache_inject(source, write)
+
+        return super(MacroFuzzEnvironment, self)._compile(source, filename)
+
+
+class TemplateCache(object):
+
+    def __init__(self):
+        self.file_cache = {}
+
+    def get_node_template(self, node):
+        key = (node['package_name'], node['original_file_path'])
+
+        if key in self.file_cache:
+            return self.file_cache[key]
+
+        template = get_template(
+            string=node.get('raw_sql'),
+            ctx={},
+            node=node
+        )
+        self.file_cache[key] = template
+
+        return template
+
+    def clear(self):
+        self.file_cache.clear()
+
+
+template_cache = TemplateCache()
+
+
+def macro_generator(node):
+    def apply_context(context):
+        def call(*args, **kwargs):
+            name = node.get('name')
+            template = template_cache.get_node_template(node)
+            module = template.make_module(context, False, context)
+
+            macro = module.__dict__[dbt.utils.get_dbt_macro_name(name)]
+            module.__dict__.update(context)
+
+            try:
+                return macro(*args, **kwargs)
+            except dbt.exceptions.MacroReturn as e:
+                return e.value
+            except (TypeError, jinja2.exceptions.TemplateRuntimeError) as e:
+                dbt.exceptions.raise_compiler_error(str(e), node)
+            except dbt.exceptions.CompilationException as e:
+                e.stack.append(node)
+                raise e
+
+        return call
+    return apply_context
+
+
+class MaterializationExtension(jinja2.ext.Extension):
+    tags = ['materialization']
+
+    def parse(self, parser):
+        node = jinja2.nodes.Macro(lineno=next(parser.stream).lineno)
+        materialization_name = \
+            parser.parse_assign_target(name_only=True).name
+
+        adapter_name = 'default'
+        node.args = []
+        node.defaults = []
+
+        while parser.stream.skip_if('comma'):
+            target = parser.parse_assign_target(name_only=True)
+
+            if target.name == 'default':
+                pass
+
+            elif target.name == 'adapter':
+                parser.stream.expect('assign')
+                value = parser.parse_expression()
+                adapter_name = value.value
+
+            else:
+                dbt.exceptions.invalid_materialization_argument(
+                    materialization_name, target.name)
+
+        node.name = dbt.utils.get_materialization_macro_name(
+            materialization_name, adapter_name)
+
+        node.body = parser.parse_statements(('name:endmaterialization',),
+                                            drop_needle=True)
+
+        return node
+
+
+class DocumentationExtension(jinja2.ext.Extension):
+    tags = ['docs']
+
+    def parse(self, parser):
+        node = jinja2.nodes.Macro(lineno=next(parser.stream).lineno)
+        docs_name = parser.parse_assign_target(name_only=True).name
+
+        node.args = []
+        node.defaults = []
+        node.name = dbt.utils.get_docs_macro_name(docs_name)
+        node.body = parser.parse_statements(('name:enddocs',),
+                                            drop_needle=True)
+        return node
+
+
+def _is_dunder_name(name):
+    return name.startswith('__') and name.endswith('__')
+
+
+def create_macro_capture_env(node):
+
+    class ParserMacroCapture(jinja2.Undefined):
+        """
+        This class sets up the parser to capture macros.
+        """
+        def __init__(self, hint=None, obj=None, name=None, exc=None):
+            super(ParserMacroCapture, self).__init__(hint=hint, name=name)
+            self.node = node
+            self.name = name
+            self.package_name = node.get('package_name')
+            # jinja uses these for safety, so we have to override them.
+            # see https://github.com/pallets/jinja/blob/master/jinja2/sandbox.py#L332-L339 # noqa
+            self.unsafe_callable = False
+            self.alters_data = False
+
+        def __deepcopy__(self, memo):
+            path = os.path.join(self.node.get('root_path'),
+                                self.node.get('original_file_path'))
+
+            logger.debug(
+                'dbt encountered an undefined variable, "{}" in node {}.{} '
+                '(source path: {})'
+                .format(self.name, self.node.get('package_name'),
+                        self.node.get('name'), path))
+
+            # match jinja's message
+            dbt.exceptions.raise_compiler_error(
+                "{!r} is undefined".format(self.name),
+                node=self.node
+            )
+
+        def __getitem__(self, name):
+            # Propagate the undefined value if a caller accesses this as if it
+            # were a dictionary
+            return self
+
+        def __getattr__(self, name):
+            if name == 'name' or _is_dunder_name(name):
+                raise AttributeError(
+                    "'{}' object has no attribute '{}'"
+                    .format(type(self).__name__, name)
+                )
+
+            self.package_name = self.name
+            self.name = name
+
+            return self
+
+        def __call__(self, *args, **kwargs):
+            return self
+
+    return ParserMacroCapture
+
+
+def get_environment(node=None, capture_macros=False):
+    args = {
+        'extensions': ['jinja2.ext.do']
+    }
+
+    if capture_macros:
+        args['undefined'] = create_macro_capture_env(node)
+
+    args['extensions'].append(MaterializationExtension)
+    args['extensions'].append(DocumentationExtension)
+
+    return MacroFuzzEnvironment(**args)
+
+
+def parse(string):
+    try:
+        return get_environment().parse(dbt.compat.to_string(string))
+
+    except (jinja2.exceptions.TemplateSyntaxError,
+            jinja2.exceptions.UndefinedError) as e:
+        e.translated = False
+        dbt.exceptions.raise_compiler_error(str(e))
+
+
+def get_template(string, ctx, node=None, capture_macros=False):
+    try:
+        env = get_environment(node, capture_macros)
+
+        template_source = dbt.compat.to_string(string)
+        return env.from_string(template_source, globals=ctx)
+
+    except (jinja2.exceptions.TemplateSyntaxError,
+            jinja2.exceptions.UndefinedError) as e:
+        e.translated = False
+        dbt.exceptions.raise_compiler_error(str(e), node)
+
+
+def render_template(template, ctx, node=None):
+    try:
+        return template.render(ctx)
+
+    except (jinja2.exceptions.TemplateSyntaxError,
+            jinja2.exceptions.UndefinedError) as e:
+        e.translated = False
+        dbt.exceptions.raise_compiler_error(str(e), node)
+
+
+def get_rendered(string, ctx, node=None,
+                 capture_macros=False):
+    template = get_template(string, ctx, node,
+                            capture_macros=capture_macros)
+
+    return render_template(template, ctx, node)
+
+
+def undefined_error(msg):
+    raise jinja2.exceptions.UndefinedError(msg)
+
+
+def extract_toplevel_blocks(data, allowed_blocks=None, collect_raw_data=True):
+    """Extract the top level blocks with matching block types from a jinja
+    file, with some special handling for block nesting.
+
+    :param str data: The data to extract blocks from.
+    :param Optional[Set[str]] allowed_blocks: The names of the blocks to
+        extract from the file. They may not be nested within if/for blocks.
+        If None, use the default values.
+    :param bool collect_raw_data: If set, raw data between matched blocks will
+        also be part of the results, as `BlockData` objects. They have a
+        `block_type_name` field of `'__dbt_data'` and will never have a
+        `block_name`.
+    :return List[Union[BlockData, BlockTag]]: A list of `BlockTag`s matching
+        the allowed block types and (if `collect_raw_data` is `True`)
+        `BlockData` objects.
+    """
+    return BlockIterator(data).lex_for_blocks(
+        allowed_blocks=allowed_blocks,
+        collect_raw_data=collect_raw_data
+    )

core/dbt/clients/registry.py

@@ -0,0 +1,75 @@
+from functools import wraps
+import six
+import requests
+from dbt.exceptions import RegistryException
+from dbt.utils import memoized
+from dbt.logger import GLOBAL_LOGGER as logger
+import os
+import time
+
+if os.getenv('DBT_PACKAGE_HUB_URL'):
+    DEFAULT_REGISTRY_BASE_URL = os.getenv('DBT_PACKAGE_HUB_URL')
+else:
+    DEFAULT_REGISTRY_BASE_URL = 'https://hub.getdbt.com/'
+
+
+def _get_url(url, registry_base_url=None):
+    if registry_base_url is None:
+        registry_base_url = DEFAULT_REGISTRY_BASE_URL
+
+    return '{}{}'.format(registry_base_url, url)
+
+
+def _wrap_exceptions(fn):
+    @wraps(fn)
+    def wrapper(*args, **kwargs):
+        max_attempts = 5
+        attempt = 0
+        while True:
+            attempt += 1
+            try:
+                return fn(*args, **kwargs)
+            except requests.exceptions.ConnectionError as exc:
+                if attempt < max_attempts:
+                    time.sleep(1)
+                    continue
+                six.raise_from(
+                    RegistryException('Unable to connect to registry hub'),
+                    exc
+                )
+    return wrapper
+
+
+@_wrap_exceptions
+def _get(path, registry_base_url=None):
+    url = _get_url(path, registry_base_url)
+    logger.debug('Making package registry request: GET {}'.format(url))
+    resp = requests.get(url)
+    logger.debug('Response from registry: GET {} {}'.format(url,
+                                                            resp.status_code))
+    resp.raise_for_status()
+    return resp.json()
+
+
+def index(registry_base_url=None):
+    return _get('api/v1/index.json', registry_base_url)
+
+
+index_cached = memoized(index)
+
+
+def packages(registry_base_url=None):
+    return _get('api/v1/packages.json', registry_base_url)
+
+
+def package(name, registry_base_url=None):
+    return _get('api/v1/{}.json'.format(name), registry_base_url)
+
+
+def package_version(name, version, registry_base_url=None):
+    return _get('api/v1/{}/{}.json'.format(name, version), registry_base_url)
+
+
+def get_available_versions(name):
+    response = package(name)
+    return list(response['versions'])

core/dbt/clients/system.py

@@ -0,0 +1,402 @@
+import errno
+import fnmatch
+import json
+import os
+import os.path
+import shutil
+import subprocess
+import sys
+import tarfile
+import requests
+import stat
+
+import dbt.compat
+import dbt.exceptions
+import dbt.utils
+
+from dbt.logger import GLOBAL_LOGGER as logger
+
+
+def find_matching(root_path,
+                  relative_paths_to_search,
+                  file_pattern):
+    """
+    Given an absolute `root_path`, a list of relative paths to that
+    absolute root path (`relative_paths_to_search`), and a `file_pattern`
+    like '*.sql', returns information about the files. For example:
+
+    > find_matching('/root/path', 'models', '*.sql')
+
+      [ { 'absolute_path': '/root/path/models/model_one.sql',
+          'relative_path': 'models/model_one.sql',
+          'searched_path': 'models' },
+        { 'absolute_path': '/root/path/models/subdirectory/model_two.sql',
+          'relative_path': 'models/subdirectory/model_two.sql',
+          'searched_path': 'models' } ]
+    """
+    matching = []
+    root_path = os.path.normpath(root_path)
+
+    for relative_path_to_search in relative_paths_to_search:
+        absolute_path_to_search = os.path.join(
+            root_path, relative_path_to_search)
+        walk_results = os.walk(absolute_path_to_search)
+
+        for current_path, subdirectories, local_files in walk_results:
+            for local_file in local_files:
+                absolute_path = os.path.join(current_path, local_file)
+                relative_path = os.path.relpath(
+                    absolute_path, absolute_path_to_search)
+
+                if fnmatch.fnmatch(local_file, file_pattern):
+                    matching.append({
+                        'searched_path': relative_path_to_search,
+                        'absolute_path': absolute_path,
+                        'relative_path': relative_path,
+                    })
+
+    return matching
+
+
+def load_file_contents(path, strip=True):
+    with open(path, 'rb') as handle:
+        to_return = handle.read().decode('utf-8')
+
+    if strip:
+        to_return = to_return.strip()
+
+    return to_return
+
+
+def make_directory(path):
+    """
+    Make a directory and any intermediate directories that don't already
+    exist. This function handles the case where two threads try to create
+    a directory at once.
+    """
+    if not os.path.exists(path):
+        # concurrent writes that try to create the same dir can fail
+        try:
+            os.makedirs(path)
+
+        except OSError as e:
+            if e.errno == errno.EEXIST:
+                pass
+            else:
+                raise e
+
+
+def make_file(path, contents='', overwrite=False):
+    """
+    Make a file at `path` assuming that the directory it resides in already
+    exists. The file is saved with contents `contents`
+    """
+    if overwrite or not os.path.exists(path):
+        with open(path, 'w') as fh:
+            fh.write(contents)
+        return True
+
+    return False
+
+
+def make_symlink(source, link_path):
+    """
+    Create a symlink at `link_path` referring to `source`.
+    """
+    if not supports_symlinks():
+        dbt.exceptions.system_error('create a symbolic link')
+
+    return os.symlink(source, link_path)
+
+
+def supports_symlinks():
+    return getattr(os, "symlink", None) is not None
+
+
+def write_file(path, contents=''):
+    make_directory(os.path.dirname(path))
+    dbt.compat.write_file(path, contents)
+
+    return True
+
+
+def write_json(path, data):
+    return write_file(path, json.dumps(data, cls=dbt.utils.JSONEncoder))
+
+
+def _windows_rmdir_readonly(func, path, exc):
+    exception_val = exc[1]
+    if exception_val.errno == errno.EACCES:
+        os.chmod(path, stat.S_IWUSR)
+        func(path)
+    else:
+        raise
+
+
+def resolve_path_from_base(path_to_resolve, base_path):
+    """
+    If path-to_resolve is a relative path, create an absolute path
+    with base_path as the base.
+
+    If path_to_resolve is an absolute path or a user path (~), just
+    resolve it to an absolute path and return.
+    """
+    return os.path.abspath(
+        os.path.join(
+            base_path,
+            os.path.expanduser(path_to_resolve)))
+
+
+def rmdir(path):
+    """
+    Recursively deletes a directory. Includes an error handler to retry with
+    different permissions on Windows. Otherwise, removing directories (eg.
+    cloned via git) can cause rmtree to throw a PermissionError exception
+    """
+    logger.debug("DEBUG** Window rmdir sys.platform: {}".format(sys.platform))
+    if sys.platform == 'win32':
+        onerror = _windows_rmdir_readonly
+    else:
+        onerror = None
+
+    return shutil.rmtree(path, onerror=onerror)
+
+
+def remove_file(path):
+    return os.remove(path)
+
+
+def path_exists(path):
+    return os.path.lexists(path)
+
+
+def path_is_symlink(path):
+    return os.path.islink(path)
+
+
+def open_dir_cmd():
+    # https://docs.python.org/2/library/sys.html#sys.platform
+    if sys.platform == 'win32':
+        return 'start'
+
+    elif sys.platform == 'darwin':
+        return 'open'
+
+    else:
+        return 'xdg-open'
+
+
+def _handle_posix_cwd_error(exc, cwd, cmd):
+    if exc.errno == errno.ENOENT:
+        message = 'Directory does not exist'
+    elif exc.errno == errno.EACCES:
+        message = 'Current user cannot access directory, check permissions'
+    elif exc.errno == errno.ENOTDIR:
+        message = 'Not a directory'
+    else:
+        message = 'Unknown OSError: {} - cwd'.format(str(exc))
+    raise dbt.exceptions.WorkingDirectoryError(cwd, cmd, message)
+
+
+def _handle_posix_cmd_error(exc, cwd, cmd):
+    if exc.errno == errno.ENOENT:
+        message = "Could not find command, ensure it is in the user's PATH"
+    elif exc.errno == errno.EACCES:
+        message = 'User does not have permissions for this command'
+    else:
+        message = 'Unknown OSError: {} - cmd'.format(str(exc))
+    raise dbt.exceptions.ExecutableError(cwd, cmd, message)
+
+
+def _handle_posix_error(exc, cwd, cmd):
+    """OSError handling for posix systems.
+
+    Some things that could happen to trigger an OSError:
+        - cwd could not exist
+            - exc.errno == ENOENT
+            - exc.filename == cwd
+        - cwd could have permissions that prevent the current user moving to it
+            - exc.errno == EACCES
+            - exc.filename == cwd
+        - cwd could exist but not be a directory
+            - exc.errno == ENOTDIR
+            - exc.filename == cwd
+        - cmd[0] could not exist
+            - exc.errno == ENOENT
+            - exc.filename == None(?)
+        - cmd[0] could exist but have permissions that prevents the current
+            user from executing it (executable bit not set for the user)
+            - exc.errno == EACCES
+            - exc.filename == None(?)
+    """
+    if getattr(exc, 'filename', None) == cwd:
+        _handle_posix_cwd_error(exc, cwd, cmd)
+    else:
+        _handle_posix_cmd_error(exc, cwd, cmd)
+
+
+def _handle_windows_error(exc, cwd, cmd):
+    cls = dbt.exceptions.CommandError
+    if exc.errno == errno.ENOENT:
+        message = ("Could not find command, ensure it is in the user's PATH "
+                   "and that the user has permissions to run it")
+        cls = dbt.exceptions.ExecutableError
+    elif exc.errno == errno.ENOEXEC:
+        message = ('Command was not executable, ensure it is valid')
+        cls = dbt.exceptions.ExecutableError
+    elif exc.errno == errno.ENOTDIR:
+        message = ('Unable to cd: path does not exist, user does not have'
+                   ' permissions, or not a directory')
+        cls = dbt.exceptions.WorkingDirectoryError
+    else:
+        message = 'Unknown error: {} (errno={}: "{}")'.format(
+            str(exc), exc.errno, errno.errorcode.get(exc.errno, '<Unknown!>')
+        )
+    raise cls(cwd, cmd, message)
+
+
+def _interpret_oserror(exc, cwd, cmd):
+    """Interpret an OSError exc and raise the appropriate dbt exception.
+
+    """
+    if len(cmd) == 0:
+        raise dbt.exceptions.CommandError(cwd, cmd)
+
+    # all of these functions raise unconditionally
+    if os.name == 'nt':
+        _handle_windows_error(exc, cwd, cmd)
+    else:
+        _handle_posix_error(exc, cwd, cmd)
+
+    # this should not be reachable, raise _something_ at least!
+    raise dbt.exceptions.InternalException(
+        'Unhandled exception in _interpret_oserror: {}'.format(exc)
+    )
+
+
+def run_cmd(cwd, cmd, env=None):
+    logger.debug('Executing "{}"'.format(' '.join(cmd)))
+    if len(cmd) == 0:
+        raise dbt.exceptions.CommandError(cwd, cmd)
+
+    # the env argument replaces the environment entirely, which has exciting
+    # consequences on Windows! Do an update instead.
+    full_env = env
+    if env is not None:
+        full_env = os.environ.copy()
+        full_env.update(env)
+
+    try:
+        proc = subprocess.Popen(
+            cmd,
+            cwd=cwd,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            env=full_env)
+
+        out, err = proc.communicate()
+    except OSError as exc:
+        _interpret_oserror(exc, cwd, cmd)
+
+    logger.debug('STDOUT: "{}"'.format(out))
+    logger.debug('STDERR: "{}"'.format(err))
+
+    if proc.returncode != 0:
+        logger.debug('command return code={}'.format(proc.returncode))
+        raise dbt.exceptions.CommandResultError(cwd, cmd, proc.returncode,
+                                                out, err)
+
+    return out, err
+
+
+def download(url, path):
+    response = requests.get(url)
+    with open(path, 'wb') as handle:
+        for block in response.iter_content(1024 * 64):
+            handle.write(block)
+
+
+def rename(from_path, to_path, force=False):
+    is_symlink = path_is_symlink(to_path)
+
+    if os.path.exists(to_path) and force:
+        if is_symlink:
+            remove_file(to_path)
+        else:
+            rmdir(to_path)
+
+    shutil.move(from_path, to_path)
+
+
+def untar_package(tar_path, dest_dir, rename_to=None):
+    tar_dir_name = None
+    with tarfile.open(tar_path, 'r') as tarball:
+        tarball.extractall(dest_dir)
+        tar_dir_name = os.path.commonprefix(tarball.getnames())
+    if rename_to:
+        downloaded_path = os.path.join(dest_dir, tar_dir_name)
+        desired_path = os.path.join(dest_dir, rename_to)
+        dbt.clients.system.rename(downloaded_path, desired_path, force=True)
+
+
+def chmod_and_retry(func, path, exc_info):
+    """Define an error handler to pass to shutil.rmtree.
+    On Windows, when a file is marked read-only as git likes to do, rmtree will
+    fail. To handle that, on errors try to make the file writable.
+    We want to retry most operations here, but listdir is one that we know will
+    be useless.
+    """
+    if func is os.listdir or os.name != 'nt':
+        raise
+    os.chmod(path, stat.S_IREAD | stat.S_IWRITE)
+    # on error,this will raise.
+    func(path)
+
+
+def _absnorm(path):
+    return os.path.normcase(os.path.abspath(path))
+
+
+def move(src, dst):
+    """A re-implementation of shutil.move that properly removes the source
+    directory on windows when it has read-only files in it and the move is
+    between two drives.
+
+    This is almost identical to the real shutil.move, except it uses our rmtree
+    and skips handling non-windows OSes since the existing one works ok there.
+    """
+    if os.name != 'nt':
+        return shutil.move(src, dst)
+
+    if os.path.isdir(dst):
+        if _absnorm(src) == _absnorm(dst):
+            os.rename(src, dst)
+            return
+
+        dst = os.path.join(dst, os.path.basename(src.rstrip('/\\')))
+        if os.path.exists(dst):
+            raise EnvironmentError("Path '{}' already exists".format(dst))
+
+    try:
+        os.rename(src, dst)
+    except OSError:
+        # probably different drives
+        if os.path.isdir(src):
+            if _absnorm(dst + '\\').startswith(_absnorm(src + '\\')):
+                # dst is inside src
+                raise EnvironmentError(
+                    "Cannot move a directory '{}' into itself '{}'"
+                    .format(src, dst)
+                )
+            shutil.copytree(src, dst, symlinks=True)
+            rmtree(src)
+        else:
+            shutil.copy2(src, dst)
+            os.unlink(src)
+
+
+def rmtree(path):
+    """Recursively remove path. On permissions errors on windows, try to remove
+    the read-only flag and try again.
+    """
+    return shutil.rmtree(path, onerror=chmod_and_retry)

core/dbt/clients/yaml_helper.py

@@ -0,0 +1,57 @@
+import dbt.compat
+import dbt.exceptions
+
+import yaml
+import yaml.scanner
+
+
+YAML_ERROR_MESSAGE = """
+Syntax error near line {line_number}
+------------------------------
+{nice_error}
+
+Raw Error:
+------------------------------
+{raw_error}
+""".strip()
+
+
+def line_no(i, line, width=3):
+    line_number = dbt.compat.to_string(i).ljust(width)
+    return "{}| {}".format(line_number, line)
+
+
+def prefix_with_line_numbers(string, no_start, no_end):
+    line_list = string.split('\n')
+
+    numbers = range(no_start, no_end)
+    relevant_lines = line_list[no_start:no_end]
+
+    return "\n".join([
+        line_no(i + 1, line) for (i, line) in zip(numbers, relevant_lines)
+    ])
+
+
+def contextualized_yaml_error(raw_contents, error):
+    mark = error.problem_mark
+
+    min_line = max(mark.line - 3, 0)
+    max_line = mark.line + 4
+
+    nice_error = prefix_with_line_numbers(raw_contents, min_line, max_line)
+
+    return YAML_ERROR_MESSAGE.format(line_number=mark.line + 1,
+                                     nice_error=nice_error,
+                                     raw_error=error)
+
+
+def load_yaml_text(contents):
+    try:
+        return yaml.safe_load(contents)
+    except (yaml.scanner.ScannerError, yaml.YAMLError) as e:
+        if hasattr(e, 'problem_mark'):
+            error = contextualized_yaml_error(contents, e)
+        else:
+            error = dbt.compat.to_string(e)
+
+        raise dbt.exceptions.ValidationException(error)

core/dbt/compat.py

@@ -0,0 +1,152 @@
+# flake8: noqa
+
+import abc
+import codecs
+import warnings
+import decimal
+
+try:
+    import cdecimal
+except ImportError:
+    DECIMALS = (decimal.Decimal,)
+else:
+    DECIMALS = (decimal.Decimal, cdecimal.Decimal)
+
+WHICH_PYTHON = None
+
+try:
+    basestring
+    WHICH_PYTHON = 2
+except NameError:
+    WHICH_PYTHON = 3
+
+if WHICH_PYTHON == 2:
+    basestring = basestring
+    bigint = long
+    NUMBERS = DECIMALS + (int, float, long)
+    import __builtin__ as builtins
+else:
+    basestring = str
+    bigint = int
+    NUMBERS = DECIMALS + (int, float)
+    import builtins
+
+if WHICH_PYTHON == 2:
+    from SimpleHTTPServer import SimpleHTTPRequestHandler
+    from SocketServer import TCPServer
+    from Queue import PriorityQueue, Empty as QueueEmpty
+    from thread import get_ident
+else:
+    from http.server import SimpleHTTPRequestHandler
+    from socketserver import TCPServer
+    from queue import PriorityQueue, Empty as QueueEmpty
+    from threading import get_ident
+
+
+def to_unicode(s):
+    if WHICH_PYTHON == 2:
+        return unicode(s)
+    else:
+        return str(s)
+
+
+def to_string(s):
+    if WHICH_PYTHON == 2:
+        if isinstance(s, unicode):
+            return s
+        elif isinstance(s, basestring):
+            return to_unicode(s)
+        else:
+            return to_unicode(str(s))
+    else:
+        if isinstance(s, basestring):
+            return s
+        else:
+            return str(s)
+
+
+def to_native_string(s):
+    if WHICH_PYTHON == 2:
+        if isinstance(s, unicode):
+            return str(s)
+        elif isinstance(s, basestring):
+            return s
+        else:
+            return str(s)
+    else:
+        if isinstance(s, basestring):
+            return s
+        else:
+            return str(s)
+
+
+def write_file(path, s):
+    if WHICH_PYTHON == 2:
+        open = codecs.open
+    else:
+        open = builtins.open
+    with open(path, 'w', encoding='utf-8') as f:
+        return f.write(to_string(s))
+
+
+def open_file(path):
+    """Open the path for reading. It must be utf-8 encoded."""
+    if WHICH_PYTHON == 2:
+        open = codecs.open
+    else:
+        open = builtins.open
+    return open(path, encoding='utf-8')
+
+
+if WHICH_PYTHON == 2:
+    BOM_UTF8 = codecs.BOM_UTF8
+else:
+    BOM_UTF8 = codecs.BOM_UTF8.decode('utf-8')
+
+
+def open_seed_file(path):
+    if WHICH_PYTHON == 2:
+        fp = open(path, 'Urb')
+    else:
+        fp = open(path, encoding='utf-8')
+    return fp
+
+
+if WHICH_PYTHON == 2:
+    # In python 2, classmethod and staticmethod do not allow setters, so you
+    # can't treat classmethods as first-class objects like you can regular
+    # functions. This rarely matters, but for metaclass shenanigans on the
+    # adapter we do want to set attributes on classmethods.
+    class _classmethod(classmethod):
+        pass
+
+    classmethod = _classmethod
+
+    # python 2.7 is missing this
+    class abstractclassmethod(classmethod):
+        __isabstractmethod__ = True
+
+        def __init__(self, func):
+            func.__isabstractmethod__ = True
+            super(abstractclassmethod, self).__init__(func)
+
+    class abstractstaticmethod(staticmethod):
+        __isabstractmethod__ = True
+
+        def __init__(self, func):
+            func.__isabstractmethod__ = True
+            super(abstractstaticmethod, self).__init__(func)
+
+else:
+    abstractclassmethod = abc.abstractclassmethod
+    abstractstaticmethod = abc.abstractstaticmethod
+    classmethod = classmethod
+
+
+def suppress_warnings():
+    # in python 2, ResourceWarnings don't exist.
+    # in python 3, suppress ResourceWarnings about unclosed sockets, as the
+    # bigquery library never closes them.
+    if WHICH_PYTHON == 3:
+        warnings.filterwarnings("ignore", category=ResourceWarning,
+                                message="unclosed.*<socket.socket.*>")

core/dbt/compilation.py

@@ -0,0 +1,252 @@
+import itertools
+import os
+from collections import defaultdict
+
+import dbt.utils
+import dbt.include
+import dbt.tracking
+
+from dbt.utils import get_materialization, NodeType, is_type
+from dbt.linker import Linker
+
+import dbt.compat
+import dbt.context.runtime
+import dbt.contracts.project
+import dbt.exceptions
+import dbt.flags
+import dbt.loader
+import dbt.config
+from dbt.contracts.graph.compiled import CompiledNode
+
+from dbt.logger import GLOBAL_LOGGER as logger
+
+graph_file_name = 'graph.gpickle'
+
+
+def print_compile_stats(stats):
+    names = {
+        NodeType.Model: 'model',
+        NodeType.Test: 'test',
+        NodeType.Snapshot: 'snapshot',
+        NodeType.Analysis: 'analyse',
+        NodeType.Macro: 'macro',
+        NodeType.Operation: 'operation',
+        NodeType.Seed: 'seed file',
+        NodeType.Source: 'source',
+    }
+
+    results = {k: 0 for k in names.keys()}
+    results.update(stats)
+
+    stat_line = ", ".join(
+        [dbt.utils.pluralize(ct, names.get(t)) for t, ct in results.items()])
+
+    logger.notice("Found {}".format(stat_line))
+
+
+def _add_prepended_cte(prepended_ctes, new_cte):
+    for dct in prepended_ctes:
+        if dct['id'] == new_cte['id']:
+            dct['sql'] = new_cte['sql']
+            return
+    prepended_ctes.append(new_cte)
+
+
+def _extend_prepended_ctes(prepended_ctes, new_prepended_ctes):
+    for new_cte in new_prepended_ctes:
+        _add_prepended_cte(prepended_ctes, new_cte)
+
+
+def prepend_ctes(model, manifest):
+    model, _, manifest = recursively_prepend_ctes(model, manifest)
+
+    return (model, manifest)
+
+
+def recursively_prepend_ctes(model, manifest):
+    if model.extra_ctes_injected:
+        return (model, model.extra_ctes, manifest)
+
+    if dbt.flags.STRICT_MODE:
+        # ensure that the cte we're adding to is compiled
+        CompiledNode(**model.serialize())
+
+    prepended_ctes = []
+
+    for cte in model.extra_ctes:
+        cte_id = cte['id']
+        cte_to_add = manifest.nodes.get(cte_id)
+        cte_to_add, new_prepended_ctes, manifest = recursively_prepend_ctes(
+            cte_to_add, manifest)
+        _extend_prepended_ctes(prepended_ctes, new_prepended_ctes)
+        new_cte_name = '__dbt__CTE__{}'.format(cte_to_add.get('name'))
+        sql = ' {} as (\n{}\n)'.format(new_cte_name, cte_to_add.compiled_sql)
+        _add_prepended_cte(prepended_ctes, {'id': cte_id, 'sql': sql})
+
+    model.prepend_ctes(prepended_ctes)
+
+    manifest.nodes[model.unique_id] = model
+
+    return (model, prepended_ctes, manifest)
+
+
+class Compiler(object):
+    def __init__(self, config):
+        self.config = config
+
+    def initialize(self):
+        dbt.clients.system.make_directory(self.config.target_path)
+        dbt.clients.system.make_directory(self.config.modules_path)
+
+    def compile_node(self, node, manifest, extra_context=None):
+        if extra_context is None:
+            extra_context = {}
+
+        logger.debug("Compiling {}".format(node.get('unique_id')))
+
+        data = node.to_dict()
+        data.update({
+            'compiled': False,
+            'compiled_sql': None,
+            'extra_ctes_injected': False,
+            'extra_ctes': [],
+            'injected_sql': None,
+        })
+        compiled_node = CompiledNode(**data)
+
+        context = dbt.context.runtime.generate(
+            compiled_node, self.config, manifest)
+        context.update(extra_context)
+
+        compiled_node.compiled_sql = dbt.clients.jinja.get_rendered(
+            node.get('raw_sql'),
+            context,
+            node)
+
+        compiled_node.compiled = True
+
+        injected_node, _ = prepend_ctes(compiled_node, manifest)
+
+        should_wrap = {NodeType.Test, NodeType.Operation}
+        if injected_node.resource_type in should_wrap:
+            # data tests get wrapped in count(*)
+            # TODO : move this somewhere more reasonable
+            if 'data' in injected_node.tags and \
+               is_type(injected_node, NodeType.Test):
+                injected_node.wrapped_sql = (
+                    "select count(*) as errors "
+                    "from (\n{test_sql}\n) sbq").format(
+                        test_sql=injected_node.injected_sql)
+            else:
+                # don't wrap schema tests or analyses.
+                injected_node.wrapped_sql = injected_node.injected_sql
+
+        elif is_type(injected_node, NodeType.Snapshot):
+            # unfortunately we do everything automagically for
+            # snapshots. in the future it'd be nice to generate
+            # the SQL at the parser level.
+            pass
+
+        elif(is_type(injected_node, NodeType.Model) and
+             get_materialization(injected_node) == 'ephemeral'):
+            pass
+
+        else:
+            injected_node.wrapped_sql = None
+
+        return injected_node
+
+    def write_graph_file(self, linker, manifest):
+        filename = graph_file_name
+        graph_path = os.path.join(self.config.target_path, filename)
+        linker.write_graph(graph_path, manifest)
+
+    def link_node(self, linker, node, manifest):
+        linker.add_node(node.unique_id)
+
+        for dependency in node.depends_on_nodes:
+            if manifest.nodes.get(dependency):
+                linker.dependency(
+                    node.unique_id,
+                    (manifest.nodes.get(dependency).unique_id))
+            else:
+                dbt.exceptions.dependency_not_found(node, dependency)
+
+    def link_graph(self, linker, manifest):
+        for node in manifest.nodes.values():
+            self.link_node(linker, node, manifest)
+
+        cycle = linker.find_cycles()
+
+        if cycle:
+            raise RuntimeError("Found a cycle: {}".format(cycle))
+
+    def compile(self, manifest, write=True):
+        linker = Linker()
+
+        self.link_graph(linker, manifest)
+
+        stats = defaultdict(int)
+
+        for node_name, node in itertools.chain(
+                manifest.nodes.items(),
+                manifest.macros.items()):
+            stats[node.resource_type] += 1
+
+        if write:
+            self.write_graph_file(linker, manifest)
+        print_compile_stats(stats)
+
+        return linker
+
+
+def compile_manifest(config, manifest, write=True):
+    compiler = Compiler(config)
+    compiler.initialize()
+    return compiler.compile(manifest, write=write)
+
+
+def _is_writable(node):
+    if not node.injected_sql:
+        return False
+
+    if dbt.utils.is_type(node, NodeType.Snapshot):
+        return False
+
+    return True
+
+
+def compile_node(adapter, config, node, manifest, extra_context, write=True):
+    compiler = Compiler(config)
+    node = compiler.compile_node(node, manifest, extra_context)
+    node = _inject_runtime_config(adapter, node, extra_context)
+
+    if write and _is_writable(node):
+        logger.debug('Writing injected SQL for node "{}"'.format(
+            node.unique_id))
+
+        written_path = dbt.writer.write_node(
+            node,
+            config.target_path,
+            'compiled',
+            node.injected_sql)
+
+        node.build_path = written_path
+
+    return node
+
+
+def _inject_runtime_config(adapter, node, extra_context):
+    wrapped_sql = node.wrapped_sql
+    context = _node_context(adapter, node)
+    context.update(extra_context)
+    sql = dbt.clients.jinja.get_rendered(wrapped_sql, context)
+    node.wrapped_sql = sql
+    return node
+
+
+def _node_context(adapter, node):
+    return {
+        "run_started_at": dbt.tracking.active_user.run_started_at,
+        "invocation_id": dbt.tracking.active_user.invocation_id,
+    }

core/dbt/config/__init__.py

@@ -0,0 +1,5 @@
+# all these are just exports, they need "noqa" so flake8 will not complain.
+from .renderer import ConfigRenderer  # noqa
+from .profile import Profile, UserConfig, PROFILES_DIR  # noqa
+from .project import Project  # noqa
+from .runtime import RuntimeConfig  # noqa

core/dbt/config/profile.py

@@ -0,0 +1,382 @@
+import os
+
+from dbt.adapters.factory import load_plugin
+from dbt.clients.system import load_file_contents
+from dbt.clients.yaml_helper import load_yaml_text
+from dbt.contracts.project import ProfileConfig
+from dbt.exceptions import DbtProfileError
+from dbt.exceptions import DbtProjectError
+from dbt.exceptions import ValidationException
+from dbt.exceptions import RuntimeException
+from dbt.logger import GLOBAL_LOGGER as logger
+from dbt.utils import parse_cli_vars
+from dbt import tracking
+from dbt.ui import printer
+
+from .renderer import ConfigRenderer
+
+DEFAULT_THREADS = 1
+DEFAULT_SEND_ANONYMOUS_USAGE_STATS = True
+DEFAULT_USE_COLORS = True
+DEFAULT_PROFILES_DIR = os.path.join(os.path.expanduser('~'), '.dbt')
+PROFILES_DIR = os.path.expanduser(
+    os.environ.get('DBT_PROFILES_DIR', DEFAULT_PROFILES_DIR)
+)
+
+INVALID_PROFILE_MESSAGE = """
+dbt encountered an error while trying to read your profiles.yml file.
+
+{error_string}
+"""
+
+
+NO_SUPPLIED_PROFILE_ERROR = """\
+dbt cannot run because no profile was specified for this dbt project.
+To specify a profile for this project, add a line like the this to
+your dbt_project.yml file:
+
+profile: [profile name]
+
+Here, [profile name] should be replaced with a profile name
+defined in your profiles.yml file. You can find profiles.yml here:
+
+{profiles_file}/profiles.yml
+""".format(profiles_file=PROFILES_DIR)
+
+
+def read_profile(profiles_dir):
+    path = os.path.join(profiles_dir, 'profiles.yml')
+
+    contents = None
+    if os.path.isfile(path):
+        try:
+            contents = load_file_contents(path, strip=False)
+            return load_yaml_text(contents)
+        except ValidationException as e:
+            msg = INVALID_PROFILE_MESSAGE.format(error_string=e)
+            raise ValidationException(msg)
+
+    return {}
+
+
+class UserConfig(object):
+    def __init__(self, send_anonymous_usage_stats, use_colors, printer_width):
+        self.send_anonymous_usage_stats = send_anonymous_usage_stats
+        self.use_colors = use_colors
+        self.printer_width = printer_width
+
+    @classmethod
+    def from_dict(cls, cfg=None):
+        if cfg is None:
+            cfg = {}
+        send_anonymous_usage_stats = cfg.get(
+            'send_anonymous_usage_stats',
+            DEFAULT_SEND_ANONYMOUS_USAGE_STATS
+        )
+        use_colors = cfg.get(
+            'use_colors',
+            DEFAULT_USE_COLORS
+        )
+        printer_width = cfg.get(
+            'printer_width'
+        )
+        return cls(send_anonymous_usage_stats, use_colors, printer_width)
+
+    def to_dict(self):
+        return {
+            'send_anonymous_usage_stats': self.send_anonymous_usage_stats,
+            'use_colors': self.use_colors,
+        }
+
+    @classmethod
+    def from_directory(cls, directory):
+        user_cfg = None
+        profile = read_profile(directory)
+        if profile:
+            user_cfg = profile.get('config', {})
+        return cls.from_dict(user_cfg)
+
+    def set_values(self, cookie_dir):
+        if self.send_anonymous_usage_stats:
+            tracking.initialize_tracking(cookie_dir)
+        else:
+            tracking.do_not_track()
+
+        if self.use_colors:
+            printer.use_colors()
+
+        if self.printer_width:
+            printer.printer_width(self.printer_width)
+
+
+class Profile(object):
+    def __init__(self, profile_name, target_name, config, threads,
+                 credentials):
+        self.profile_name = profile_name
+        self.target_name = target_name
+        if isinstance(config, dict):
+            config = UserConfig.from_dict(config)
+        self.config = config
+        self.threads = threads
+        self.credentials = credentials
+
+    def to_profile_info(self, serialize_credentials=False):
+        """Unlike to_project_config, this dict is not a mirror of any existing
+        on-disk data structure. It's used when creating a new profile from an
+        existing one.
+
+        :param serialize_credentials bool: If True, serialize the credentials.
+            Otherwise, the Credentials object will be copied.
+        :returns dict: The serialized profile.
+        """
+        result = {
+            'profile_name': self.profile_name,
+            'target_name': self.target_name,
+            'config': self.config.to_dict(),
+            'threads': self.threads,
+            'credentials': self.credentials.incorporate(),
+        }
+        if serialize_credentials:
+            result['credentials'] = result['credentials'].serialize()
+        return result
+
+    def __str__(self):
+        return str(self.to_profile_info())
+
+    def __eq__(self, other):
+        if not (isinstance(other, self.__class__) and
+                isinstance(self, other.__class__)):
+            return False
+        return self.to_profile_info() == other.to_profile_info()
+
+    def validate(self):
+        if self.credentials:
+            self.credentials.validate()
+        try:
+            ProfileConfig(**self.to_profile_info(serialize_credentials=True))
+        except ValidationException as exc:
+            raise DbtProfileError(str(exc))
+
+    @staticmethod
+    def _credentials_from_profile(profile, profile_name, target_name):
+        # credentials carry their 'type' in their actual type, not their
+        # attributes. We do want this in order to pick our Credentials class.
+        if 'type' not in profile:
+            raise DbtProfileError(
+                'required field "type" not found in profile {} and target {}'
+                .format(profile_name, target_name))
+
+        typename = profile.pop('type')
+
+        try:
+            cls = load_plugin(typename)
+            credentials = cls(**profile)
+        except RuntimeException as e:
+            raise DbtProfileError(
+                'Credentials in profile "{}", target "{}" invalid: {}'
+                .format(profile_name, target_name, str(e))
+            )
+        return credentials
+
+    @staticmethod
+    def pick_profile_name(args_profile_name, project_profile_name=None):
+        profile_name = project_profile_name
+        if args_profile_name is not None:
+            profile_name = args_profile_name
+        if profile_name is None:
+            raise DbtProjectError(NO_SUPPLIED_PROFILE_ERROR)
+        return profile_name
+
+    @staticmethod
+    def _get_profile_data(profile, profile_name, target_name):
+        if 'outputs' not in profile:
+            raise DbtProfileError(
+                "outputs not specified in profile '{}'".format(profile_name)
+            )
+        outputs = profile['outputs']
+
+        if target_name not in outputs:
+            outputs = '\n'.join(' - {}'.format(output)
+                                for output in outputs)
+            msg = ("The profile '{}' does not have a target named '{}'. The "
+                   "valid target names for this profile are:\n{}"
+                   .format(profile_name, target_name, outputs))
+            raise DbtProfileError(msg, result_type='invalid_target')
+        profile_data = outputs[target_name]
+        return profile_data
+
+    @classmethod
+    def from_credentials(cls, credentials, threads, profile_name, target_name,
+                         user_cfg=None):
+        """Create a profile from an existing set of Credentials and the
+        remaining information.
+
+        :param credentials dict: The credentials dict for this profile.
+        :param threads int: The number of threads to use for connections.
+        :param profile_name str: The profile name used for this profile.
+        :param target_name str: The target name used for this profile.
+        :param user_cfg Optional[dict]: The user-level config block from the
+            raw profiles, if specified.
+        :raises DbtProfileError: If the profile is invalid.
+        :returns Profile: The new Profile object.
+        """
+        config = UserConfig.from_dict(user_cfg)
+        profile = cls(
+            profile_name=profile_name,
+            target_name=target_name,
+            config=config,
+            threads=threads,
+            credentials=credentials
+        )
+        profile.validate()
+        return profile
+
+    @classmethod
+    def render_profile(cls, raw_profile, profile_name, target_override,
+                       cli_vars):
+        """This is a containment zone for the hateful way we're rendering
+        profiles.
+        """
+        renderer = ConfigRenderer(cli_vars=cli_vars)
+
+        # rendering profiles is a bit complex. Two constraints cause trouble:
+        # 1) users should be able to use environment/cli variables to specify
+        #    the target in their profile.
+        # 2) Missing environment/cli variables in profiles/targets that don't
+        #    end up getting selected should not cause errors.
+        # so first we'll just render the target name, then we use that rendered
+        # name to extract a profile that we can render.
+        if target_override is not None:
+            target_name = target_override
+        elif 'target' in raw_profile:
+            # render the target if it was parsed from yaml
+            target_name = renderer.render_value(raw_profile['target'])
+        else:
+            target_name = 'default'
+            logger.debug(
+                "target not specified in profile '{}', using '{}'"
+                .format(profile_name, target_name)
+            )
+
+        raw_profile_data = cls._get_profile_data(
+            raw_profile, profile_name, target_name
+        )
+
+        profile_data = renderer.render_profile_data(raw_profile_data)
+        return target_name, profile_data
+
+    @classmethod
+    def from_raw_profile_info(cls, raw_profile, profile_name, cli_vars,
+                              user_cfg=None, target_override=None,
+                              threads_override=None):
+        """Create a profile from its raw profile information.
+
+         (this is an intermediate step, mostly useful for unit testing)
+
+        :param raw_profile dict: The profile data for a single profile, from
+            disk as yaml and its values rendered with jinja.
+        :param profile_name str: The profile name used.
+        :param cli_vars dict: The command-line variables passed as arguments,
+            as a dict.
+        :param user_cfg Optional[dict]: The global config for the user, if it
+            was present.
+        :param target_override Optional[str]: The target to use, if provided on
+            the command line.
+        :param threads_override Optional[str]: The thread count to use, if
+            provided on the command line.
+        :raises DbtProfileError: If the profile is invalid or missing, or the
+            target could not be found
+        :returns Profile: The new Profile object.
+        """
+        # user_cfg is not rendered since it only contains booleans.
+        # TODO: should it be, and the values coerced to bool?
+        target_name, profile_data = cls.render_profile(
+            raw_profile, profile_name, target_override, cli_vars
+        )
+
+        # valid connections never include the number of threads, but it's
+        # stored on a per-connection level in the raw configs
+        threads = profile_data.pop('threads', DEFAULT_THREADS)
+        if threads_override is not None:
+            threads = threads_override
+
+        credentials = cls._credentials_from_profile(
+            profile_data, profile_name, target_name
+        )
+
+        return cls.from_credentials(
+            credentials=credentials,
+            profile_name=profile_name,
+            target_name=target_name,
+            threads=threads,
+            user_cfg=user_cfg
+        )
+
+    @classmethod
+    def from_raw_profiles(cls, raw_profiles, profile_name, cli_vars,
+                          target_override=None, threads_override=None):
+        """
+        :param raw_profiles dict: The profile data, from disk as yaml.
+        :param profile_name str: The profile name to use.
+        :param cli_vars dict: The command-line variables passed as arguments,
+            as a dict.
+        :param target_override Optional[str]: The target to use, if provided on
+            the command line.
+        :param threads_override Optional[str]: The thread count to use, if
+            provided on the command line.
+        :raises DbtProjectError: If there is no profile name specified in the
+            project or the command line arguments
+        :raises DbtProfileError: If the profile is invalid or missing, or the
+            target could not be found
+        :returns Profile: The new Profile object.
+        """
+        if profile_name not in raw_profiles:
+            raise DbtProjectError(
+                "Could not find profile named '{}'".format(profile_name)
+            )
+
+        # First, we've already got our final decision on profile name, and we
+        # don't render keys, so we can pluck that out
+        raw_profile = raw_profiles[profile_name]
+
+        user_cfg = raw_profiles.get('config')
+
+        return cls.from_raw_profile_info(
+            raw_profile=raw_profile,
+            profile_name=profile_name,
+            cli_vars=cli_vars,
+            user_cfg=user_cfg,
+            target_override=target_override,
+            threads_override=threads_override,
+        )
+
+    @classmethod
+    def from_args(cls, args, project_profile_name=None):
+        """Given the raw profiles as read from disk and the name of the desired
+        profile if specified, return the profile component of the runtime
+        config.
+
+        :param args argparse.Namespace: The arguments as parsed from the cli.
+        :param project_profile_name Optional[str]: The profile name, if
+            specified in a project.
+        :raises DbtProjectError: If there is no profile name specified in the
+            project or the command line arguments, or if the specified profile
+            is not found
+        :raises DbtProfileError: If the profile is invalid or missing, or the
+            target could not be found.
+        :returns Profile: The new Profile object.
+        """
+        cli_vars = parse_cli_vars(getattr(args, 'vars', '{}'))
+        threads_override = getattr(args, 'threads', None)
+        target_override = getattr(args, 'target', None)
+        raw_profiles = read_profile(args.profiles_dir)
+        profile_name = cls.pick_profile_name(args.profile,
+                                             project_profile_name)
+
+        return cls.from_raw_profiles(
+            raw_profiles=raw_profiles,
+            profile_name=profile_name,
+            cli_vars=cli_vars,
+            target_override=target_override,
+            threads_override=threads_override
+        )

core/dbt/config/project.py

@@ -0,0 +1,449 @@
+from copy import deepcopy
+import hashlib
+import os
+
+from dbt import compat
+from dbt.clients.system import resolve_path_from_base
+from dbt.clients.system import path_exists
+from dbt.clients.system import load_file_contents
+from dbt.clients.yaml_helper import load_yaml_text
+from dbt.exceptions import DbtProjectError
+from dbt.exceptions import RecursionException
+from dbt.exceptions import SemverException
+from dbt.exceptions import ValidationException
+from dbt.exceptions import warn_or_error
+from dbt.semver import VersionSpecifier
+from dbt.semver import versions_compatible
+from dbt.version import get_installed_version
+from dbt.ui import printer
+from dbt.utils import deep_map
+from dbt.utils import parse_cli_vars
+from dbt.parser.source_config import SourceConfig
+
+from dbt.contracts.project import Project as ProjectContract
+from dbt.contracts.project import PackageConfig
+
+from .renderer import ConfigRenderer
+
+
+UNUSED_RESOURCE_CONFIGURATION_PATH_MESSAGE = """\
+WARNING: Configuration paths exist in your dbt_project.yml file which do not \
+apply to any resources.
+There are {} unused configuration paths:\n{}
+"""
+
+
+INVALID_VERSION_ERROR = """\
+This version of dbt is not supported with the '{package}' package.
+  Installed version of dbt: {installed}
+  Required version of dbt for '{package}': {version_spec}
+Check the requirements for the '{package}' package, or run dbt again with \
+--no-version-check
+"""
+
+
+IMPOSSIBLE_VERSION_ERROR = """\
+The package version requirement can never be satisfied for the '{package}
+package.
+  Required versions of dbt for '{package}': {version_spec}
+Check the requirements for the '{package}' package, or run dbt again with \
+--no-version-check
+"""
+
+
+def _list_if_none(value):
+    if value is None:
+        value = []
+    return value
+
+
+def _dict_if_none(value):
+    if value is None:
+        value = {}
+    return value
+
+
+def _list_if_none_or_string(value):
+    value = _list_if_none(value)
+    if isinstance(value, compat.basestring):
+        return [value]
+    return value
+
+
+def _load_yaml(path):
+    contents = load_file_contents(path)
+    return load_yaml_text(contents)
+
+
+def _get_config_paths(config, path=(), paths=None):
+    if paths is None:
+        paths = set()
+
+    for key, value in config.items():
+        if isinstance(value, dict):
+            if key in SourceConfig.ConfigKeys:
+                if path not in paths:
+                    paths.add(path)
+            else:
+                _get_config_paths(value, path + (key,), paths)
+        else:
+            if path not in paths:
+                paths.add(path)
+
+    return frozenset(paths)
+
+
+def _is_config_used(path, fqns):
+    if fqns:
+        for fqn in fqns:
+            if len(path) <= len(fqn) and fqn[:len(path)] == path:
+                return True
+    return False
+
+
+def package_data_from_root(project_root):
+    package_filepath = resolve_path_from_base(
+        'packages.yml', project_root
+    )
+
+    if path_exists(package_filepath):
+        packages_dict = _load_yaml(package_filepath)
+    else:
+        packages_dict = None
+    return packages_dict
+
+
+def package_config_from_data(packages_data):
+    if packages_data is None:
+        packages_data = {'packages': []}
+
+    try:
+        packages = PackageConfig(**packages_data)
+    except ValidationException as e:
+        raise DbtProjectError('Invalid package config: {}'.format(str(e)))
+    return packages
+
+
+def _parse_versions(versions):
+    """Parse multiple versions as read from disk. The versions value may be any
+    one of:
+        - a single version string ('>0.12.1')
+        - a single string specifying multiple comma-separated versions
+            ('>0.11.1,<=0.12.2')
+        - an array of single-version strings (['>0.11.1', '<=0.12.2'])
+
+    Regardless, this will return a list of VersionSpecifiers
+    """
+    if isinstance(versions, compat.basestring):
+        versions = versions.split(',')
+    return [VersionSpecifier.from_version_string(v) for v in versions]
+
+
+class Project(object):
+    def __init__(self, project_name, version, project_root, profile_name,
+                 source_paths, macro_paths, data_paths, test_paths,
+                 analysis_paths, docs_paths, target_path, snapshot_paths,
+                 clean_targets, log_path, modules_path, quoting, models,
+                 on_run_start, on_run_end, archive, seeds, dbt_version,
+                 packages):
+        self.project_name = project_name
+        self.version = version
+        self.project_root = project_root
+        self.profile_name = profile_name
+        self.source_paths = source_paths
+        self.macro_paths = macro_paths
+        self.data_paths = data_paths
+        self.test_paths = test_paths
+        self.analysis_paths = analysis_paths
+        self.docs_paths = docs_paths
+        self.target_path = target_path
+        self.snapshot_paths = snapshot_paths
+        self.clean_targets = clean_targets
+        self.log_path = log_path
+        self.modules_path = modules_path
+        self.quoting = quoting
+        self.models = models
+        self.on_run_start = on_run_start
+        self.on_run_end = on_run_end
+        self.archive = archive
+        self.seeds = seeds
+        self.dbt_version = dbt_version
+        self.packages = packages
+
+    @staticmethod
+    def _preprocess(project_dict):
+        """Pre-process certain special keys to convert them from None values
+        into empty containers, and to turn strings into arrays of strings.
+        """
+        handlers = {
+            ('archive',): _list_if_none,
+            ('on-run-start',): _list_if_none_or_string,
+            ('on-run-end',): _list_if_none_or_string,
+        }
+
+        for k in ('models', 'seeds'):
+            handlers[(k,)] = _dict_if_none
+            handlers[(k, 'vars')] = _dict_if_none
+            handlers[(k, 'pre-hook')] = _list_if_none_or_string
+            handlers[(k, 'post-hook')] = _list_if_none_or_string
+        handlers[('seeds', 'column_types')] = _dict_if_none
+
+        def converter(value, keypath):
+            if keypath in handlers:
+                handler = handlers[keypath]
+                return handler(value)
+            else:
+                return value
+
+        return deep_map(converter, project_dict)
+
+    @classmethod
+    def from_project_config(cls, project_dict, packages_dict=None):
+        """Create a project from its project and package configuration, as read
+        by yaml.safe_load().
+
+        :param project_dict dict: The dictionary as read from disk
+        :param packages_dict Optional[dict]: If it exists, the packages file as
+            read from disk.
+        :raises DbtProjectError: If the project is missing or invalid, or if
+            the packages file exists and is invalid.
+        :returns Project: The project, with defaults populated.
+        """
+        try:
+            project_dict = cls._preprocess(project_dict)
+        except RecursionException:
+            raise DbtProjectError(
+                'Cycle detected: Project input has a reference to itself',
+                project=project_dict
+            )
+        # just for validation.
+        try:
+            ProjectContract(**project_dict)
+        except ValidationException as e:
+            raise DbtProjectError(str(e))
+
+        # name/version are required in the Project definition, so we can assume
+        # they are present
+        name = project_dict['name']
+        version = project_dict['version']
+        # this is added at project_dict parse time and should always be here
+        # once we see it.
+        project_root = project_dict['project-root']
+        # this is only optional in the sense that if it's not present, it needs
+        # to have been a cli argument.
+        profile_name = project_dict.get('profile')
+        # these are optional
+        source_paths = project_dict.get('source-paths', ['models'])
+        macro_paths = project_dict.get('macro-paths', ['macros'])
+        data_paths = project_dict.get('data-paths', ['data'])
+        test_paths = project_dict.get('test-paths', ['test'])
+        analysis_paths = project_dict.get('analysis-paths', [])
+        docs_paths = project_dict.get('docs-paths', source_paths[:])
+        target_path = project_dict.get('target-path', 'target')
+        snapshot_paths = project_dict.get('snapshot-paths', ['snapshots'])
+        # should this also include the modules path by default?
+        clean_targets = project_dict.get('clean-targets', [target_path])
+        log_path = project_dict.get('log-path', 'logs')
+        modules_path = project_dict.get('modules-path', 'dbt_modules')
+        # in the default case we'll populate this once we know the adapter type
+        quoting = project_dict.get('quoting', {})
+
+        models = project_dict.get('models', {})
+        on_run_start = project_dict.get('on-run-start', [])
+        on_run_end = project_dict.get('on-run-end', [])
+        archive = project_dict.get('archive', [])
+        seeds = project_dict.get('seeds', {})
+        dbt_raw_version = project_dict.get('require-dbt-version', '>=0.0.0')
+
+        try:
+            dbt_version = _parse_versions(dbt_raw_version)
+        except SemverException as e:
+            raise DbtProjectError(str(e))
+
+        packages = package_config_from_data(packages_dict)
+
+        project = cls(
+            project_name=name,
+            version=version,
+            project_root=project_root,
+            profile_name=profile_name,
+            source_paths=source_paths,
+            macro_paths=macro_paths,
+            data_paths=data_paths,
+            test_paths=test_paths,
+            analysis_paths=analysis_paths,
+            docs_paths=docs_paths,
+            target_path=target_path,
+            snapshot_paths=snapshot_paths,
+            clean_targets=clean_targets,
+            log_path=log_path,
+            modules_path=modules_path,
+            quoting=quoting,
+            models=models,
+            on_run_start=on_run_start,
+            on_run_end=on_run_end,
+            archive=archive,
+            seeds=seeds,
+            dbt_version=dbt_version,
+            packages=packages
+        )
+        # sanity check - this means an internal issue
+        project.validate()
+        return project
+
+    def __str__(self):
+        cfg = self.to_project_config(with_packages=True)
+        return str(cfg)
+
+    def __eq__(self, other):
+        if not (isinstance(other, self.__class__) and
+                isinstance(self, other.__class__)):
+            return False
+        return self.to_project_config(with_packages=True) == \
+            other.to_project_config(with_packages=True)
+
+    def to_project_config(self, with_packages=False):
+        """Return a dict representation of the config that could be written to
+        disk with `yaml.safe_dump` to get this configuration.
+
+        :param with_packages bool: If True, include the serialized packages
+            file in the root.
+        :returns dict: The serialized profile.
+        """
+        result = deepcopy({
+            'name': self.project_name,
+            'version': self.version,
+            'project-root': self.project_root,
+            'profile': self.profile_name,
+            'source-paths': self.source_paths,
+            'macro-paths': self.macro_paths,
+            'data-paths': self.data_paths,
+            'test-paths': self.test_paths,
+            'analysis-paths': self.analysis_paths,
+            'docs-paths': self.docs_paths,
+            'target-path': self.target_path,
+            'snapshot-paths': self.snapshot_paths,
+            'clean-targets': self.clean_targets,
+            'log-path': self.log_path,
+            'quoting': self.quoting,
+            'models': self.models,
+            'on-run-start': self.on_run_start,
+            'on-run-end': self.on_run_end,
+            'archive': self.archive,
+            'seeds': self.seeds,
+            'require-dbt-version': [
+                v.to_version_string() for v in self.dbt_version
+            ],
+        })
+        if with_packages:
+            result.update(self.packages.serialize())
+        return result
+
+    def validate(self):
+        try:
+            ProjectContract(**self.to_project_config())
+        except ValidationException as exc:
+            raise DbtProjectError(str(exc))
+
+    @classmethod
+    def from_project_root(cls, project_root, cli_vars):
+        """Create a project from a root directory. Reads in dbt_project.yml and
+        packages.yml, if it exists.
+
+        :param project_root str: The path to the project root to load.
+        :raises DbtProjectError: If the project is missing or invalid, or if
+            the packages file exists and is invalid.
+        :returns Project: The project, with defaults populated.
+        """
+        project_root = os.path.normpath(project_root)
+        project_yaml_filepath = os.path.join(project_root, 'dbt_project.yml')
+
+        # get the project.yml contents
+        if not path_exists(project_yaml_filepath):
+            raise DbtProjectError(
+                'no dbt_project.yml found at expected path {}'
+                .format(project_yaml_filepath)
+            )
+
+        if isinstance(cli_vars, compat.basestring):
+            cli_vars = parse_cli_vars(cli_vars)
+        renderer = ConfigRenderer(cli_vars)
+
+        project_dict = _load_yaml(project_yaml_filepath)
+        rendered_project = renderer.render_project(project_dict)
+        rendered_project['project-root'] = project_root
+        packages_dict = package_data_from_root(project_root)
+        return cls.from_project_config(rendered_project, packages_dict)
+
+    @classmethod
+    def from_current_directory(cls, cli_vars):
+        return cls.from_project_root(os.getcwd(), cli_vars)
+
+    @classmethod
+    def from_args(cls, args):
+        return cls.from_current_directory(getattr(args, 'vars', '{}'))
+
+    def hashed_name(self):
+        return hashlib.md5(self.project_name.encode('utf-8')).hexdigest()
+
+    def get_resource_config_paths(self):
+        """Return a dictionary with 'seeds' and 'models' keys whose values are
+        lists of lists of strings, where each inner list of strings represents
+        a configured path in the resource.
+        """
+        return {
+            'models': _get_config_paths(self.models),
+            'seeds': _get_config_paths(self.seeds),
+        }
+
+    def get_unused_resource_config_paths(self, resource_fqns, disabled):
+        """Return a list of lists of strings, where each inner list of strings
+        represents a type + FQN path of a resource configuration that is not
+        used.
+        """
+        disabled_fqns = frozenset(tuple(fqn) for fqn in disabled)
+        resource_config_paths = self.get_resource_config_paths()
+        unused_resource_config_paths = []
+        for resource_type, config_paths in resource_config_paths.items():
+            used_fqns = resource_fqns.get(resource_type, frozenset())
+            fqns = used_fqns | disabled_fqns
+
+            for config_path in config_paths:
+                if not _is_config_used(config_path, fqns):
+                    unused_resource_config_paths.append(
+                        (resource_type,) + config_path
+                    )
+        return unused_resource_config_paths
+
+    def warn_for_unused_resource_config_paths(self, resource_fqns, disabled):
+        unused = self.get_unused_resource_config_paths(resource_fqns, disabled)
+        if len(unused) == 0:
+            return
+
+        msg = UNUSED_RESOURCE_CONFIGURATION_PATH_MESSAGE.format(
+            len(unused),
+            '\n'.join('- {}'.format('.'.join(u)) for u in unused)
+        )
+        warn_or_error(msg, log_fmt=printer.yellow('{}'))
+
+    def validate_version(self):
+        """Ensure this package works with the installed version of dbt."""
+        installed = get_installed_version()
+        if not versions_compatible(*self.dbt_version):
+            msg = IMPOSSIBLE_VERSION_ERROR.format(
+                package=self.project_name,
+                version_spec=[
+                    x.to_version_string() for x in self.dbt_version
+                ]
+            )
+            raise DbtProjectError(msg)
+
+        if not versions_compatible(installed, *self.dbt_version):
+            msg = INVALID_VERSION_ERROR.format(
+                package=self.project_name,
+                installed=installed.to_version_string(),
+                version_spec=[
+                    x.to_version_string() for x in self.dbt_version
+                ]
+            )
+            raise DbtProjectError(msg)

core/dbt/config/renderer.py

@@ -0,0 +1,111 @@
+from dbt import compat
+from dbt.clients.jinja import get_rendered
+from dbt.context.common import generate_config_context
+from dbt.exceptions import DbtProfileError
+from dbt.exceptions import DbtProjectError
+from dbt.exceptions import RecursionException
+from dbt.utils import deep_map
+
+
+class ConfigRenderer(object):
+    """A renderer provides configuration rendering for a given set of cli
+    variables and a render type.
+    """
+    def __init__(self, cli_vars):
+        self.context = generate_config_context(cli_vars)
+
+    @staticmethod
+    def _is_hook_or_model_vars_path(keypath):
+        if not keypath:
+            return False
+
+        first = keypath[0]
+        # run hooks
+        if first in {'on-run-start', 'on-run-end'}:
+            return True
+        # models have two things to avoid
+        if first in {'seeds', 'models'}:
+            # model-level hooks
+            if 'pre-hook' in keypath or 'post-hook' in keypath:
+                return True
+            # model-level 'vars' declarations
+            if 'vars' in keypath:
+                return True
+
+        return False
+
+    def _render_project_entry(self, value, keypath):
+        """Render an entry, in case it's jinja. This is meant to be passed to
+        deep_map.
+
+        If the parsed entry is a string and has the name 'port', this will
+        attempt to cast it to an int, and on failure will return the parsed
+        string.
+
+        :param value Any: The value to potentially render
+        :param key str: The key to convert on.
+        :return Any: The rendered entry.
+        """
+        # hooks should be treated as raw sql, they'll get rendered later.
+        # Same goes for 'vars' declarations inside 'models'/'seeds'.
+        if self._is_hook_or_model_vars_path(keypath):
+            return value
+
+        return self.render_value(value)
+
+    def render_value(self, value, keypath=None):
+        # keypath is ignored.
+        # if it wasn't read as a string, ignore it
+        if not isinstance(value, compat.basestring):
+            return value
+        # force the result of rendering into this python version's native
+        # string type
+        return compat.to_native_string(get_rendered(value, self.context))
+
+    def _render_profile_data(self, value, keypath):
+        result = self.render_value(value)
+        if len(keypath) == 1 and keypath[-1] == 'port':
+            try:
+                result = int(result)
+            except ValueError:
+                # let the validator or connection handle this
+                pass
+        return result
+
+    def _render_schema_source_data(self, value, keypath):
+        # things to not render:
+        # - descriptions
+        if len(keypath) > 0 and keypath[-1] == 'description':
+            return value
+
+        return self.render_value(value)
+
+    def render_project(self, as_parsed):
+        """Render the parsed data, returning a new dict (or whatever was read).
+        """
+        try:
+            return deep_map(self._render_project_entry, as_parsed)
+        except RecursionException:
+            raise DbtProjectError(
+                'Cycle detected: Project input has a reference to itself',
+                project=as_parsed
+            )
+
+    def render_profile_data(self, as_parsed):
+        """Render the chosen profile entry, as it was parsed."""
+        try:
+            return deep_map(self._render_profile_data, as_parsed)
+        except RecursionException:
+            raise DbtProfileError(
+                'Cycle detected: Profile input has a reference to itself',
+                project=as_parsed
+            )
+
+    def render_schema_source(self, as_parsed):
+        try:
+            return deep_map(self._render_schema_source_data, as_parsed)
+        except RecursionException:
+            raise DbtProfileError(
+                'Cycle detected: schema.yml input has a reference to itself',
+                project=as_parsed
+            )

core/dbt/config/runtime.py

@@ -0,0 +1,205 @@
+from copy import deepcopy
+
+from dbt.utils import parse_cli_vars
+from dbt.contracts.project import Configuration
+from dbt.exceptions import DbtProjectError
+from dbt.exceptions import ValidationException
+from dbt.adapters.factory import get_relation_class_by_name
+
+from .profile import Profile
+from .project import Project
+
+
+_ARCHIVE_REMOVED_MESSAGE = '''
+The `archive` section in `dbt_project.yml` is no longer supported. Please use a
+`snapshot` block instead. For more information on snapshot blocks and a script
+to help migrate these archives, please consult the 0.14.0 migration guide:
+
+https://docs.getdbt.com/v0.14/docs/upgrading-to-014
+'''.strip()
+
+
+class RuntimeConfig(Project, Profile):
+    """The runtime configuration, as constructed from its components. There's a
+    lot because there is a lot of stuff!
+    """
+    def __init__(self, project_name, version, project_root, source_paths,
+                 macro_paths, data_paths, test_paths, analysis_paths,
+                 docs_paths, target_path, snapshot_paths, clean_targets,
+                 log_path, modules_path, quoting, models, on_run_start,
+                 on_run_end, archive, seeds, dbt_version, profile_name,
+                 target_name, config, threads, credentials, packages, args):
+        # 'vars'
+        self.args = args
+        self.cli_vars = parse_cli_vars(getattr(args, 'vars', '{}'))
+        # 'project'
+        Project.__init__(
+            self,
+            project_name=project_name,
+            version=version,
+            project_root=project_root,
+            profile_name=profile_name,
+            source_paths=source_paths,
+            macro_paths=macro_paths,
+            data_paths=data_paths,
+            test_paths=test_paths,
+            analysis_paths=analysis_paths,
+            docs_paths=docs_paths,
+            target_path=target_path,
+            snapshot_paths=snapshot_paths,
+            clean_targets=clean_targets,
+            log_path=log_path,
+            modules_path=modules_path,
+            quoting=quoting,
+            models=models,
+            on_run_start=on_run_start,
+            on_run_end=on_run_end,
+            archive=archive,
+            seeds=seeds,
+            dbt_version=dbt_version,
+            packages=packages
+        )
+        # 'profile'
+        Profile.__init__(
+            self,
+            profile_name=profile_name,
+            target_name=target_name,
+            config=config,
+            threads=threads,
+            credentials=credentials
+        )
+        self.validate()
+
+    @classmethod
+    def from_parts(cls, project, profile, args, allow_archive_configs=False):
+        """Instantiate a RuntimeConfig from its components.
+
+        :param profile Profile: A parsed dbt Profile.
+        :param project Project: A parsed dbt Project.
+        :param args argparse.Namespace: The parsed command-line arguments.
+        :param allow_archive_configs bool: If True, ignore archive blocks in
+            configs. This flag exists to enable archive migration.
+        :returns RuntimeConfig: The new configuration.
+        """
+        quoting = deepcopy(
+            get_relation_class_by_name(profile.credentials.type)
+            .DEFAULTS['quote_policy']
+        )
+        quoting.update(project.quoting)
+        if project.archive and not allow_archive_configs:
+            # if the user has an `archive` section, raise an error
+            raise DbtProjectError(_ARCHIVE_REMOVED_MESSAGE)
+
+        return cls(
+            project_name=project.project_name,
+            version=project.version,
+            project_root=project.project_root,
+            source_paths=project.source_paths,
+            macro_paths=project.macro_paths,
+            data_paths=project.data_paths,
+            test_paths=project.test_paths,
+            analysis_paths=project.analysis_paths,
+            docs_paths=project.docs_paths,
+            target_path=project.target_path,
+            snapshot_paths=project.snapshot_paths,
+            clean_targets=project.clean_targets,
+            log_path=project.log_path,
+            modules_path=project.modules_path,
+            quoting=quoting,
+            models=project.models,
+            on_run_start=project.on_run_start,
+            on_run_end=project.on_run_end,
+            archive=project.archive,
+            seeds=project.seeds,
+            dbt_version=project.dbt_version,
+            packages=project.packages,
+            profile_name=profile.profile_name,
+            target_name=profile.target_name,
+            config=profile.config,
+            threads=profile.threads,
+            credentials=profile.credentials,
+            args=args
+        )
+
+    def new_project(self, project_root):
+        """Given a new project root, read in its project dictionary, supply the
+        existing project's profile info, and create a new project file.
+
+        :param project_root str: A filepath to a dbt project.
+        :raises DbtProfileError: If the profile is invalid.
+        :raises DbtProjectError: If project is missing or invalid.
+        :returns RuntimeConfig: The new configuration.
+        """
+        # copy profile
+        profile = Profile(**self.to_profile_info())
+        profile.validate()
+        # load the new project and its packages. Don't pass cli variables.
+        project = Project.from_project_root(project_root, {})
+
+        cfg = self.from_parts(
+            project=project,
+            profile=profile,
+            args=deepcopy(self.args),
+        )
+        # force our quoting back onto the new project.
+        cfg.quoting = deepcopy(self.quoting)
+        return cfg
+
+    def serialize(self):
+        """Serialize the full configuration to a single dictionary. For any
+        instance that has passed validate() (which happens in __init__), it
+        matches the Configuration contract.
+
+        Note that args are not serialized.
+
+        :returns dict: The serialized configuration.
+        """
+        result = self.to_project_config(with_packages=True)
+        result.update(self.to_profile_info(serialize_credentials=True))
+        result['cli_vars'] = deepcopy(self.cli_vars)
+        return result
+
+    def __str__(self):
+        return str(self.serialize())
+
+    def validate(self):
+        """Validate the configuration against its contract.
+
+        :raises DbtProjectError: If the configuration fails validation.
+        """
+        try:
+            Configuration(**self.serialize())
+        except ValidationException as e:
+            raise DbtProjectError(str(e))
+
+        if getattr(self.args, 'version_check', False):
+            self.validate_version()
+
+    @classmethod
+    def from_args(cls, args, allow_archive_configs=False):
+        """Given arguments, read in dbt_project.yml from the current directory,
+        read in packages.yml if it exists, and use them to find the profile to
+        load.
+
+        :param args argparse.Namespace: The arguments as parsed from the cli.
+        :param allow_archive_configs bool: If True, ignore archive blocks in
+            configs. This flag exists to enable archive migration.
+        :raises DbtProjectError: If the project is invalid or missing.
+        :raises DbtProfileError: If the profile is invalid or missing.
+        :raises ValidationException: If the cli variables are invalid.
+        """
+        # build the project and read in packages.yml
+        project = Project.from_args(args)
+
+        # build the profile
+        profile = Profile.from_args(
+            args=args,
+            project_profile_name=project.profile_name
+        )
+
+        return cls.from_parts(
+            project=project,
+            profile=profile,
+            args=args,
+            allow_archive_configs=allow_archive_configs
+        )

core/dbt/context/common.py

@@ -0,0 +1,496 @@
+import json
+import os
+
+from dbt.adapters.factory import get_adapter
+from dbt.compat import basestring
+from dbt.node_types import NodeType
+from dbt.contracts.graph.parsed import ParsedMacro, ParsedNode
+from dbt.include.global_project import PACKAGES
+from dbt.include.global_project import PROJECT_NAME as GLOBAL_PROJECT_NAME
+
+import dbt.clients.jinja
+import dbt.clients.agate_helper
+import dbt.flags
+import dbt.tracking
+import dbt.writer
+import dbt.utils
+
+from dbt.logger import GLOBAL_LOGGER as logger  # noqa
+
+
+# These modules are added to the context. Consider alternative
+# approaches which will extend well to potentially many modules
+import pytz
+import datetime
+
+
+class RelationProxy(object):
+    def __init__(self, adapter):
+        self.quoting_config = adapter.config.quoting
+        self.relation_type = adapter.Relation
+
+    def __getattr__(self, key):
+        return getattr(self.relation_type, key)
+
+    def create_from_source(self, *args, **kwargs):
+        # bypass our create when creating from source so as not to mess up
+        # the source quoting
+        return self.relation_type.create_from_source(*args, **kwargs)
+
+    def create(self, *args, **kwargs):
+        kwargs['quote_policy'] = dbt.utils.merge(
+            self.quoting_config,
+            kwargs.pop('quote_policy', {})
+        )
+        return self.relation_type.create(*args, **kwargs)
+
+
+class BaseDatabaseWrapper(object):
+    """
+    Wrapper for runtime database interaction. Applies the runtime quote policy
+    via a relation proxy.
+    """
+    def __init__(self, adapter):
+        self.adapter = adapter
+        self.Relation = RelationProxy(adapter)
+
+    def __getattr__(self, name):
+        raise NotImplementedError('subclasses need to implement this')
+
+    @property
+    def config(self):
+        return self.adapter.config
+
+    def type(self):
+        return self.adapter.type()
+
+    def commit(self):
+        return self.adapter.commit_if_has_connection()
+
+
+class BaseResolver(object):
+    def __init__(self, db_wrapper, model, config, manifest):
+        self.db_wrapper = db_wrapper
+        self.model = model
+        self.config = config
+        self.manifest = manifest
+
+    @property
+    def current_project(self):
+        return self.config.project_name
+
+    @property
+    def Relation(self):
+        return self.db_wrapper.Relation
+
+
+def _add_macro_map(context, package_name, macro_map):
+    """Update an existing context in-place, adding the given macro map to the
+    appropriate package namespace. Adapter packages get inserted into the
+    global namespace.
+    """
+    key = package_name
+    if package_name in PACKAGES:
+        key = GLOBAL_PROJECT_NAME
+    if key not in context:
+        context[key] = {}
+
+    context[key].update(macro_map)
+
+
+def _add_macros(context, model, manifest):
+    macros_to_add = {'global': [], 'local': []}
+
+    for unique_id, macro in manifest.macros.items():
+        if macro.resource_type != NodeType.Macro:
+            continue
+        package_name = macro.package_name
+
+        macro_map = {
+            macro.name: macro.generator(context)
+        }
+
+        # adapter packages are part of the global project space
+        _add_macro_map(context, package_name, macro_map)
+
+        if package_name == model.package_name:
+            macros_to_add['local'].append(macro_map)
+        elif package_name in PACKAGES:
+            macros_to_add['global'].append(macro_map)
+
+    # Load global macros before local macros -- local takes precedence
+    unprefixed_macros = macros_to_add['global'] + macros_to_add['local']
+    for macro_map in unprefixed_macros:
+        context.update(macro_map)
+
+    return context
+
+
+def _add_tracking(context):
+    if dbt.tracking.active_user is not None:
+        context = dbt.utils.merge(context, {
+            "run_started_at": dbt.tracking.active_user.run_started_at,
+            "invocation_id": dbt.tracking.active_user.invocation_id,
+        })
+    else:
+        context = dbt.utils.merge(context, {
+            "run_started_at": None,
+            "invocation_id": None
+        })
+
+    return context
+
+
+def _add_validation(context):
+    def validate_any(*args):
+        def inner(value):
+            for arg in args:
+                if isinstance(arg, type) and isinstance(value, arg):
+                    return
+                elif value == arg:
+                    return
+            raise dbt.exceptions.ValidationException(
+                'Expected value "{}" to be one of {}'
+                .format(value, ','.join(map(str, args))))
+        return inner
+
+    validation_utils = dbt.utils.AttrDict({
+        'any': validate_any,
+    })
+
+    return dbt.utils.merge(
+        context,
+        {'validation': validation_utils})
+
+
+def env_var(var, default=None):
+    if var in os.environ:
+        return os.environ[var]
+    elif default is not None:
+        return default
+    else:
+        msg = "Env var required but not provided: '{}'".format(var)
+        dbt.clients.jinja.undefined_error(msg)
+
+
+def _store_result(sql_results):
+    def call(name, status, agate_table=None):
+        if agate_table is None:
+            agate_table = dbt.clients.agate_helper.empty_table()
+
+        sql_results[name] = dbt.utils.AttrDict({
+            'status': status,
+            'data': dbt.clients.agate_helper.as_matrix(agate_table),
+            'table': agate_table
+        })
+        return ''
+
+    return call
+
+
+def _load_result(sql_results):
+    def call(name):
+        return sql_results.get(name)
+
+    return call
+
+
+def _debug_here():
+    import sys
+    import ipdb
+    frame = sys._getframe(3)
+    ipdb.set_trace(frame)
+
+
+def _add_sql_handlers(context):
+    sql_results = {}
+    return dbt.utils.merge(context, {
+        '_sql_results': sql_results,
+        'store_result': _store_result(sql_results),
+        'load_result': _load_result(sql_results),
+    })
+
+
+def log(msg, info=False):
+    if info:
+        logger.info(msg)
+    else:
+        logger.debug(msg)
+    return ''
+
+
+class Var(object):
+    UndefinedVarError = "Required var '{}' not found in config:\nVars "\
+                        "supplied to {} = {}"
+    _VAR_NOTSET = object()
+
+    def __init__(self, model, context, overrides):
+        self.model = model
+        self.context = context
+
+        # These are hard-overrides (eg. CLI vars) that should take
+        # precedence over context-based var definitions
+        self.overrides = overrides
+
+        if isinstance(model, dict) and model.get('unique_id'):
+            local_vars = model.get('config', {}).get('vars', {})
+            self.model_name = model.get('name')
+        elif isinstance(model, ParsedMacro):
+            local_vars = {}  # macros have no config
+            self.model_name = model.name
+        elif isinstance(model, ParsedNode):
+            local_vars = model.config.get('vars', {})
+            self.model_name = model.name
+        elif model is None:
+            # during config parsing we have no model and no local vars
+            self.model_name = '<Configuration>'
+            local_vars = {}
+        else:
+            # still used for wrapping
+            self.model_name = model.nice_name
+            local_vars = model.config.get('vars', {})
+
+        self.local_vars = dbt.utils.merge(local_vars, overrides)
+
+    def pretty_dict(self, data):
+        return json.dumps(data, sort_keys=True, indent=4)
+
+    def get_missing_var(self, var_name):
+        pretty_vars = self.pretty_dict(self.local_vars)
+        msg = self.UndefinedVarError.format(
+            var_name, self.model_name, pretty_vars
+        )
+        dbt.exceptions.raise_compiler_error(msg, self.model)
+
+    def assert_var_defined(self, var_name, default):
+        if var_name not in self.local_vars and default is self._VAR_NOTSET:
+            return self.get_missing_var(var_name)
+
+    def get_rendered_var(self, var_name):
+        raw = self.local_vars[var_name]
+        # if bool/int/float/etc are passed in, don't compile anything
+        if not isinstance(raw, basestring):
+            return raw
+
+        return dbt.clients.jinja.get_rendered(raw, self.context)
+
+    def __call__(self, var_name, default=_VAR_NOTSET):
+        if var_name in self.local_vars:
+            return self.get_rendered_var(var_name)
+        elif default is not self._VAR_NOTSET:
+            return default
+        else:
+            return self.get_missing_var(var_name)
+
+
+def write(node, target_path, subdirectory):
+    def fn(payload):
+        node['build_path'] = dbt.writer.write_node(
+            node, target_path, subdirectory, payload)
+        return ''
+
+    return fn
+
+
+def render(context, node):
+    def fn(string):
+        return dbt.clients.jinja.get_rendered(string, context, node)
+
+    return fn
+
+
+def fromjson(string, default=None):
+    try:
+        return json.loads(string)
+    except ValueError:
+        return default
+
+
+def tojson(value, default=None):
+    try:
+        return json.dumps(value)
+    except ValueError:
+        return default
+
+
+def try_or_compiler_error(model):
+    def impl(message_if_exception, func, *args, **kwargs):
+        try:
+            return func(*args, **kwargs)
+        except Exception:
+            dbt.exceptions.raise_compiler_error(message_if_exception, model)
+    return impl
+
+
+def _return(value):
+    raise dbt.exceptions.MacroReturn(value)
+
+
+def get_this_relation(db_wrapper, config, model):
+    return db_wrapper.Relation.create_from_node(config, model)
+
+
+def get_pytz_module_context():
+    context_exports = pytz.__all__
+
+    return {
+        name: getattr(pytz, name) for name in context_exports
+    }
+
+
+def get_datetime_module_context():
+    context_exports = [
+        'date',
+        'datetime',
+        'time',
+        'timedelta',
+        'tzinfo'
+    ]
+
+    return {
+        name: getattr(datetime, name) for name in context_exports
+    }
+
+
+def get_context_modules():
+    return {
+        'pytz': get_pytz_module_context(),
+        'datetime': get_datetime_module_context(),
+    }
+
+
+def generate_config_context(cli_vars):
+    context = {
+        'env_var': env_var,
+        'modules': get_context_modules(),
+    }
+    context['var'] = Var(None, context, cli_vars)
+    return _add_tracking(context)
+
+
+def generate_base(model, model_dict, config, manifest, source_config,
+                  provider, adapter=None):
+    """Generate the common aspects of the config dict."""
+    if provider is None:
+        raise dbt.exceptions.InternalException(
+            "Invalid provider given to context: {}".format(provider))
+
+    target_name = config.target_name
+    target = config.to_profile_info()
+    del target['credentials']
+    target.update(config.credentials.serialize(with_aliases=True))
+    target['type'] = config.credentials.type
+    target.pop('pass', None)
+    target['name'] = target_name
+
+    adapter = get_adapter(config)
+
+    context = {'env': target}
+
+    pre_hooks = None
+    post_hooks = None
+
+    db_wrapper = provider.DatabaseWrapper(adapter)
+
+    context = dbt.utils.merge(context, {
+        "adapter": db_wrapper,
+        "api": {
+            "Relation": db_wrapper.Relation,
+            "Column": adapter.Column,
+        },
+        "column": adapter.Column,
+        "config": provider.Config(model_dict, source_config),
+        "database": config.credentials.database,
+        "env_var": env_var,
+        "exceptions": dbt.exceptions.wrapped_exports(model),
+        "execute": provider.execute,
+        "flags": dbt.flags,
+        "graph": manifest.flat_graph,
+        "log": log,
+        "model": model_dict,
+        "modules": get_context_modules(),
+        "post_hooks": post_hooks,
+        "pre_hooks": pre_hooks,
+        "ref": provider.ref(db_wrapper, model, config, manifest),
+        "return": _return,
+        "schema": config.credentials.schema,
+        "sql": None,
+        "sql_now": adapter.date_function(),
+        "source": provider.source(db_wrapper, model, config, manifest),
+        "fromjson": fromjson,
+        "tojson": tojson,
+        "target": target,
+        "try_or_compiler_error": try_or_compiler_error(model)
+    })
+    if os.environ.get('DBT_MACRO_DEBUGGING'):
+        context['debug'] = _debug_here
+
+    return context
+
+
+def modify_generated_context(context, model, model_dict, config, manifest,
+                             provider):
+    cli_var_overrides = config.cli_vars
+
+    context = _add_tracking(context)
+    context = _add_validation(context)
+    context = _add_sql_handlers(context)
+
+    # we make a copy of the context for each of these ^^
+
+    context = _add_macros(context, model, manifest)
+
+    context["write"] = write(model_dict, config.target_path, 'run')
+    context["render"] = render(context, model_dict)
+    context["var"] = provider.Var(model, context=context,
+                                  overrides=cli_var_overrides)
+    context['context'] = context
+
+    return context
+
+
+def generate_execute_macro(model, config, manifest, provider):
+    """Internally, macros can be executed like nodes, with some restrictions:
+
+     - they don't have have all values available that nodes do:
+        - 'this', 'pre_hooks', 'post_hooks', and 'sql' are missing
+        - 'schema' does not use any 'model' information
+     - they can't be configured with config() directives
+    """
+    model_dict = model.serialize()
+    context = generate_base(model, model_dict, config, manifest, None,
+                            provider)
+
+    return modify_generated_context(context, model, model_dict, config,
+                                    manifest, provider)
+
+
+def generate_model(model, config, manifest, source_config, provider):
+    model_dict = model.to_dict()
+    context = generate_base(model, model_dict, config, manifest,
+                            source_config, provider)
+    # operations (hooks) don't get a 'this'
+    if model.resource_type != NodeType.Operation:
+        this = get_this_relation(context['adapter'], config, model_dict)
+        context['this'] = this
+    # overwrite schema/database if we have them, and hooks + sql
+    context.update({
+        'schema': model.get('schema', context['schema']),
+        'database': model.get('database', context['database']),
+        'pre_hooks': model.config.get('pre-hook'),
+        'post_hooks': model.config.get('post-hook'),
+        'sql': model.get('injected_sql'),
+    })
+
+    return modify_generated_context(context, model, model_dict, config,
+                                    manifest, provider)
+
+
+def generate(model, config, manifest, source_config=None, provider=None):
+    """
+    Not meant to be called directly. Call with either:
+        dbt.context.parser.generate
+    or
+        dbt.context.runtime.generate
+    """
+    return generate_model(model, config, manifest, source_config, provider)

core/dbt/context/operation.py

@@ -0,0 +1,29 @@
+import dbt.context.common
+from dbt.context import runtime
+from dbt.exceptions import raise_compiler_error
+
+
+class RefResolver(runtime.BaseRefResolver):
+    def __call__(self, *args):
+        # When you call ref(), this is what happens at operation runtime
+        target_model, name = self.resolve(args)
+        return self.create_relation(target_model, name)
+
+    def create_ephemeral_relation(self, target_model, name):
+        # In operations, we can't ref() ephemeral nodes, because ParsedMacros
+        # do not support set_cte
+        raise_compiler_error(
+            'Operations can not ref() ephemeral nodes, but {} is ephemeral'
+            .format(target_model.name),
+            self.model
+        )
+
+
+class Provider(runtime.Provider):
+    ref = RefResolver
+
+
+def generate(model, runtime_config, manifest):
+    return dbt.context.common.generate_execute_macro(
+        model, runtime_config, manifest, Provider()
+    )

core/dbt/context/parser.py

@@ -0,0 +1,146 @@
+import dbt.exceptions
+
+import dbt.context.common
+from dbt.adapters.factory import get_adapter
+
+
+def docs(unparsed, docrefs, column_name=None):
+
+    def do_docs(*args):
+        if len(args) != 1 and len(args) != 2:
+            dbt.exceptions.doc_invalid_args(unparsed, args)
+        doc_package_name = ''
+        doc_name = args[0]
+        if len(args) == 2:
+            doc_package_name = args[1]
+
+        docref = {
+            'documentation_package': doc_package_name,
+            'documentation_name': doc_name,
+        }
+        if column_name is not None:
+            docref['column_name'] = column_name
+
+        docrefs.append(docref)
+
+        # IDK
+        return True
+
+    return do_docs
+
+
+class Config(object):
+    def __init__(self, model, source_config):
+        self.model = model
+        self.source_config = source_config
+
+    def _transform_config(self, config):
+        for oldkey in ('pre_hook', 'post_hook'):
+            if oldkey in config:
+                newkey = oldkey.replace('_', '-')
+                if newkey in config:
+                    dbt.exceptions.raise_compiler_error(
+                        'Invalid config, has conflicting keys "{}" and "{}"'
+                        .format(oldkey, newkey),
+                        self.model
+                    )
+                config[newkey] = config.pop(oldkey)
+        return config
+
+    def __call__(self, *args, **kwargs):
+        if len(args) == 1 and len(kwargs) == 0:
+            opts = args[0]
+        elif len(args) == 0 and len(kwargs) > 0:
+            opts = kwargs
+        else:
+            dbt.exceptions.raise_compiler_error(
+                "Invalid inline model config",
+                self.model)
+
+        opts = self._transform_config(opts)
+
+        self.source_config.update_in_model_config(opts)
+        return ''
+
+    def set(self, name, value):
+        return self.__call__({name: value})
+
+    def require(self, name, validator=None):
+        return ''
+
+    def get(self, name, validator=None, default=None):
+        return ''
+
+
+class DatabaseWrapper(dbt.context.common.BaseDatabaseWrapper):
+    """The parser subclass of the database wrapper applies any explicit
+    parse-time overrides.
+    """
+    def __getattr__(self, name):
+        override = (name in self.adapter._available_ and
+                    name in self.adapter._parse_replacements_)
+
+        if override:
+            return self.adapter._parse_replacements_[name]
+        elif name in self.adapter._available_:
+            return getattr(self.adapter, name)
+        else:
+            raise AttributeError(
+                "'{}' object has no attribute '{}'".format(
+                    self.__class__.__name__, name
+                )
+            )
+
+
+class Var(dbt.context.common.Var):
+    def get_missing_var(self, var_name):
+        # in the parser, just always return None.
+        return None
+
+
+class RefResolver(dbt.context.common.BaseResolver):
+    def __call__(self, *args):
+        # When you call ref(), this is what happens at parse time
+        if len(args) == 1 or len(args) == 2:
+            self.model.refs.append(list(args))
+
+        else:
+            dbt.exceptions.ref_invalid_args(self.model, args)
+
+        return self.Relation.create_from_node(self.config, self.model)
+
+
+class SourceResolver(dbt.context.common.BaseResolver):
+    def __call__(self, source_name, table_name):
+        # When you call source(), this is what happens at parse time
+        self.model.sources.append([source_name, table_name])
+        return self.Relation.create_from_node(self.config, self.model)
+
+
+class Provider(object):
+    execute = False
+    Config = Config
+    DatabaseWrapper = DatabaseWrapper
+    Var = Var
+    ref = RefResolver
+    source = SourceResolver
+
+
+def generate(model, runtime_config, manifest, source_config):
+    # during parsing, we don't have a connection, but we might need one, so we
+    # have to acquire it.
+    # In the future, it would be nice to lazily open the connection, as in some
+    # projects it would be possible to parse without connecting to the db
+    with get_adapter(runtime_config).connection_named(model.get('name')):
+        return dbt.context.common.generate(
+            model, runtime_config, manifest, source_config, Provider()
+        )
+
+
+def generate_macro(model, runtime_config, manifest):
+    # parser.generate_macro is called by the get_${attr}_func family of Parser
+    # methods, which preparse and cache the generate_${attr}_name family of
+    # macros for use during parsing
+    return dbt.context.common.generate_execute_macro(
+        model, runtime_config, manifest, Provider()
+    )

core/dbt/context/runtime.py

@@ -0,0 +1,150 @@
+from dbt.utils import get_materialization, add_ephemeral_model_prefix
+
+import dbt.clients.jinja
+import dbt.context.common
+import dbt.flags
+from dbt.parser import ParserUtils
+
+from dbt.logger import GLOBAL_LOGGER as logger  # noqa
+
+
+class BaseRefResolver(dbt.context.common.BaseResolver):
+    def resolve(self, args):
+        name = None
+        package = None
+
+        if len(args) == 1:
+            name = args[0]
+        elif len(args) == 2:
+            package, name = args
+        else:
+            dbt.exceptions.ref_invalid_args(self.model, args)
+
+        target_model = ParserUtils.resolve_ref(
+            self.manifest,
+            name,
+            package,
+            self.current_project,
+            self.model.package_name)
+
+        if target_model is None or target_model is ParserUtils.DISABLED:
+            dbt.exceptions.ref_target_not_found(
+                self.model,
+                name,
+                package)
+        return target_model, name
+
+    def create_ephemeral_relation(self, target_model, name):
+        self.model.set_cte(target_model.unique_id, None)
+        return self.Relation.create(
+            type=self.Relation.CTE,
+            identifier=add_ephemeral_model_prefix(name)
+        ).quote(identifier=False)
+
+    def create_relation(self, target_model, name):
+        if get_materialization(target_model) == 'ephemeral':
+            return self.create_ephemeral_relation(target_model, name)
+        else:
+            return self.Relation.create_from_node(self.config, target_model)
+
+
+class RefResolver(BaseRefResolver):
+    def validate(self, resolved, args):
+        if resolved.unique_id not in self.model.depends_on.get('nodes'):
+            dbt.exceptions.ref_bad_context(self.model, args)
+
+    def __call__(self, *args):
+        # When you call ref(), this is what happens at runtime
+        target_model, name = self.resolve(args)
+        self.validate(target_model, args)
+        return self.create_relation(target_model, name)
+
+
+class SourceResolver(dbt.context.common.BaseResolver):
+    def resolve(self, source_name, table_name):
+        target_source = ParserUtils.resolve_source(
+            self.manifest,
+            source_name,
+            table_name,
+            self.current_project,
+            self.model.package_name
+        )
+
+        if target_source is None:
+            dbt.exceptions.source_target_not_found(
+                self.model,
+                source_name,
+                table_name)
+        return target_source
+
+    def __call__(self, source_name, table_name):
+        """When you call source(), this is what happens at runtime"""
+        target_source = self.resolve(source_name, table_name)
+        return self.Relation.create_from_source(target_source)
+
+
+class Config:
+    def __init__(self, model, source_config=None):
+        self.model = model
+        # we never use or get a source config, only the parser cares
+
+    def __call__(*args, **kwargs):
+        return ''
+
+    def set(self, name, value):
+        return self.__call__({name: value})
+
+    def _validate(self, validator, value):
+        validator(value)
+
+    def require(self, name, validator=None):
+        if name not in self.model['config']:
+            dbt.exceptions.missing_config(self.model, name)
+
+        to_return = self.model['config'][name]
+
+        if validator is not None:
+            self._validate(validator, to_return)
+
+        return to_return
+
+    def get(self, name, validator=None, default=None):
+        to_return = self.model['config'].get(name, default)
+
+        if validator is not None and default is not None:
+            self._validate(validator, to_return)
+
+        return to_return
+
+
+class DatabaseWrapper(dbt.context.common.BaseDatabaseWrapper):
+    """The runtime database wrapper exposes everything the adapter marks
+    available.
+    """
+    def __getattr__(self, name):
+        if name in self.adapter._available_:
+            return getattr(self.adapter, name)
+        else:
+            raise AttributeError(
+                "'{}' object has no attribute '{}'".format(
+                    self.__class__.__name__, name
+                )
+            )
+
+
+class Var(dbt.context.common.Var):
+    pass
+
+
+class Provider(object):
+    execute = True
+    Config = Config
+    DatabaseWrapper = DatabaseWrapper
+    Var = Var
+    ref = RefResolver
+    source = SourceResolver
+
+
+def generate(model, runtime_config, manifest):
+    return dbt.context.common.generate(
+        model, runtime_config, manifest, None, Provider())

core/dbt/contracts/common.py

@@ -0,0 +1,11 @@
+
+
+def named_property(name, doc=None):
+    def get_prop(self):
+        return self._contents.get(name)
+
+    def set_prop(self, value):
+        self._contents[name] = value
+        self.validate()
+
+    return property(get_prop, set_prop, doc=doc)

core/dbt/contracts/connection.py

@@ -0,0 +1,71 @@
+from dbt.api.object import APIObject
+from dbt.contracts.common import named_property
+
+
+CONNECTION_CONTRACT = {
+    'type': 'object',
+    'additionalProperties': False,
+    'properties': {
+        'type': {
+            'type': 'string',
+            # valid python identifiers only
+            'pattern': r'^[A-Za-z_][A-Za-z0-9_]+$',
+        },
+        'name': {
+            'type': ['null', 'string'],
+        },
+        'state': {
+            'enum': ['init', 'open', 'closed', 'fail'],
+        },
+        'transaction_open': {
+            'type': 'boolean',
+        },
+        # we can't serialize this so we can't require it as part of the
+        # contract.
+        # 'handle': {
+        #     'type': ['null', 'object'],
+        # },
+        # credentials are validated separately by the adapter packages
+        'credentials': {
+            'description': (
+                'The credentials object here should match the connection type.'
+            ),
+            'type': 'object',
+            'additionalProperties': True,
+        }
+    },
+    'required': [
+        'type', 'name', 'state', 'transaction_open', 'credentials'
+    ],
+}
+
+
+class Connection(APIObject):
+    SCHEMA = CONNECTION_CONTRACT
+
+    def __init__(self, credentials, *args, **kwargs):
+        # we can't serialize handles
+        self._handle = kwargs.pop('handle')
+        super(Connection, self).__init__(credentials=credentials.serialize(),
+                                         *args, **kwargs)
+        # this will validate itself in its own __init__.
+        self._credentials = credentials
+
+    @property
+    def credentials(self):
+        return self._credentials
+
+    @property
+    def handle(self):
+        return self._handle
+
+    @handle.setter
+    def handle(self, value):
+        self._handle = value
+
+    name = named_property('name', 'The name of this connection')
+    state = named_property('state', 'The state of the connection')
+    transaction_open = named_property(
+        'transaction_open',
+        'True if there is an open transaction, False otherwise.'
+    )

core/dbt/contracts/graph/compiled.py

@@ -0,0 +1,236 @@
+from dbt.api import APIObject
+from dbt.utils import deep_merge
+from dbt.contracts.graph.parsed import PARSED_NODE_CONTRACT, \
+    PARSED_MACRO_CONTRACT, ParsedNode
+
+import dbt.compat
+
+import sqlparse
+
+INJECTED_CTE_CONTRACT = {
+    'type': 'object',
+    'additionalProperties': False,
+    'description': 'A single entry in the CTEs list',
+    'properties': {
+        'id': {
+            'type': 'string',
+            'description': 'The id of the CTE',
+        },
+        'sql': {
+            'type': ['string', 'null'],
+            'description': 'The compiled SQL of the CTE',
+            'additionalProperties': True,
+        },
+    },
+    'required': ['id', 'sql'],
+}
+
+
+COMPILED_NODE_CONTRACT = deep_merge(
+    PARSED_NODE_CONTRACT,
+    {
+        # TODO: when we add 'extra_ctes' back in, flip this back to False
+        'additionalProperties': True,
+        'properties': {
+            'compiled': {
+                'description': (
+                    'This is true after the node has been compiled, but ctes '
+                    'have not necessarily been injected into the node.'
+                ),
+                'type': 'boolean'
+            },
+            'compiled_sql': {
+                'type': ['string', 'null'],
+            },
+            'extra_ctes_injected': {
+                'description': (
+                    'This is true after extra ctes have been injected into '
+                    'the compiled node.'
+                ),
+                'type': 'boolean',
+            },
+            'extra_ctes': {
+                'type': 'array',
+                'description': 'The injected CTEs for a model',
+                'items': INJECTED_CTE_CONTRACT,
+            },
+            'injected_sql': {
+                'type': ['string', 'null'],
+                'description': 'The SQL after CTEs have been injected',
+            },
+            'wrapped_sql': {
+                'type': ['string', 'null'],
+                'description': (
+                    'The SQL after it has been wrapped (for tests, '
+                    'operations, and analysis)'
+                ),
+            },
+        },
+        'required': PARSED_NODE_CONTRACT['required'] + [
+            'compiled', 'compiled_sql', 'extra_ctes_injected',
+            'injected_sql', 'extra_ctes'
+        ]
+    }
+)
+
+
+COMPILED_NODES_CONTRACT = {
+    'type': 'object',
+    'additionalProperties': False,
+    'description': (
+        'A collection of the compiled nodes, stored by their unique IDs.'
+    ),
+    'patternProperties': {
+        '.*': COMPILED_NODE_CONTRACT
+    },
+}
+
+
+COMPILED_MACRO_CONTRACT = PARSED_MACRO_CONTRACT
+
+
+COMPILED_MACROS_CONTRACT = {
+    'type': 'object',
+    'additionalProperties': False,
+    'description': (
+        'A collection of the compiled macros, stored by their unique IDs.'
+    ),
+    'patternProperties': {
+        '.*': COMPILED_MACRO_CONTRACT
+    },
+}
+
+
+COMPILED_GRAPH_CONTRACT = {
+    'type': 'object',
+    'additionalProperties': False,
+    'description': (
+        'The full compiled graph, with both the required nodes and required '
+        'macros.'
+    ),
+    'properties': {
+        'nodes': COMPILED_NODES_CONTRACT,
+        'macros': COMPILED_MACROS_CONTRACT,
+    },
+    'required': ['nodes', 'macros'],
+}
+
+
+def _inject_ctes_into_sql(sql, ctes):
+    """
+    `ctes` is a dict of CTEs in the form:
+
+      {
+        "cte_id_1": "__dbt__CTE__ephemeral as (select * from table)",
+        "cte_id_2": "__dbt__CTE__events as (select id, type from events)"
+      }
+
+    Given `sql` like:
+
+      "with internal_cte as (select * from sessions)
+       select * from internal_cte"
+
+    This will spit out:
+
+      "with __dbt__CTE__ephemeral as (select * from table),
+            __dbt__CTE__events as (select id, type from events),
+            with internal_cte as (select * from sessions)
+       select * from internal_cte"
+
+    (Whitespace enhanced for readability.)
+    """
+    if len(ctes) == 0:
+        return sql
+
+    parsed_stmts = sqlparse.parse(sql)
+    parsed = parsed_stmts[0]
+
+    with_stmt = None
+    for token in parsed.tokens:
+        if token.is_keyword and token.normalized == 'WITH':
+            with_stmt = token
+            break
+
+    if with_stmt is None:
+        # no with stmt, add one, and inject CTEs right at the beginning
+        first_token = parsed.token_first()
+        with_stmt = sqlparse.sql.Token(sqlparse.tokens.Keyword, 'with')
+        parsed.insert_before(first_token, with_stmt)
+    else:
+        # stmt exists, add a comma (which will come after injected CTEs)
+        trailing_comma = sqlparse.sql.Token(sqlparse.tokens.Punctuation, ',')
+        parsed.insert_after(with_stmt, trailing_comma)
+
+    token = sqlparse.sql.Token(
+        sqlparse.tokens.Keyword,
+        ", ".join(c['sql'] for c in ctes)
+    )
+    parsed.insert_after(with_stmt, token)
+
+    return dbt.compat.to_string(parsed)
+
+
+class CompiledNode(ParsedNode):
+    SCHEMA = COMPILED_NODE_CONTRACT
+
+    def prepend_ctes(self, prepended_ctes):
+        self._contents['extra_ctes_injected'] = True
+        self._contents['extra_ctes'] = prepended_ctes
+        self._contents['injected_sql'] = _inject_ctes_into_sql(
+            self.compiled_sql,
+            prepended_ctes
+        )
+        self.validate()
+
+    @property
+    def extra_ctes_injected(self):
+        return self._contents.get('extra_ctes_injected')
+
+    @property
+    def extra_ctes(self):
+        return self._contents.get('extra_ctes')
+
+    @property
+    def compiled(self):
+        return self._contents.get('compiled')
+
+    @compiled.setter
+    def compiled(self, value):
+        self._contents['compiled'] = value
+
+    @property
+    def injected_sql(self):
+        return self._contents.get('injected_sql')
+
+    @property
+    def compiled_sql(self):
+        return self._contents.get('compiled_sql')
+
+    @compiled_sql.setter
+    def compiled_sql(self, value):
+        self._contents['compiled_sql'] = value
+
+    @property
+    def wrapped_sql(self):
+        return self._contents.get('wrapped_sql')
+
+    @wrapped_sql.setter
+    def wrapped_sql(self, value):
+        self._contents['wrapped_sql'] = value
+
+    def set_cte(self, cte_id, sql):
+        """This is the equivalent of what self.extra_ctes[cte_id] = sql would
+        do if extra_ctes were an OrderedDict
+        """
+        for cte in self.extra_ctes:
+            if cte['id'] == cte_id:
+                cte['sql'] = sql
+                break
+        else:
+            self.extra_ctes.append(
+                {'id': cte_id, 'sql': sql}
+            )
+
+
+class CompiledGraph(APIObject):
+    SCHEMA = COMPILED_GRAPH_CONTRACT

core/dbt/contracts/graph/manifest.py

@@ -0,0 +1,418 @@
+from dbt.api import APIObject
+from dbt.contracts.graph.parsed import PARSED_NODE_CONTRACT, \
+    PARSED_MACRO_CONTRACT, PARSED_DOCUMENTATION_CONTRACT, \
+    PARSED_SOURCE_DEFINITION_CONTRACT
+from dbt.contracts.graph.compiled import COMPILED_NODE_CONTRACT, CompiledNode
+from dbt.exceptions import raise_duplicate_resource_name
+from dbt.node_types import NodeType
+from dbt.logger import GLOBAL_LOGGER as logger
+from dbt import tracking
+import dbt.utils
+
+# We allow either parsed or compiled nodes, or parsed sources, as some
+# 'compile()' calls in the runner actually just return the original parsed
+# node they were given.
+COMPILE_RESULT_NODE_CONTRACT = {
+    'anyOf': [
+        PARSED_NODE_CONTRACT,
+        COMPILED_NODE_CONTRACT,
+        PARSED_SOURCE_DEFINITION_CONTRACT,
+    ]
+}
+
+
+COMPILE_RESULT_NODES_CONTRACT = {
+    'type': 'object',
+    'additionalProperties': False,
+    'description': (
+        'A collection of the parsed nodes, stored by their unique IDs.'
+    ),
+    'patternProperties': {
+        '.*': COMPILE_RESULT_NODE_CONTRACT
+    },
+}
+
+
+PARSED_MACROS_CONTRACT = {
+    'type': 'object',
+    'additionalProperties': False,
+    'description': (
+        'A collection of the parsed macros, stored by their unique IDs.'
+    ),
+    'patternProperties': {
+        '.*': PARSED_MACRO_CONTRACT
+    },
+}
+
+
+PARSED_DOCUMENTATIONS_CONTRACT = {
+    'type': 'object',
+    'additionalProperties': False,
+    'description': (
+        'A collection of the parsed docs, stored by their uniqe IDs.'
+    ),
+    'patternProperties': {
+        '.*': PARSED_DOCUMENTATION_CONTRACT,
+    },
+}
+
+
+NODE_EDGE_MAP = {
+    'type': 'object',
+    'additionalProperties': False,
+    'description': 'A map of node relationships',
+    'patternProperties': {
+        '.*': {
+            'type': 'array',
+            'items': {
+                'type': 'string',
+                'description': 'A node name',
+            }
+        }
+    }
+}
+
+
+PARSED_MANIFEST_CONTRACT = {
+    'type': 'object',
+    'additionalProperties': False,
+    'description': (
+        'The full parsed manifest of the graph, with both the required nodes'
+        ' and required macros.'
+    ),
+    'properties': {
+        'nodes': COMPILE_RESULT_NODES_CONTRACT,
+        'macros': PARSED_MACROS_CONTRACT,
+        'docs': PARSED_DOCUMENTATIONS_CONTRACT,
+        'disabled': {
+            'type': 'array',
+            'items': PARSED_NODE_CONTRACT,
+            'description': 'An array of disabled nodes',
+        },
+        'generated_at': {
+            'type': 'string',
+            'format': 'date-time',
+            'description': (
+                'The time at which the manifest was generated'
+            ),
+        },
+        'parent_map': NODE_EDGE_MAP,
+        'child_map': NODE_EDGE_MAP,
+        'metadata': {
+            'type': 'object',
+            'additionalProperties': False,
+            'properties': {
+                'project_id': {
+                    'type': ('string', 'null'),
+                    'description': (
+                        'The anonymized ID of the project. Persists as long '
+                        'as the project name stays the same.'
+                    ),
+                    'pattern': '[0-9a-f]{32}',
+                },
+                'user_id': {
+                    'type': ('string', 'null'),
+                    'description': (
+                        'The user ID assigned by dbt. Persists per-user as '
+                        'long as the user cookie file remains in place.'
+                    ),
+                    'pattern': (
+                        '[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-'
+                        '[0-9a-f]{12}'
+                    ),
+                },
+                'send_anonymous_usage_stats': {
+                    'type': ('boolean', 'null'),
+                    'description': (
+                        'Whether or not to send anonymized usage statistics.'
+                    ),
+                },
+            },
+            'required': [
+                'project_id', 'user_id', 'send_anonymous_usage_stats',
+            ],
+        },
+    },
+    'required': ['nodes', 'macros', 'docs', 'generated_at', 'metadata'],
+}
+
+
+class CompileResultNode(CompiledNode):
+    SCHEMA = COMPILE_RESULT_NODE_CONTRACT
+
+
+def _sort_values(dct):
+    """Given a dictionary, sort each value. This makes output deterministic,
+    which helps for tests.
+    """
+    return {k: sorted(v) for k, v in dct.items()}
+
+
+def build_edges(nodes):
+    """Build the forward and backward edges on the given list of ParsedNodes
+    and return them as two separate dictionaries, each mapping unique IDs to
+    lists of edges.
+    """
+    backward_edges = {}
+    # pre-populate the forward edge dict for simplicity
+    forward_edges = {node.unique_id: [] for node in nodes}
+    for node in nodes:
+        backward_edges[node.unique_id] = node.depends_on_nodes[:]
+        for unique_id in node.depends_on_nodes:
+            forward_edges[unique_id].append(node.unique_id)
+    return _sort_values(forward_edges), _sort_values(backward_edges)
+
+
+class Manifest(APIObject):
+    SCHEMA = PARSED_MANIFEST_CONTRACT
+    """The manifest for the full graph, after parsing and during compilation.
+    Nodes may be either ParsedNodes or CompiledNodes or a mix, depending upon
+    the current state of the compiler. Macros will always be ParsedMacros and
+    docs will always be ParsedDocumentations.
+    """
+    def __init__(self, nodes, macros, docs, generated_at, disabled,
+                 config=None):
+        """The constructor. nodes and macros are dictionaries mapping unique
+        IDs to ParsedNode/CompiledNode and ParsedMacro objects, respectively.
+        docs is a dictionary mapping unique IDs to ParsedDocumentation objects.
+        generated_at is a text timestamp in RFC 3339 format.
+        disabled is a list of disabled FQNs (as strings).
+        """
+        metadata = self.get_metadata(config)
+        self.nodes = nodes
+        self.macros = macros
+        self.docs = docs
+        self.generated_at = generated_at
+        self.metadata = metadata
+        self.disabled = disabled
+        self.flat_graph = None
+        super(Manifest, self).__init__()
+
+    @staticmethod
+    def get_metadata(config):
+        project_id = None
+        user_id = None
+        send_anonymous_usage_stats = None
+
+        if config is not None:
+            project_id = config.hashed_name()
+
+        if tracking.active_user is not None:
+            user_id = tracking.active_user.id
+            send_anonymous_usage_stats = not tracking.active_user.do_not_track
+
+        return {
+            'project_id': project_id,
+            'user_id': user_id,
+            'send_anonymous_usage_stats': send_anonymous_usage_stats,
+        }
+
+    def serialize(self):
+        """Convert the parsed manifest to a nested dict structure that we can
+        safely serialize to JSON.
+        """
+        forward_edges, backward_edges = build_edges(self.nodes.values())
+
+        return {
+            'nodes': {k: v.serialize() for k, v in self.nodes.items()},
+            'macros': {k: v.serialize() for k, v in self.macros.items()},
+            'docs': {k: v.serialize() for k, v in self.docs.items()},
+            'parent_map': backward_edges,
+            'child_map': forward_edges,
+            'generated_at': self.generated_at,
+            'metadata': self.metadata,
+            'disabled': [v.serialize() for v in self.disabled],
+        }
+
+    def build_flat_graph(self):
+        """This attribute is used in context.common by each node, so we want to
+        only build it once and avoid any concurrency issues around it.
+        Make sure you don't call this until you're done with building your
+        manifest!
+        """
+        self.flat_graph = {
+            'nodes': {
+                k: v.serialize() for k, v in self.nodes.items()
+            },
+        }
+
+    def find_disabled_by_name(self, name, package=None):
+        return dbt.utils.find_in_list_by_name(self.disabled, name, package,
+                                              NodeType.refable())
+
+    def _find_by_name(self, name, package, subgraph, nodetype):
+        """
+
+        Find a node by its given name in the appropriate sugraph. If package is
+        None, all pacakges will be searched.
+        nodetype should be a list of NodeTypes to accept.
+        """
+        if subgraph == 'nodes':
+            search = self.nodes
+        elif subgraph == 'macros':
+            search = self.macros
+        else:
+            raise NotImplementedError(
+                'subgraph search for {} not implemented'.format(subgraph)
+            )
+        return dbt.utils.find_in_subgraph_by_name(
+            search,
+            name,
+            package,
+            nodetype)
+
+    def find_docs_by_name(self, name, package=None):
+        for unique_id, doc in self.docs.items():
+            parts = unique_id.split('.')
+            if len(parts) != 2:
+                msg = "documentation names cannot contain '.' characters"
+                dbt.exceptions.raise_compiler_error(msg, doc)
+
+            found_package, found_node = parts
+
+            if (name == found_node and package in {None, found_package}):
+                return doc
+        return None
+
+    def find_macro_by_name(self, name, package):
+        """Find a macro in the graph by its name and package name, or None for
+        any package.
+        """
+        return self._find_by_name(name, package, 'macros', [NodeType.Macro])
+
+    def find_refable_by_name(self, name, package):
+        """Find any valid target for "ref()" in the graph by its name and
+        package name, or None for any package.
+        """
+        return self._find_by_name(name, package, 'nodes', NodeType.refable())
+
+    def find_source_by_name(self, source_name, table_name, package):
+        """Find any valid target for "source()" in the graph by its name and
+        package name, or None for any package.
+        """
+        name = '{}.{}'.format(source_name, table_name)
+        return self._find_by_name(name, package, 'nodes', [NodeType.Source])
+
+    def get_materialization_macro(self, materialization_name,
+                                  adapter_type=None):
+        macro_name = dbt.utils.get_materialization_macro_name(
+            materialization_name=materialization_name,
+            adapter_type=adapter_type,
+            with_prefix=False)
+
+        macro = self.find_macro_by_name(
+            macro_name,
+            None)
+
+        if adapter_type not in ('default', None) and macro is None:
+            macro_name = dbt.utils.get_materialization_macro_name(
+                materialization_name=materialization_name,
+                adapter_type='default',
+                with_prefix=False)
+            macro = self.find_macro_by_name(
+                macro_name,
+                None)
+
+        return macro
+
+    def get_resource_fqns(self):
+        resource_fqns = {}
+        for unique_id, node in self.nodes.items():
+            if node.resource_type == NodeType.Source:
+                continue  # sources have no FQNs and can't be configured
+            resource_type_plural = node.resource_type + 's'
+            if resource_type_plural not in resource_fqns:
+                resource_fqns[resource_type_plural] = set()
+            resource_fqns[resource_type_plural].add(tuple(node.fqn))
+
+        return resource_fqns
+
+    def _filter_subgraph(self, subgraph, predicate):
+        """
+        Given a subgraph of the manifest, and a predicate, filter
+        the subgraph using that predicate. Generates a list of nodes.
+        """
+        to_return = []
+
+        for unique_id, item in subgraph.items():
+            if predicate(item):
+                to_return.append(item)
+
+        return to_return
+
+    def _model_matches_schema_and_table(self, schema, table, model):
+        if model.resource_type == NodeType.Source:
+            return (model.schema.lower() == schema.lower() and
+                    model.identifier.lower() == table.lower())
+        return (model.schema.lower() == schema.lower() and
+                model.alias.lower() == table.lower())
+
+    def get_unique_ids_for_schema_and_table(self, schema, table):
+        """
+        Given a schema and table, find matching models, and return
+        their unique_ids. A schema and table may have more than one
+        match if the relation matches both a source and a seed, for instance.
+        """
+        def predicate(model):
+            return self._model_matches_schema_and_table(schema, table, model)
+
+        matching = list(self._filter_subgraph(self.nodes, predicate))
+        return [match.get('unique_id') for match in matching]
+
+    def add_nodes(self, new_nodes):
+        """Add the given dict of new nodes to the manifest."""
+        for unique_id, node in new_nodes.items():
+            if unique_id in self.nodes:
+                raise_duplicate_resource_name(node, self.nodes[unique_id])
+            self.nodes[unique_id] = node
+
+    def patch_nodes(self, patches):
+        """Patch nodes with the given dict of patches. Note that this consumes
+        the input!
+        """
+        # because we don't have any mapping from node _names_ to nodes, and we
+        # only have the node name in the patch, we have to iterate over all the
+        # nodes looking for matching names. We could use _find_by_name if we
+        # were ok with doing an O(n*m) search (one nodes scan per patch)
+        for node in self.nodes.values():
+            if node.resource_type != NodeType.Model:
+                continue
+            patch = patches.pop(node.name, None)
+            if not patch:
+                continue
+            node.patch(patch)
+
+        # log debug-level warning about nodes we couldn't find
+        if patches:
+            for patch in patches.values():
+                # since patches aren't nodes, we can't use the existing
+                # target_not_found warning
+                logger.debug((
+                    'WARNING: Found documentation for model "{}" which was '
+                    'not found or is disabled').format(patch.name)
+                )
+
+    def __getattr__(self, name):
+        raise AttributeError("'{}' object has no attribute '{}'".format(
+            type(self).__name__, name)
+        )
+
+    def get_used_schemas(self, resource_types=None):
+        return frozenset({
+            (node.database, node.schema)
+            for node in self.nodes.values()
+            if not resource_types or node.resource_type in resource_types
+        })
+
+    def get_used_databases(self):
+        return frozenset(node.database for node in self.nodes.values())
+
+    def deepcopy(self, config=None):
+        return Manifest(
+            nodes={k: v.incorporate() for k, v in self.nodes.items()},
+            macros={k: v.incorporate() for k, v in self.macros.items()},
+            docs={k: v.incorporate() for k, v in self.docs.items()},
+            generated_at=self.generated_at,
+            disabled=[n.incorporate() for n in self.disabled],
+            config=config
+        )

core/dbt/contracts/graph/parsed.py

@@ -0,0 +1,815 @@
+from dbt.api import APIObject
+from dbt.utils import deep_merge
+from dbt.node_types import NodeType
+
+import dbt.clients.jinja
+
+from dbt.contracts.graph.unparsed import UNPARSED_NODE_CONTRACT, \
+    UNPARSED_MACRO_CONTRACT, UNPARSED_DOCUMENTATION_FILE_CONTRACT, \
+    UNPARSED_BASE_CONTRACT, TIME_CONTRACT
+
+from dbt.logger import GLOBAL_LOGGER as logger  # noqa
+
+
+# TODO: which of these do we _really_ support? or is it both?
+HOOK_CONTRACT = {
+    'anyOf': [
+        {
+            'type': 'object',
+            'additionalProperties': False,
+            'properties': {
+                'sql': {
+                    'type': 'string',
+                },
+                'transaction': {
+                    'type': 'boolean',
+                },
+                'index': {
+                    'type': 'integer',
+                }
+            },
+            'required': ['sql', 'transaction'],
+        },
+        {
+            'type': 'string',
+        },
+    ],
+}
+
+
+CONFIG_CONTRACT = {
+    'type': 'object',
+    'additionalProperties': True,
+    'properties': {
+        'enabled': {
+            'type': 'boolean',
+        },
+        'materialized': {
+            'type': 'string',
+        },
+        'persist_docs': {
+            'type': 'object',
+            'additionalProperties': True,
+        },
+        'post-hook': {
+            'type': 'array',
+            'items': HOOK_CONTRACT,
+        },
+        'pre-hook': {
+            'type': 'array',
+            'items': HOOK_CONTRACT,
+        },
+        'vars': {
+            'type': 'object',
+            'additionalProperties': True,
+        },
+        'quoting': {
+            'type': 'object',
+            'additionalProperties': True,
+        },
+        'column_types': {
+            'type': 'object',
+            'additionalProperties': True,
+        },
+        'tags': {
+            'anyOf': [
+                {
+                    'type': 'array',
+                    'items': {
+                        'type': 'string'
+                    },
+                },
+                {
+                    'type': 'string'
+                }
+            ]
+        },
+        'severity': {
+            'type': 'string',
+            'pattern': '([eE][rR][rR][oO][rR]|[wW][aA][rR][nN])',
+        },
+    },
+    'required': [
+        'enabled', 'materialized', 'post-hook', 'pre-hook', 'vars',
+        'quoting', 'column_types', 'tags', 'persist_docs'
+    ]
+}
+
+
+#  Note that description must be present, but may be empty.
+COLUMN_INFO_CONTRACT = {
+    'type': 'object',
+    'additionalProperties': False,
+    'description': 'Information about a single column in a model',
+    'properties': {
+        'name': {
+            'type': 'string',
+            'description': 'The column name',
+        },
+        'description': {
+            'type': 'string',
+            'description': 'A description of the column',
+        },
+    },
+    'required': ['name', 'description'],
+}
+
+
+# Docrefs are not quite like regular references, as they indicate what they
+# apply to as well as what they are referring to (so the doc package + doc
+# name, but also the column name if relevant). This is because column
+# descriptions are rendered separately from their models.
+DOCREF_CONTRACT = {
+    'type': 'object',
+    'properties': {
+        'documentation_name': {
+            'type': 'string',
+            'description': 'The name of the documentation block referred to',
+        },
+        'documentation_package': {
+            'type': 'string',
+            'description': (
+                'If provided, the documentation package name referred to'
+            ),
+        },
+        'column_name': {
+            'type': 'string',
+            'description': (
+                'If the documentation refers to a column instead of the '
+                'model, the column name should be set'
+            ),
+        },
+    },
+    'required': ['documentation_name', 'documentation_package']
+}
+
+
+HAS_FQN_CONTRACT = {
+    'properties': {
+        'fqn': {
+            'type': 'array',
+            'items': {
+                'type': 'string',
+            }
+        },
+    },
+    'required': ['fqn'],
+}
+
+
+HAS_UNIQUE_ID_CONTRACT = {
+    'properties': {
+        'unique_id': {
+            'type': 'string',
+            'minLength': 1,
+        },
+    },
+    'required': ['unique_id'],
+}
+
+CAN_REF_CONTRACT = {
+    'properties': {
+        'refs': {
+            'type': 'array',
+            'items': {
+                'type': 'array',
+                'description': (
+                    'The list of arguments passed to a single ref call.'
+                ),
+            },
+            'description': (
+                'The list of call arguments, one list of arguments per '
+                'call.'
+            )
+        },
+        'sources': {
+            'type': 'array',
+            'items': {
+                'type': 'array',
+                'description': (
+                    'The list of arguments passed to a single source call.'
+                ),
+            },
+            'description': (
+                'The list of call arguments, one list of arguments per '
+                'call.'
+            )
+        },
+        'depends_on': {
+            'type': 'object',
+            'additionalProperties': False,
+            'properties': {
+                'nodes': {
+                    'type': 'array',
+                    'items': {
+                        'type': 'string',
+                        'minLength': 1,
+                        'description': (
+                            'A node unique ID that this depends on.'
+                        )
+                    }
+                },
+                'macros': {
+                    'type': 'array',
+                    'items': {
+                        'type': 'string',
+                        'minLength': 1,
+                        'description': (
+                            'A macro unique ID that this depends on.'
+                        )
+                    }
+                },
+            },
+            'description': (
+                'A list of unique IDs for nodes and macros that this '
+                'node depends upon.'
+            ),
+            'required': ['nodes', 'macros'],
+        },
+    },
+    'required': ['refs', 'sources', 'depends_on'],
+}
+
+
+HAS_DOCREFS_CONTRACT = {
+    'properties': {
+        'docrefs': {
+            'type': 'array',
+            'items': DOCREF_CONTRACT,
+        },
+    },
+}
+
+
+HAS_DESCRIPTION_CONTRACT = {
+    'properties': {
+        'description': {
+            'type': 'string',
+            'description': 'A user-supplied description of the model',
+        },
+        'columns': {
+            'type': 'object',
+            'properties': {
+                '.*': COLUMN_INFO_CONTRACT,
+            },
+        },
+    },
+    'required': ['description', 'columns'],
+}
+
+# does this belong inside another contract?
+HAS_CONFIG_CONTRACT = {
+    'properties': {
+        'config': CONFIG_CONTRACT,
+    },
+    'required': ['config'],
+}
+
+
+COLUMN_TEST_CONTRACT = {
+    'properties': {
+        'column_name': {
+            'type': 'string',
+            'description': (
+                'In tests parsed from a v2 schema, the column the test is '
+                'associated with (if there is one)'
+            )
+        },
+    }
+}
+
+
+HAS_RELATION_METADATA_CONTRACT = {
+    'properties': {
+        'database': {
+            'type': 'string',
+            'description': (
+                'The actual database string that this will build into.'
+            )
+        },
+        'schema': {
+            'type': 'string',
+            'description': (
+                'The actual schema string that this will build into.'
+            )
+        },
+    },
+    'required': ['database', 'schema'],
+}
+
+
+PARSED_NODE_CONTRACT = deep_merge(
+    UNPARSED_NODE_CONTRACT,
+    HAS_UNIQUE_ID_CONTRACT,
+    HAS_FQN_CONTRACT,
+    CAN_REF_CONTRACT,
+    HAS_DOCREFS_CONTRACT,
+    HAS_DESCRIPTION_CONTRACT,
+    HAS_CONFIG_CONTRACT,
+    COLUMN_TEST_CONTRACT,
+    HAS_RELATION_METADATA_CONTRACT,
+    {
+        'properties': {
+            'alias': {
+                'type': 'string',
+                'description': (
+                    'The name of the relation that this will build into'
+                )
+            },
+            # TODO: move this into a class property.
+            'empty': {
+                'type': 'boolean',
+                'description': 'True if the SQL is empty',
+            },
+            'tags': {
+                'type': 'array',
+                'items': {
+                    'type': 'string',
+                }
+            },
+            # this is really nodes-only
+            'patch_path': {
+                'type': 'string',
+                'description': (
+                    'The path to the patch source if the node was patched'
+                ),
+            },
+            'build_path': {
+                'type': 'string',
+                'description': (
+                    'In seeds, the path to the source file used during build.'
+                ),
+            },
+        },
+        'required': ['empty', 'tags', 'alias'],
+    }
+)
+
+
+class ParsedNode(APIObject):
+    SCHEMA = PARSED_NODE_CONTRACT
+
+    def __init__(self, agate_table=None, **kwargs):
+        self.agate_table = agate_table
+        kwargs.setdefault('columns', {})
+        kwargs.setdefault('description', '')
+        super(ParsedNode, self).__init__(**kwargs)
+
+    @property
+    def is_refable(self):
+        return self.resource_type in NodeType.refable()
+
+    @property
+    def is_ephemeral(self):
+        return self.get('config', {}).get('materialized') == 'ephemeral'
+
+    @property
+    def is_ephemeral_model(self):
+        return self.is_refable and self.is_ephemeral
+
+    @property
+    def depends_on_nodes(self):
+        """Return the list of node IDs that this node depends on."""
+        return self.depends_on['nodes']
+
+    def to_dict(self):
+        """Similar to 'serialize', but tacks the agate_table attribute in too.
+        Why we need this:
+            - networkx demands that the attr_dict it gets (the node) be a dict
+                or subclass and does not respect the abstract Mapping class
+            - many jinja things access the agate_table attribute (member) of
+                the node dict.
+            - the nodes are passed around between those two contexts in a way
+                that I don't quite have clear enough yet.
+        """
+        ret = self.serialize()
+        # note: not a copy/deep copy.
+        ret['agate_table'] = self.agate_table
+        return ret
+
+    def to_shallow_dict(self):
+        ret = self._contents.copy()
+        ret['agate_table'] = self.agate_table
+        return ret
+
+    def patch(self, patch):
+        """Given a ParsedNodePatch, add the new information to the node."""
+        # explicitly pick out the parts to update so we don't inadvertently
+        # step on the model name or anything
+        self._contents.update({
+            'patch_path': patch.original_file_path,
+            'description': patch.description,
+            'columns': patch.columns,
+            'docrefs': patch.docrefs,
+        })
+        # patches always trigger re-validation
+        self.validate()
+
+    def get_materialization(self):
+        return self.config.get('materialized')
+
+    @property
+    def build_path(self):
+        return self._contents.get('build_path')
+
+    @build_path.setter
+    def build_path(self, value):
+        self._contents['build_path'] = value
+
+    @property
+    def database(self):
+        return self._contents['database']
+
+    @database.setter
+    def database(self, value):
+        self._contents['database'] = value
+
+    @property
+    def schema(self):
+        return self._contents['schema']
+
+    @schema.setter
+    def schema(self, value):
+        self._contents['schema'] = value
+
+    @property
+    def alias(self):
+        return self._contents['alias']
+
+    @alias.setter
+    def alias(self, value):
+        self._contents['alias'] = value
+
+    @property
+    def config(self):
+        return self._contents['config']
+
+    @config.setter
+    def config(self, value):
+        self._contents['config'] = value
+
+
+SNAPSHOT_CONFIG_CONTRACT = {
+    'properties': {
+        'target_database': {
+            'type': 'string',
+        },
+        'target_schema': {
+            'type': 'string',
+        },
+        'unique_key': {
+            'type': 'string',
+        },
+        'anyOf': [
+            {
+                'properties': {
+                    'strategy': {
+                        'enum': ['timestamp'],
+                    },
+                    'updated_at': {
+                        'type': 'string',
+                        'description': (
+                            'The column name with the timestamp to compare'
+                        ),
+                    },
+                },
+                'required': ['updated_at'],
+            },
+            {
+                'properties': {
+                    'strategy': {
+                        'enum': ['check'],
+                    },
+                    'check_cols': {
+                        'oneOf': [
+                            {
+                                'type': 'array',
+                                'items': {'type': 'string'},
+                                'description': 'The columns to check',
+                                'minLength': 1,
+                            },
+                            {
+                                'enum': ['all'],
+                                'description': 'Check all columns',
+                            },
+                        ],
+                    },
+                },
+                'required': ['check_cols'],
+            }
+        ]
+    },
+    'required': [
+        'target_schema', 'unique_key', 'strategy',
+    ],
+}
+
+
+PARSED_SNAPSHOT_NODE_CONTRACT = deep_merge(
+    PARSED_NODE_CONTRACT,
+    {
+        'properties': {
+            'config': SNAPSHOT_CONFIG_CONTRACT,
+            'resource_type': {
+                'enum': [NodeType.Snapshot],
+            },
+        },
+    }
+)
+
+
+class ParsedSnapshotNode(ParsedNode):
+    SCHEMA = PARSED_SNAPSHOT_NODE_CONTRACT
+
+
+# The parsed node update is only the 'patch', not the test. The test became a
+# regular parsed node. Note that description and columns must be present, but
+# may be empty.
+PARSED_NODE_PATCH_CONTRACT = {
+    'type': 'object',
+    'additionalProperties': False,
+    'description': 'A collection of values that can be set on a node',
+    'properties': {
+        'name': {
+            'type': 'string',
+            'description': 'The name of the node this modifies',
+        },
+        'description': {
+            'type': 'string',
+            'description': 'The description of the node to add',
+        },
+        'original_file_path': {
+            'type': 'string',
+            'description': (
+                'Relative path to the originating file path for the patch '
+                'from the project root'
+            ),
+        },
+        'columns': {
+            'type': 'object',
+            'properties': {
+                '.*': COLUMN_INFO_CONTRACT,
+            }
+        },
+        'docrefs': {
+            'type': 'array',
+            'items': DOCREF_CONTRACT,
+        }
+    },
+    'required': [
+        'name', 'original_file_path', 'description', 'columns', 'docrefs'
+    ],
+}
+
+
+class ParsedNodePatch(APIObject):
+    SCHEMA = PARSED_NODE_PATCH_CONTRACT
+
+
+PARSED_MACRO_CONTRACT = deep_merge(
+    UNPARSED_MACRO_CONTRACT,
+    {
+        # This is required for the 'generator' field to work.
+        # TODO: fix before release
+        'additionalProperties': True,
+        'properties': {
+            'name': {
+                'type': 'string',
+                'description': (
+                    'Name of this node. For models, this is used as the '
+                    'identifier in the database.'),
+                'minLength': 1,
+                'maxLength': 127,
+            },
+            'resource_type': {
+                'enum': [
+                    NodeType.Macro,
+                ],
+            },
+            'unique_id': {
+                'type': 'string',
+                'minLength': 1,
+                'maxLength': 255,
+            },
+            'tags': {
+                'description': (
+                    'An array of arbitrary strings to use as tags.'
+                ),
+                'type': 'array',
+                'items': {
+                    'type': 'string',
+                },
+            },
+            'depends_on': {
+                'type': 'object',
+                'additionalProperties': False,
+                'properties': {
+                    'macros': {
+                        'type': 'array',
+                        'items': {
+                            'type': 'string',
+                            'minLength': 1,
+                            'maxLength': 255,
+                            'description': 'A single macro unique ID.'
+                        }
+                    }
+                },
+                'description': 'A list of all macros this macro depends on.',
+                'required': ['macros'],
+            },
+        },
+        'required': [
+            'resource_type', 'unique_id', 'tags', 'depends_on', 'name',
+        ]
+    }
+)
+
+
+class ParsedMacro(APIObject):
+    SCHEMA = PARSED_MACRO_CONTRACT
+
+    @property
+    def generator(self):
+        """
+        Returns a function that can be called to render the macro results.
+        """
+        # TODO: we can generate self.template from the other properties
+        # available in this class. should we just generate this here?
+        return dbt.clients.jinja.macro_generator(self._contents)
+
+
+# This is just the file + its ID
+PARSED_DOCUMENTATION_CONTRACT = deep_merge(
+    UNPARSED_DOCUMENTATION_FILE_CONTRACT,
+    {
+        'properties': {
+            'name': {
+                'type': 'string',
+                'description': (
+                    'Name of this node, as referred to by doc() references'
+                ),
+            },
+            'unique_id': {
+                'type': 'string',
+                'minLength': 1,
+                'maxLength': 255,
+                'description': (
+                    'The unique ID of this node as stored in the manifest'
+                ),
+            },
+            'block_contents': {
+                'type': 'string',
+                'description': 'The contents of just the docs block',
+            },
+        },
+        'required': ['name', 'unique_id', 'block_contents'],
+    }
+)
+
+
+NODE_EDGE_MAP = {
+    'type': 'object',
+    'additionalProperties': False,
+    'description': 'A map of node relationships',
+    'patternProperties': {
+        '.*': {
+            'type': 'array',
+            'items': {
+                'type': 'string',
+                'description': 'A node name',
+            }
+        }
+    }
+}
+
+
+class ParsedDocumentation(APIObject):
+    SCHEMA = PARSED_DOCUMENTATION_CONTRACT
+
+
+class Hook(APIObject):
+    SCHEMA = HOOK_CONTRACT
+
+
+FRESHNESS_CONTRACT = {
+    'properties': {
+        'loaded_at_field': {
+            'type': ['null', 'string'],
+            'description': 'The field to use as the "loaded at" timestamp',
+        },
+        'freshness': {
+            'anyOf': [
+                {'type': 'null'},
+                {
+                    'type': 'object',
+                    'additionalProperties': False,
+                    'properties': {
+                        'warn_after': TIME_CONTRACT,
+                        'error_after': TIME_CONTRACT,
+                    },
+                },
+            ],
+        },
+    },
+}
+
+
+QUOTING_CONTRACT = {
+    'properties': {
+        'quoting': {
+            'type': 'object',
+            'additionalProperties': False,
+            'properties': {
+                'database': {'type': 'boolean'},
+                'schema': {'type': 'boolean'},
+                'identifier': {'type': 'boolean'},
+            },
+        },
+    },
+    'required': ['quoting'],
+}
+
+
+PARSED_SOURCE_DEFINITION_CONTRACT = deep_merge(
+    UNPARSED_BASE_CONTRACT,
+    FRESHNESS_CONTRACT,
+    QUOTING_CONTRACT,
+    HAS_DESCRIPTION_CONTRACT,
+    HAS_UNIQUE_ID_CONTRACT,
+    HAS_DOCREFS_CONTRACT,
+    HAS_RELATION_METADATA_CONTRACT,
+    HAS_FQN_CONTRACT,
+    {
+        'description': (
+            'A source table definition, as parsed from the one provided in the'
+            '"tables" subsection of the "sources" section of schema.yml'
+        ),
+        'properties': {
+            'name': {
+                'type': 'string',
+                'description': (
+                    'The name of this node, which is the name of the model it'
+                    'refers to'
+                ),
+                'minLength': 1,
+            },
+            'source_name': {
+                'type': 'string',
+                'description': 'The reference name of the source definition',
+                'minLength': 1,
+            },
+            'source_description': {
+                'type': 'string',
+                'description': 'The user-supplied description of the source',
+            },
+            'loader': {
+                'type': 'string',
+                'description': 'The user-defined loader for this source',
+            },
+            'identifier': {
+                'type': 'string',
+                'description': 'The identifier for the source table',
+                'minLength': 1,
+            },
+            # the manifest search stuff really requires this, sadly
+            'resource_type': {
+                'enum': [NodeType.Source],
+            },
+        },
+        # note that while required, loaded_at_field and freshness may be null
+        'required': [
+            'source_name', 'source_description', 'loaded_at_field', 'loader',
+            'freshness', 'description', 'columns', 'docrefs', 'identifier',
+        ],
+    }
+)
+
+
+class ParsedSourceDefinition(APIObject):
+    SCHEMA = PARSED_SOURCE_DEFINITION_CONTRACT
+    is_ephemeral_model = False
+
+    def to_shallow_dict(self):
+        return self._contents.copy()
+
+    # provide some emtpy/meaningless properties so these look more like
+    # ParsedNodes
+    @property
+    def depends_on_nodes(self):
+        return []
+
+    @property
+    def refs(self):
+        return []
+
+    @property
+    def sources(self):
+        return []
+
+    @property
+    def tags(self):
+        return []
+
+    @property
+    def has_freshness(self):
+        return bool(self.freshness) and self.loaded_at_field is not None

core/dbt/contracts/graph/unparsed.py

@@ -0,0 +1,389 @@
+from dbt.api import APIObject
+
+from dbt.node_types import NodeType
+from dbt.utils import deep_merge
+
+
+UNPARSED_BASE_CONTRACT = {
+    'type': 'object',
+    'additionalProperties': False,
+    'properties': {
+        'package_name': {
+            'type': 'string',
+        },
+        # filesystem
+        'root_path': {
+            'type': 'string',
+            'description': 'The absolute path to the project root',
+        },
+        'path': {
+            'type': 'string',
+            'description': (
+                'Relative path to the source file from the project root. '
+                'Usually the same as original_file_path, but in some cases '
+                'dbt will generate a path.'),
+        },
+        'original_file_path': {
+            'type': 'string',
+            'description': (
+                'Relative path to the originating file from the project root.'
+            ),
+        }
+    },
+    'required': ['package_name', 'root_path', 'path', 'original_file_path']
+}
+
+UNPARSED_HAS_SQL_CONTRACT = {
+    'properties': {
+        'raw_sql': {
+            'type': 'string',
+            'description': (
+                'For nodes defined in SQL files, this is just the contents '
+                'of that file. For schema tests, snapshots, etc. this is '
+                'generated by dbt.'),
+        },
+        'index': {
+            'type': 'integer',
+        }
+    },
+    'required': ['raw_sql']
+}
+
+UNPARSED_MACRO_CONTRACT = deep_merge(
+    UNPARSED_BASE_CONTRACT,
+    UNPARSED_HAS_SQL_CONTRACT
+)
+
+UNPARSED_NODE_CONTRACT = deep_merge(
+    UNPARSED_BASE_CONTRACT,
+    UNPARSED_HAS_SQL_CONTRACT,
+    {
+        'properties': {
+            'name': {
+                'type': 'string',
+                'description': (
+                    'Name of this node. For models, this is used as the '
+                    'identifier in the database.'),
+                'minLength': 1,
+            },
+            'resource_type': {
+                'enum': [
+                    NodeType.Model,
+                    NodeType.Test,
+                    NodeType.Analysis,
+                    NodeType.Operation,
+                    NodeType.Seed,
+                    # we need this if parse_node is going to handle snapshots.
+                    NodeType.Snapshot,
+                    NodeType.RPCCall,
+                ]
+            },
+        },
+        'required': ['resource_type', 'name']
+    }
+)
+
+
+class UnparsedMacro(APIObject):
+    SCHEMA = UNPARSED_MACRO_CONTRACT
+
+
+class UnparsedNode(APIObject):
+    SCHEMA = UNPARSED_NODE_CONTRACT
+
+
+COLUMN_TEST_CONTRACT = {
+    'type': 'object',
+    'additionalProperties': False,
+    'properties': {
+        'name': {
+            'type': 'string',
+            'description': 'The name of the column this test is for',
+        },
+        'description': {
+            'type': 'string',
+            'description': 'The description of this test',
+        },
+        'tests': {
+            'type': 'array',
+            'items': {
+                "anyOf": [
+                    # 'not_null', 'unique', ...
+                    {'type': 'string'},
+                    # 'relationships: {...}', 'accepted_values: {...}'
+                    {'type': 'object', 'additionalProperties': True}
+                ],
+            },
+            'description': 'The list of tests to perform',
+        },
+    },
+    'required': ['name'],
+}
+
+
+UNPARSED_COLUMN_DESCRIPTION_CONTRACT = {
+    'properties': {
+        'columns': {
+            'type': 'array',
+            'items': COLUMN_TEST_CONTRACT,
+        },
+    },
+}
+
+
+UNPARSED_NODE_DESCRIPTION_CONTRACT = {
+    'properties': {
+        'name': {
+            'type': 'string',
+            'description': (
+                'The name of this node, which is the name of the model it'
+                'refers to'
+            ),
+            'minLength': 1,
+        },
+        'description': {
+            'type': 'string',
+            'description': (
+                'The raw string description of the node after parsing the yaml'
+            ),
+        },
+        'tests': {
+            'type': 'array',
+            'items': {
+                "anyOf": [
+                    {'type': 'string'},
+                    {'type': 'object', 'additionalProperties': True}
+                ],
+            },
+        },
+    },
+    'required': ['name'],
+}
+
+
+UNPARSED_NODE_UPDATE_CONTRACT = deep_merge(
+    UNPARSED_NODE_DESCRIPTION_CONTRACT,
+    UNPARSED_COLUMN_DESCRIPTION_CONTRACT,
+    {
+        'type': 'object',
+        'additionalProperties': False,
+        'description': (
+            'A collection of the unparsed node updates, as provided in the '
+            '"models" section of schema.yml'
+        ),
+    }
+)
+
+
+class UnparsedNodeUpdate(APIObject):
+    """An unparsed node update is the blueprint for tests to be added and nodes
+    to be updated, referencing a certain node (specifically, a Model or
+    Source).
+    """
+    SCHEMA = UNPARSED_NODE_UPDATE_CONTRACT
+
+
+TIME_CONTRACT = {
+    'type': 'object',
+    'additionalProperties': False,
+    'properties': {
+        'count': {
+            'type': 'integer',
+        },
+        'period': {
+            'enum': ['minute', 'hour', 'day'],
+        },
+    },
+    'required': ['count', 'period'],
+}
+
+
+_FRESHNESS_CONTRACT = {
+    'type': 'object',
+    'additionalProperties': False,
+    'properties': {
+        'warn_after': {
+            'anyOf': [
+                {'type': 'null'},
+                TIME_CONTRACT,
+            ]
+        },
+        'error_after': {
+            'anyOf': [
+                {'type': 'null'},
+                TIME_CONTRACT,
+
+            ]
+        },
+    },
+}
+
+
+_QUOTING_CONTRACT = {
+    'type': 'object',
+    'additionalProperties': False,
+    'properties': {
+        'database': {'type': 'boolean'},
+        'schema': {'type': 'boolean'},
+        'identifier': {'type': 'boolean'},
+    },
+}
+
+
+QUOTING_CONTRACT = {
+    'properties': {
+        'quoting': {
+            'anyOf': [
+                {'type': 'null'},
+                _QUOTING_CONTRACT,
+            ],
+        },
+    },
+}
+
+
+FRESHNESS_CONTRACT = {
+    'properties': {
+        'loaded_at_field': {
+            'type': ['null', 'string'],
+            'description': 'The field to use as the "loaded at" timestamp',
+        },
+        'freshness': {
+            'anyOf': [
+                {'type': 'null'},
+                _FRESHNESS_CONTRACT,
+            ],
+        },
+    },
+}
+
+
+UNPARSED_SOURCE_TABLE_DEFINITION_CONTRACT = deep_merge(
+    UNPARSED_NODE_DESCRIPTION_CONTRACT,
+    UNPARSED_COLUMN_DESCRIPTION_CONTRACT,
+    FRESHNESS_CONTRACT,
+    QUOTING_CONTRACT,
+    {
+        'description': (
+            'A source table definition, as provided in the "tables" '
+            'subsection of the "sources" section of schema.yml'
+        ),
+        'properties': {
+            'identifier': {
+                'type': 'string',
+                'description': 'The identifier for the source table',
+                'minLength': 1,
+            },
+        },
+    }
+)
+
+
+UNPARSED_SOURCE_DEFINITION_CONTRACT = deep_merge(
+    FRESHNESS_CONTRACT,
+    QUOTING_CONTRACT,
+    {
+        'type': 'object',
+        'additionalProperties': False,
+        'description': (
+            'A collection of the unparsed sources, as provided in the '
+            '"sources" section of schema.yml'
+        ),
+        'properties': {
+            'name': {
+                'type': 'string',
+                'description': 'The reference name of the source definition',
+                'minLength': 1,
+            },
+            'loader': {
+                'type': 'string',
+                'description': 'The user-defined loader for this source',
+                'minLength': 1,
+            },
+            'description': {
+                'type': 'string',
+                'description': 'The user-supplied description of the source',
+            },
+            'database': {
+                'type': 'string',
+                'description': 'The database name for the source table',
+                'minLength': 1,
+            },
+            'schema': {
+                'type': 'string',
+                'description': 'The schema name for the source table',
+                'minLength': 1,
+            },
+            'tables': {
+                'type': 'array',
+                'items': UNPARSED_SOURCE_TABLE_DEFINITION_CONTRACT,
+                'description': 'The tables for this source',
+                'minLength': 1,
+            },
+        },
+        'required': ['name'],
+    }
+)
+
+
+class UnparsedTableDefinition(APIObject):
+    SCHEMA = UNPARSED_SOURCE_TABLE_DEFINITION_CONTRACT
+
+
+class UnparsedSourceDefinition(APIObject):
+    SCHEMA = UNPARSED_SOURCE_DEFINITION_CONTRACT
+
+    @property
+    def tables(self):
+        return [UnparsedTableDefinition(**t) for t in self.get('tables', [])]
+
+
+UNPARSED_DOCUMENTATION_FILE_CONTRACT = {
+    'type': 'object',
+    'additionalProperties': False,
+    'properties': {
+        'package_name': {
+            'type': 'string',
+        },
+        # filesystem
+        'root_path': {
+            'type': 'string',
+            'description': 'The absolute path to the project root',
+        },
+        'path': {
+            'type': 'string',
+            'description': (
+                'Relative path to the source file from the project root. '
+                'Usually the same as original_file_path, but in some cases '
+                'dbt will generate a path.'),
+        },
+        'original_file_path': {
+            'type': 'string',
+            'description': (
+                'Relative path to the originating file from the project root.'
+            ),
+        },
+        'file_contents': {
+            'type': 'string',
+            'description': (
+                'The raw text provided in the documentation block, presumably '
+                'markdown.'
+            ),
+        },
+        # TODO: I would like to remove this, but some graph error handling
+        # cares about it.
+        'resource_type': {
+            'enum': [
+                NodeType.Documentation,
+            ]
+        },
+    },
+    'required': [
+        'package_name', 'root_path', 'path', 'original_file_path',
+        'file_contents', 'resource_type'
+    ],
+}
+
+
+class UnparsedDocumentationFile(APIObject):
+    SCHEMA = UNPARSED_DOCUMENTATION_FILE_CONTRACT

core/dbt/contracts/project.py

@@ -0,0 +1,400 @@
+from dbt.api.object import APIObject
+from dbt.logger import GLOBAL_LOGGER as logger  # noqa
+from dbt.utils import deep_merge
+
+
+ARCHIVE_TABLE_CONFIG_CONTRACT = {
+    'type': 'object',
+    'additionalProperties': False,
+    'properties': {
+        'source_table': {'type': 'string'},
+        'target_table': {'type': 'string'},
+        'updated_at': {'type': 'string'},
+        'unique_key': {'type': 'string'},
+    },
+    'required': ['source_table', 'target_table', 'updated_at', 'unique_key'],
+}
+
+
+ARCHIVE_CONFIG_CONTRACT = {
+    'type': 'object',
+    'additionalProperties': False,
+    'properties': {
+        'source_database': {'type': 'string'},
+        'target_database': {'type': 'string'},
+        'source_schema': {'type': 'string'},
+        'target_schema': {'type': 'string'},
+        'tables': {
+            'type': 'array',
+            'items': ARCHIVE_TABLE_CONFIG_CONTRACT,
+        }
+    },
+    'required': ['source_schema', 'target_schema', 'tables'],
+}
+
+
+PROJECT_CONTRACT = {
+    'type': 'object',
+    'description': 'The project configuration.',
+    'additionalProperties': False,
+    'properties': {
+        'name': {
+            'type': 'string',
+            'pattern': r'^[^\d\W]\w*\Z',
+        },
+        'version': {
+            'anyOf': [
+                {
+                    'type': 'string',
+                    'pattern': (
+                        # this does not support the full semver (does not
+                        # allow a trailing -fooXYZ) and is not restrictive
+                        # enough for full semver, (allows '1.0'). But it's like
+                        # 'semver lite'.
+                        r'^(?:0|[1-9]\d*)\.(?:0|[1-9]\d*)(\.(?:0|[1-9]\d*))?$'
+                    ),
+                },
+                {
+                    # the internal global_project/dbt_project.yml is actually
+                    # 1.0. Heaven only knows how many users have done the same
+                    'type': 'number',
+                },
+            ],
+        },
+        'project-root': {
+            'type': 'string',
+        },
+        'source-paths': {
+            'type': 'array',
+            'items': {'type': 'string'},
+        },
+        'macro-paths': {
+            'type': 'array',
+            'items': {'type': 'string'},
+        },
+        'data-paths': {
+            'type': 'array',
+            'items': {'type': 'string'},
+        },
+        'test-paths': {
+            'type': 'array',
+            'items': {'type': 'string'},
+        },
+        'analysis-paths': {
+            'type': 'array',
+            'items': {'type': 'string'},
+        },
+        'docs-paths': {
+            'type': 'array',
+            'items': {'type': 'string'},
+        },
+        'target-path': {
+            'type': 'string',
+        },
+        'snapshot-paths': {
+            'type': 'array',
+            'items': {'type': 'string'},
+        },
+        'clean-targets': {
+            'type': 'array',
+            'items': {'type': 'string'},
+        },
+        'profile': {
+            'type': ['null', 'string'],
+        },
+        'log-path': {
+            'type': 'string',
+        },
+        'modules-path': {
+            'type': 'string',
+        },
+        'quoting': {
+            'type': 'object',
+            'additionalProperties': False,
+            'properties': {
+                'identifier': {
+                    'type': 'boolean',
+                },
+                'schema': {
+                    'type': 'boolean',
+                },
+                'database': {
+                    'type': 'boolean',
+                },
+                'project': {
+                    'type': 'boolean',
+                }
+            },
+        },
+        'models': {
+            'type': 'object',
+            'additionalProperties': True,
+        },
+        'on-run-start': {
+            'type': 'array',
+            'items': {'type': 'string'},
+        },
+        'on-run-end': {
+            'type': 'array',
+            'items': {'type': 'string'},
+        },
+        'archive': {
+            'type': 'array',
+            'items': ARCHIVE_CONFIG_CONTRACT,
+        },
+        'seeds': {
+            'type': 'object',
+            'additionalProperties': True,
+        },
+        # we validate the regex separately, using the pattern in dbt.semver
+        'require-dbt-version': {
+            'type': ['string', 'array'],
+            'items': {'type': 'string'},
+        },
+    },
+    'required': ['name', 'version'],
+}
+
+
+class Project(APIObject):
+    SCHEMA = PROJECT_CONTRACT
+
+
+LOCAL_PACKAGE_CONTRACT = {
+    'type': 'object',
+    'additionalProperties': False,
+    'properties': {
+        'local': {
+            'type': 'string',
+            'description': 'The absolute path to the local package.',
+        },
+        'required': ['local'],
+    },
+}
+
+
+GIT_PACKAGE_CONTRACT = {
+    'type': 'object',
+    'additionalProperties': False,
+    'properties': {
+        'git': {
+            'type': 'string',
+            'description': (
+                'The URL to the git repository that stores the pacakge'
+            ),
+        },
+        'revision': {
+            'type': ['string', 'array'],
+            'items': {'type': 'string'},
+            'description': 'The git revision to use, if it is not tip',
+        },
+        'warn-unpinned': {
+            'type': 'boolean',
+        }
+    },
+    'required': ['git'],
+}
+
+
+VERSION_SPECIFICATION_CONTRACT = {
+    'type': 'object',
+    'additionalProperties': False,
+    'properties': {
+        'major': {
+            'type': ['string', 'null'],
+        },
+        'minor': {
+            'type': ['string', 'null'],
+        },
+        'patch': {
+            'type': ['string', 'null'],
+        },
+        'prerelease': {
+            'type': ['string', 'null'],
+        },
+        'build': {
+            'type': ['string', 'null'],
+        },
+        'matcher': {
+            'type': 'string',
+            'enum': ['=', '>=', '<=', '>', '<'],
+        },
+    },
+    'required': ['major', 'minor', 'patch', 'prerelease', 'build', 'matcher'],
+}
+
+
+REGISTRY_PACKAGE_CONTRACT = {
+    'type': 'object',
+    'additionalProperties': False,
+    'properties': {
+        'package': {
+            'type': 'string',
+            'description': 'The name of the package',
+        },
+        'version': {
+            'type': ['string', 'array'],
+            'items': {
+                'anyOf': [
+                    VERSION_SPECIFICATION_CONTRACT,
+                    {'type': 'string'}
+                ],
+            },
+            'description': 'The version of the package',
+        },
+    },
+    'required': ['package', 'version'],
+}
+
+
+PACKAGE_FILE_CONTRACT = {
+    'type': 'object',
+    'additionalProperties': False,
+    'properties': {
+        'packages': {
+            'type': 'array',
+            'items': {
+                'anyOf': [
+                    LOCAL_PACKAGE_CONTRACT,
+                    GIT_PACKAGE_CONTRACT,
+                    REGISTRY_PACKAGE_CONTRACT,
+                ],
+            },
+        },
+    },
+    'required': ['packages'],
+}
+
+
+# the metadata from the registry has extra things that we don't care about.
+REGISTRY_PACKAGE_METADATA_CONTRACT = deep_merge(
+    PACKAGE_FILE_CONTRACT,
+    {
+        'additionalProperties': True,
+        'properties': {
+            'name': {
+                'type': 'string',
+            },
+            'downloads': {
+                'type': 'object',
+                'additionalProperties': True,
+                'properties': {
+                    'tarball': {
+                        'type': 'string',
+                    },
+                },
+                'required': ['tarball']
+            },
+        },
+        'required': PACKAGE_FILE_CONTRACT['required'][:] + ['downloads']
+    }
+)
+
+
+class PackageConfig(APIObject):
+    SCHEMA = PACKAGE_FILE_CONTRACT
+
+
+USER_CONFIG_CONTRACT = {
+    'type': 'object',
+    'additionalProperties': True,
+    'properties': {
+        'send_anonymous_usage_stats': {
+            'type': 'boolean',
+        },
+        'use_colors': {
+            'type': 'boolean',
+        },
+    },
+}
+
+
+PROFILE_INFO_CONTRACT = {
+    'type': 'object',
+    'additionalProperties': False,
+    'properties': {
+        'profile_name': {
+            'type': 'string',
+        },
+        'target_name': {
+            'type': 'string',
+        },
+        'config': USER_CONFIG_CONTRACT,
+        'threads': {
+            'type': 'number',
+        },
+        'credentials': {
+            'type': 'object',
+            'additionalProperties': True,
+        },
+    },
+    'required': [
+        'profile_name', 'target_name', 'config', 'threads', 'credentials'
+    ],
+}
+
+
+class ProfileConfig(APIObject):
+    SCHEMA = PROFILE_INFO_CONTRACT
+
+
+def _merge_requirements(base, *args):
+    required = base[:]
+    for arg in args:
+        required.extend(arg['required'])
+    return required
+
+
+CONFIG_CONTRACT = deep_merge(
+    PROJECT_CONTRACT,
+    PACKAGE_FILE_CONTRACT,
+    PROFILE_INFO_CONTRACT,
+    {
+        'properties': {
+            'cli_vars': {
+                'type': 'object',
+                'additionalProperties': True,
+            },
+            # override quoting: both 'identifier' and 'schema' must be
+            # populated
+            'quoting': {
+                'required': ['identifier', 'schema'],
+            },
+        },
+        'required': _merge_requirements(
+            ['cli_vars'],
+            PROJECT_CONTRACT,
+            PACKAGE_FILE_CONTRACT,
+            PROFILE_INFO_CONTRACT
+        ),
+    },
+)
+
+
+def update_config_contract(typename, connection):
+    PROFILE_INFO_CONTRACT['properties']['credentials']['anyOf'].append(
+        connection.SCHEMA
+    )
+    CONFIG_CONTRACT['properties']['credentials']['anyOf'].append(
+        connection.SCHEMA
+    )
+
+
+class Configuration(APIObject):
+    SCHEMA = CONFIG_CONTRACT
+
+
+PROJECTS_LIST_PROJECT = {
+    'type': 'object',
+    'additionalProperties': False,
+    'patternProperties': {
+        '.*': CONFIG_CONTRACT,
+    },
+}
+
+
+class ProjectList(APIObject):
+    SCHEMA = PROJECTS_LIST_PROJECT
+
+    def serialize(self):
+        return {k: v.serialize() for k, v in self._contents.items()}

core/dbt/contracts/results.py

@@ -0,0 +1,544 @@
+from dbt.api.object import APIObject
+from dbt.utils import deep_merge, timestring
+from dbt.contracts.common import named_property
+from dbt.contracts.graph.manifest import COMPILE_RESULT_NODE_CONTRACT
+from dbt.contracts.graph.unparsed import TIME_CONTRACT
+from dbt.contracts.graph.parsed import PARSED_SOURCE_DEFINITION_CONTRACT
+
+
+TIMING_INFO_CONTRACT = {
+    'type': 'object',
+    'properties': {
+        'name': {
+            'type': 'string',
+        },
+        'started_at': {
+            'type': 'string',
+            'format': 'date-time',
+        },
+        'completed_at': {
+            'type': 'string',
+            'format': 'date-time',
+        },
+    }
+}
+
+
+class TimingInfo(APIObject):
+
+    SCHEMA = TIMING_INFO_CONTRACT
+
+    @classmethod
+    def create(cls, name):
+        return cls(name=name)
+
+    def begin(self):
+        self.set('started_at', timestring())
+
+    def end(self):
+        self.set('completed_at', timestring())
+
+
+class collect_timing_info:
+    def __init__(self, name):
+        self.timing_info = TimingInfo.create(name)
+
+    def __enter__(self):
+        self.timing_info.begin()
+        return self.timing_info
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        self.timing_info.end()
+
+
+class NodeSerializable(APIObject):
+
+    def serialize(self):
+        result = super(NodeSerializable, self).serialize()
+        result['node'] = self.node.serialize()
+        return result
+
+
+PARTIAL_RESULT_CONTRACT = {
+    'type': 'object',
+    'additionalProperties': False,
+    'description': 'The partial result of a single node being run',
+    'properties': {
+        'error': {
+            'type': ['string', 'null'],
+            'description': 'The error string, or None if there was no error',
+        },
+        'status': {
+            'type': ['string', 'null', 'number', 'boolean'],
+            'description': 'The status result of the node execution',
+        },
+        'execution_time': {
+            'type': 'number',
+            'description': 'The execution time, in seconds',
+        },
+        'thread_id': {
+            'type': ['string', 'null'],
+            'description': 'ID of the executing thread, e.g. Thread-3',
+        },
+        'timing': {
+            'type': 'array',
+            'items': TIMING_INFO_CONTRACT,
+        },
+        'node': COMPILE_RESULT_NODE_CONTRACT,
+    },
+    'required': ['node', 'status', 'error', 'execution_time', 'thread_id',
+                 'timing'],
+}
+
+
+class PartialResult(NodeSerializable):
+    """Represent a "partial" execution result, i.e. one that has not (fully)
+    been executed.
+
+    This may be an ephemeral node (they are not compiled) or any error.
+    """
+    SCHEMA = PARTIAL_RESULT_CONTRACT
+
+    def __init__(self, node, error=None, status=None, execution_time=0,
+                 thread_id=None, timing=None):
+        if timing is None:
+            timing = []
+        super(PartialResult, self).__init__(
+            node=node,
+            error=error,
+            status=status,
+            execution_time=execution_time,
+            thread_id=thread_id,
+            timing=timing,
+        )
+
+    # if the result got to the point where it could be skipped/failed, we would
+    # be returning a real result, not a partial.
+    @property
+    def skipped(self):
+        return False
+
+    @property
+    def failed(self):
+        return None
+
+
+RUN_MODEL_RESULT_CONTRACT = deep_merge(PARTIAL_RESULT_CONTRACT, {
+    'description': 'The result of a single node being run',
+    'properties': {
+        'skip': {
+            'type': 'boolean',
+            'description': 'True if this node was skipped',
+        },
+        'warn': {
+            'type': ['boolean', 'null'],
+            'description': 'True if this node succeeded with a warning',
+        },
+        'fail': {
+            'type': ['boolean', 'null'],
+            'description': 'On tests, true if the test failed',
+        },
+    },
+    'required': ['skip', 'fail', 'warn']
+})
+
+
+class RunModelResult(NodeSerializable):
+    SCHEMA = RUN_MODEL_RESULT_CONTRACT
+
+    def __init__(self, node, error=None, skip=False, status=None, failed=None,
+                 warned=None, thread_id=None, timing=None, execution_time=0):
+        if timing is None:
+            timing = []
+        super(RunModelResult, self).__init__(
+            node=node,
+            error=error,
+            skip=skip,
+            status=status,
+            fail=failed,
+            warn=warned,
+            execution_time=execution_time,
+            thread_id=thread_id,
+            timing=timing,
+        )
+
+    # these all get set after the fact, generally
+    error = named_property('error',
+                           'If there was an error, the text of that error')
+    skip = named_property('skip', 'True if the model was skipped')
+    warn = named_property('warn', 'True if this was a test and it warned')
+    fail = named_property('fail', 'True if this was a test and it failed')
+    status = named_property('status', 'The status of the model execution')
+    execution_time = named_property('execution_time',
+                                    'The time in seconds to execute the model')
+    thread_id = named_property(
+        'thread_id',
+        'ID of the executing thread, e.g. Thread-3'
+    )
+    timing = named_property(
+        'timing',
+        'List of TimingInfo objects'
+    )
+
+    @property
+    def failed(self):
+        return self.fail
+
+    @property
+    def warned(self):
+        return self.warn
+
+    @property
+    def skipped(self):
+        return self.skip
+
+
+EXECUTION_RESULT_CONTRACT = {
+    'type': 'object',
+    'additionalProperties': False,
+    'description': 'The result of a single dbt invocation',
+    'properties': {
+        'results': {
+            'type': 'array',
+            'items': {
+                'anyOf': [
+                    RUN_MODEL_RESULT_CONTRACT,
+                    PARTIAL_RESULT_CONTRACT,
+                ]
+            },
+            'description': 'An array of results, one per model',
+        },
+        'generated_at': {
+            'type': 'string',
+            'format': 'date-time',
+            'description': (
+                'The time at which the execution result was generated'
+            ),
+        },
+        'elapsed_time': {
+            'type': 'number',
+            'description': (
+                'The time elapsed from before_run to after_run (hooks are not '
+                'included)'
+            ),
+        }
+    },
+    'required': ['results', 'generated_at', 'elapsed_time'],
+}
+
+
+class ExecutionResult(APIObject):
+    SCHEMA = EXECUTION_RESULT_CONTRACT
+
+    def serialize(self):
+        return {
+            'results': [r.serialize() for r in self.results],
+            'generated_at': self.generated_at,
+            'elapsed_time': self.elapsed_time,
+        }
+
+
+SOURCE_FRESHNESS_RESULT_CONTRACT = deep_merge(PARTIAL_RESULT_CONTRACT, {
+    'properties': {
+        'max_loaded_at': {
+            'type': 'string',
+            'format': 'date-time',
+        },
+        'snapshotted_at': {
+            'type': 'string',
+            'format': 'date-time',
+        },
+        'age': {
+            'type': 'number',
+        },
+        'status': {
+            'enum': ['pass', 'warn', 'error']
+        },
+        'node': PARSED_SOURCE_DEFINITION_CONTRACT,
+    },
+    'required': ['max_loaded_at', 'snapshotted_at', 'age']
+})
+
+
+class SourceFreshnessResult(NodeSerializable):
+    SCHEMA = SOURCE_FRESHNESS_RESULT_CONTRACT
+
+    def __init__(self, node, max_loaded_at, snapshotted_at,
+                 age, status, thread_id, error=None,
+                 timing=None, execution_time=0):
+        max_loaded_at = max_loaded_at.isoformat()
+        snapshotted_at = snapshotted_at.isoformat()
+        if timing is None:
+            timing = []
+        super(SourceFreshnessResult, self).__init__(
+            node=node,
+            max_loaded_at=max_loaded_at,
+            snapshotted_at=snapshotted_at,
+            age=age,
+            status=status,
+            thread_id=thread_id,
+            error=error,
+            timing=timing,
+            execution_time=execution_time
+        )
+
+    @property
+    def failed(self):
+        return self.status == 'error'
+
+    @property
+    def warned(self):
+        return self.status == 'warn'
+
+    @property
+    def skipped(self):
+        return False
+
+
+FRESHNESS_METADATA_CONTRACT = {
+    'type': 'object',
+    'additionalProperties': False,
+    'properties': {
+        'generated_at': {
+            'type': 'string',
+            'format': 'date-time',
+            'description': (
+                'The time at which the execution result was generated'
+            ),
+        },
+        'elapsed_time': {
+            'type': 'number',
+            'description': (
+                'The time elapsed from before_run to after_run (hooks '
+                'are not included)'
+            ),
+        },
+    },
+    'required': ['generated_at', 'elapsed_time']
+}
+
+
+FRESHNESS_RESULTS_CONTRACT = deep_merge(FRESHNESS_METADATA_CONTRACT, {
+    'description': 'The result of a single dbt source freshness invocation',
+    'properties': {
+        'results': {
+            'type': 'array',
+            'items': {
+                'anyOf': [
+                    PARTIAL_RESULT_CONTRACT,
+                    SOURCE_FRESHNESS_RESULT_CONTRACT,
+                ],
+            },
+        },
+    },
+    'required': ['results'],
+})
+
+
+class FreshnessExecutionResult(APIObject):
+    SCHEMA = FRESHNESS_RESULTS_CONTRACT
+
+    def __init__(self, elapsed_time, generated_at, results):
+        super(FreshnessExecutionResult, self).__init__(
+            elapsed_time=elapsed_time,
+            generated_at=generated_at,
+            results=results
+        )
+
+    def serialize(self):
+        return {
+            'generated_at': self.generated_at,
+            'elapsed_time': self.elapsed_time,
+            'results': [s.serialize() for s in self.results]
+        }
+
+    def write(self, path):
+        """Create a new object with the desired output schema and write it."""
+        meta = {
+            'generated_at': self.generated_at,
+            'elapsed_time': self.elapsed_time,
+        }
+        sources = {}
+        for result in self.results:
+            unique_id = result.node.unique_id
+            if result.error is not None:
+                result_dict = {
+                    'error': result.error,
+                    'state': 'runtime error'
+                }
+            else:
+                result_dict = {
+                    'max_loaded_at': result.max_loaded_at,
+                    'snapshotted_at': result.snapshotted_at,
+                    'max_loaded_at_time_ago_in_s': result.age,
+                    'state': result.status,
+                    'criteria': result.node.freshness,
+                }
+            sources[unique_id] = result_dict
+        output = FreshnessRunOutput(meta=meta, sources=sources)
+        output.write(path)
+
+
+def _copykeys(src, keys, **updates):
+    return {k: getattr(src, k) for k in keys}
+
+
+SOURCE_FRESHNESS_OUTPUT_ERROR_CONTRACT = {
+    'type': 'object',
+    'additionalProperties': False,
+    'description': (
+        'The source freshness output for a single source table',
+    ),
+    'properties': {
+        'error': {
+            'type': 'string',
+            'description': 'The error string',
+        },
+        'state': {
+            'enum': ['runtime error'],
+        },
+    }
+}
+
+
+SOURCE_FRESHNESS_OUTPUT_CONTRACT = {
+    'type': 'object',
+    'additionalProperties': False,
+    'description': (
+        'The source freshness output for a single source table',
+    ),
+    'properties': {
+        'max_loaded_at': {
+            'type': 'string',
+            'format': 'date-time',
+        },
+        'snapshotted_at': {
+            'type': 'string',
+            'format': 'date-time',
+        },
+        'max_loaded_at_time_ago_in_s': {
+            'type': 'number',
+        },
+        'state': {
+            'enum': ['pass', 'warn', 'error']
+        },
+        'criteria': {
+            'type': 'object',
+            'additionalProperties': False,
+            'properties': {
+                'warn_after': TIME_CONTRACT,
+                'error_after': TIME_CONTRACT,
+            },
+        },
+        'required': ['state', 'criteria', 'max_loaded_at', 'snapshotted_at',
+                     'max_loaded_at_time_ago_in_s']
+    }
+}
+
+
+FRESHNESS_RUN_OUTPUT_CONTRACT = {
+    'type': 'object',
+    'additionalProperties': False,
+    'description': 'The output contract for dbt source freshness invocations',
+    'properties': {
+        'meta': FRESHNESS_METADATA_CONTRACT,
+        'sources': {
+            'type': 'object',
+            'additionalProperties': False,
+            'description': (
+                'A collection of the source results, stored by their unique '
+                'IDs.'
+            ),
+            'patternProperties': {
+                '.*': {
+                    'anyOf': [
+                        SOURCE_FRESHNESS_OUTPUT_ERROR_CONTRACT,
+                        SOURCE_FRESHNESS_OUTPUT_CONTRACT
+                    ],
+                },
+            },
+        }
+    }
+}
+
+
+class FreshnessRunOutput(APIObject):
+    SCHEMA = FRESHNESS_RUN_OUTPUT_CONTRACT
+
+    def __init__(self, meta, sources):
+        super(FreshnessRunOutput, self).__init__(meta=meta, sources=sources)
+
+
+REMOTE_COMPILE_RESULT_CONTRACT = {
+    'type': 'object',
+    'additionalProperties': False,
+    'properties': {
+        'raw_sql': {
+            'type': 'string',
+        },
+        'compiled_sql': {
+            'type': 'string',
+        },
+        'timing': {
+            'type': 'array',
+            'items': TIMING_INFO_CONTRACT,
+        },
+    },
+    'required': ['raw_sql', 'compiled_sql', 'timing']
+}
+
+
+class RemoteCompileResult(APIObject):
+    SCHEMA = REMOTE_COMPILE_RESULT_CONTRACT
+
+    def __init__(self, raw_sql, compiled_sql, node, timing=None, **kwargs):
+        if timing is None:
+            timing = []
+        # this should not show up in the serialized output.
+        self.node = node
+        super(RemoteCompileResult, self).__init__(
+            raw_sql=raw_sql,
+            compiled_sql=compiled_sql,
+            timing=timing,
+            **kwargs
+        )
+
+    @property
+    def error(self):
+        return None
+
+
+REMOTE_RUN_RESULT_CONTRACT = deep_merge(REMOTE_COMPILE_RESULT_CONTRACT, {
+    'properties': {
+        'table': {
+            'type': 'object',
+            'properties': {
+                'column_names': {
+                    'type': 'array',
+                    'items': {'type': 'string'},
+                },
+                'rows': {
+                    'type': 'array',
+                    # any item type is ok
+                },
+            },
+            'required': ['rows', 'column_names'],
+        },
+    },
+    'required': ['table'],
+})
+
+
+class RemoteRunResult(RemoteCompileResult):
+    SCHEMA = REMOTE_RUN_RESULT_CONTRACT
+
+    def __init__(self, raw_sql, compiled_sql, node, timing=None, table=None):
+        if table is None:
+            table = []
+        super(RemoteRunResult, self).__init__(
+            raw_sql=raw_sql,
+            compiled_sql=compiled_sql,
+            timing=timing,
+            table=table,
+            node=node
+        )

core/dbt/deprecations.py

@@ -0,0 +1,97 @@
+import dbt.links
+import dbt.flags
+
+
+class DBTDeprecation(object):
+    name = None
+    description = None
+
+    def show(self, *args, **kwargs):
+        if self.name not in active_deprecations:
+            desc = self.description.format(**kwargs)
+            dbt.exceptions.warn_or_error(
+                "* Deprecation Warning: {}\n".format(desc)
+            )
+            active_deprecations.add(self.name)
+
+
+class DBTRepositoriesDeprecation(DBTDeprecation):
+    name = "repositories"
+    description = """The dbt_project.yml configuration option 'repositories' is
+  deprecated. Please place dependencies in the `packages.yml` file instead.
+  The 'repositories' option will be removed in a future version of dbt.
+
+  For more information, see: https://docs.getdbt.com/docs/package-management
+
+  # Example packages.yml contents:
+
+{recommendation}
+  """
+
+
+class GenerateSchemaNameSingleArgDeprecated(DBTDeprecation):
+    name = 'generate-schema-name-single-arg'
+    description = '''As of dbt v0.14.0, the `generate_schema_name` macro
+  accepts a second "node" argument. The one-argument form of `generate_schema_name`
+  is deprecated, and will become unsupported in a future release.
+
+  For more information, see:
+    https://docs.getdbt.com/v0.14/docs/upgrading-to-014
+  '''  # noqa
+
+
+class ArchiveDeprecated(DBTDeprecation):
+    name = 'archives'
+    description = '''As of dbt v0.14.0, the `dbt archive` command is renamed to
+  `dbt snapshot` and "archives" are "snapshots". The `dbt archive` command will
+  be removed in a future release.
+
+  For more information, see:
+    https://docs.getdbt.com/v0.14/docs/upgrading-to-014
+  '''
+
+
+_adapter_renamed_description = """\
+The adapter function `adapter.{old_name}` is deprecated and will be removed in
+ a future release of dbt. Please use `adapter.{new_name}` instead.
+ Documentation for {new_name} can be found here:
+ https://docs.getdbt.com/docs/adapter"""
+
+
+def renamed_method(old_name, new_name):
+    class AdapterDeprecationWarning(DBTDeprecation):
+        name = 'adapter:{}'.format(old_name)
+        description = _adapter_renamed_description.format(old_name=old_name,
+                                                          new_name=new_name)
+
+    dep = AdapterDeprecationWarning()
+    deprecations_list.append(dep)
+    deprecations[dep.name] = dep
+
+
+def warn(name, *args, **kwargs):
+    if name not in deprecations:
+        # this should (hopefully) never happen
+        raise RuntimeError(
+            "Error showing deprecation warning: {}".format(name)
+        )
+
+    deprecations[name].show(*args, **kwargs)
+
+
+# these are globally available
+# since modules are only imported once, active_deprecations is a singleton
+
+active_deprecations = set()
+
+deprecations_list = [
+    DBTRepositoriesDeprecation(),
+    GenerateSchemaNameSingleArgDeprecated(),
+    ArchiveDeprecated(),
+]
+
+deprecations = {d.name: d for d in deprecations_list}
+
+
+def reset_deprecations():
+    active_deprecations.clear()

core/dbt/exceptions.py

@@ -0,0 +1,718 @@
+import sys
+import six
+import functools
+
+from dbt.compat import builtins
+from dbt.logger import GLOBAL_LOGGER as logger
+import dbt.flags
+
+
+class Exception(builtins.Exception):
+    CODE = -32000
+    MESSAGE = "Server Error"
+
+    def data(self):
+        # if overriding, make sure the result is json-serializable.
+        return {
+            'type': self.__class__.__name__,
+            'message': str(self),
+        }
+
+
+class MacroReturn(builtins.BaseException):
+    """
+    Hack of all hacks
+    """
+
+    def __init__(self, value):
+        self.value = value
+
+
+class InternalException(Exception):
+    pass
+
+
+class RuntimeException(RuntimeError, Exception):
+    CODE = 10001
+    MESSAGE = "Runtime error"
+
+    def __init__(self, msg, node=None):
+        self.stack = []
+        self.node = node
+        self.msg = msg
+
+    @property
+    def type(self):
+        return 'Runtime'
+
+    def node_to_string(self, node):
+        if node is None:
+            return "<Unknown>"
+
+        return "{} {} ({})".format(
+            node.get('resource_type'),
+            node.get('name', 'unknown'),
+            node.get('original_file_path'))
+
+    def process_stack(self):
+        lines = []
+        stack = self.stack + [self.node]
+        first = True
+
+        if len(stack) > 1:
+            lines.append("")
+
+            for item in stack:
+                msg = 'called by'
+
+                if first:
+                    msg = 'in'
+                    first = False
+
+                lines.append("> {} {}".format(
+                    msg,
+                    self.node_to_string(item)))
+
+        return lines
+
+    def __str__(self, prefix="! "):
+        node_string = ""
+
+        if self.node is not None:
+            node_string = " in {}".format(self.node_to_string(self.node))
+
+        if hasattr(self.msg, 'split'):
+            split_msg = self.msg.split("\n")
+        else:
+            # can't use basestring here, as on python2 it's an abstract class
+            split_msg = str(self.msg).split("\n")
+
+        lines = ["{}{}".format(self.type + ' Error',
+                               node_string)] + split_msg
+
+        lines += self.process_stack()
+
+        return lines[0] + "\n" + "\n".join(
+            ["  " + line for line in lines[1:]])
+
+    def data(self):
+        result = Exception.data(self)
+        if self.node is None:
+            return result
+
+        result.update({
+            'raw_sql': self.node.get('raw_sql'),
+            'compiled_sql': self.node.get('injected_sql'),
+        })
+        return result
+
+
+class RPCFailureResult(RuntimeException):
+    CODE = 10002
+    MESSAGE = "RPC execution error"
+
+
+class RPCTimeoutException(RuntimeException):
+    CODE = 10008
+    MESSAGE = 'RPC timeout error'
+
+    def __init__(self, timeout):
+        super(RPCTimeoutException, self).__init__(self.MESSAGE)
+        self.timeout = timeout
+
+    def data(self):
+        result = super(RPCTimeoutException, self).data()
+        result.update({
+            'timeout': self.timeout,
+            'message': 'RPC timed out after {}s'.format(self.timeout),
+        })
+        return result
+
+
+class RPCKilledException(RuntimeException):
+    CODE = 10009
+    MESSAGE = 'RPC process killed'
+
+    def __init__(self, signum):
+        self.signum = signum
+        self.message = 'RPC process killed by signal {}'.format(self.signum)
+        super(RPCKilledException, self).__init__(self.message)
+
+    def data(self):
+        return {
+            'signum': self.signum,
+            'message': self.message,
+        }
+
+
+class DatabaseException(RuntimeException):
+    CODE = 10003
+    MESSAGE = "Database Error"
+
+    def process_stack(self):
+        lines = []
+
+        if self.node is not None and self.node.get('build_path'):
+            lines.append(
+                "compiled SQL at {}".format(self.node.get('build_path')))
+
+        return lines + RuntimeException.process_stack(self)
+
+    @property
+    def type(self):
+        return 'Database'
+
+
+class CompilationException(RuntimeException):
+    CODE = 10004
+    MESSAGE = "Compilation Error"
+
+    @property
+    def type(self):
+        return 'Compilation'
+
+
+class RecursionException(RuntimeException):
+    pass
+
+
+class ValidationException(RuntimeException):
+    CODE = 10005
+    MESSAGE = "Validation Error"
+
+
+class JSONValidationException(ValidationException):
+    def __init__(self, typename, errors):
+        self.typename = typename
+        self.errors = errors
+        self.errors_message = ', '.join(errors)
+        msg = 'Invalid arguments passed to "{}" instance: {}'.format(
+            self.typename, self.errors_message
+        )
+        super(JSONValidationException, self).__init__(msg)
+
+    def __reduce__(self):
+        # see https://stackoverflow.com/a/36342588 for why this is necessary
+        return (JSONValidationException, (self.typename, self.errors))
+
+
+class AliasException(ValidationException):
+    pass
+
+
+class DependencyException(Exception):
+    # this can happen due to raise_dependency_error and its callers
+    CODE = 10006
+    MESSAGE = "Dependency Error"
+
+
+class DbtConfigError(RuntimeException):
+    CODE = 10007
+    MESSAGE = "DBT Configuration Error"
+
+    def __init__(self, message, project=None, result_type='invalid_project'):
+        self.project = project
+        super(DbtConfigError, self).__init__(message)
+        self.result_type = result_type
+
+
+class DbtProjectError(DbtConfigError):
+    pass
+
+
+class DbtProfileError(DbtConfigError):
+    pass
+
+
+class SemverException(Exception):
+    def __init__(self, msg=None):
+        self.msg = msg
+        if msg is not None:
+            super(SemverException, self).__init__(msg)
+
+
+class VersionsNotCompatibleException(SemverException):
+    pass
+
+
+class NotImplementedException(Exception):
+    pass
+
+
+class FailedToConnectException(DatabaseException):
+    pass
+
+
+class CommandError(RuntimeException):
+    def __init__(self, cwd, cmd, message='Error running command'):
+        super(CommandError, self).__init__(message)
+        self.cwd = cwd
+        self.cmd = cmd
+        self.args = (cwd, cmd, message)
+
+    def __str__(self):
+        if len(self.cmd) == 0:
+            return '{}: No arguments given'.format(self.msg)
+        return '{}: "{}"'.format(self.msg, self.cmd[0])
+
+
+class ExecutableError(CommandError):
+    def __init__(self, cwd, cmd, message):
+        super(ExecutableError, self).__init__(cwd, cmd, message)
+
+
+class WorkingDirectoryError(CommandError):
+    def __init__(self, cwd, cmd, message):
+        super(WorkingDirectoryError, self).__init__(cwd, cmd, message)
+
+    def __str__(self):
+        return '{}: "{}"'.format(self.msg, self.cwd)
+
+
+class CommandResultError(CommandError):
+    def __init__(self, cwd, cmd, returncode, stdout, stderr,
+                 message='Got a non-zero returncode'):
+        super(CommandResultError, self).__init__(cwd, cmd, message)
+        self.returncode = returncode
+        self.stdout = stdout
+        self.stderr = stderr
+        self.args = (cwd, cmd, returncode, stdout, stderr, message)
+
+    def __str__(self):
+        return '{} running: {}'.format(self.msg, self.cmd)
+
+
+def raise_compiler_error(msg, node=None):
+    raise CompilationException(msg, node)
+
+
+def raise_database_error(msg, node=None):
+    raise DatabaseException(msg, node)
+
+
+def raise_dependency_error(msg):
+    raise DependencyException(msg)
+
+
+def invalid_type_error(method_name, arg_name, got_value, expected_type,
+                       version='0.13.0'):
+    """Raise a CompilationException when an adapter method available to macros
+    has changed.
+    """
+    got_type = type(got_value)
+    msg = ("As of {version}, 'adapter.{method_name}' expects argument "
+           "'{arg_name}' to be of type '{expected_type}', instead got "
+           "{got_value} ({got_type})")
+    raise_compiler_error(msg.format(version=version, method_name=method_name,
+                         arg_name=arg_name, expected_type=expected_type,
+                         got_value=got_value, got_type=got_type))
+
+
+def ref_invalid_args(model, args):
+    raise_compiler_error(
+        "ref() takes at most two arguments ({} given)".format(len(args)),
+        model)
+
+
+def ref_bad_context(model, args):
+    ref_args = ', '.join("'{}'".format(a) for a in args)
+    ref_string = '{{{{ ref({}) }}}}'.format(ref_args)
+
+    base_error_msg = """dbt was unable to infer all dependencies for the model "{model_name}".
+This typically happens when ref() is placed within a conditional block.
+
+To fix this, add the following hint to the top of the model "{model_name}":
+
+-- depends_on: {ref_string}"""
+    # This explicitly references model['name'], instead of model['alias'], for
+    # better error messages. Ex. If models foo_users and bar_users are aliased
+    # to 'users', in their respective schemas, then you would want to see
+    # 'bar_users' in your error messge instead of just 'users'.
+    error_msg = base_error_msg.format(
+        model_name=model['name'],
+        model_path=model['path'],
+        ref_string=ref_string
+    )
+    raise_compiler_error(error_msg, model)
+
+
+def doc_invalid_args(model, args):
+    raise_compiler_error(
+        "doc() takes at most two arguments ({} given)".format(len(args)),
+        model)
+
+
+def doc_target_not_found(model, target_doc_name, target_doc_package):
+    target_package_string = ''
+
+    if target_doc_package is not None:
+        target_package_string = "in package '{}' ".format(target_doc_package)
+
+    msg = (
+        "Documentation for '{}' depends on doc '{}' {} which was not found"
+    ).format(
+        model.get('unique_id'),
+        target_doc_name,
+        target_package_string
+    )
+    raise_compiler_error(msg, model)
+
+
+def _get_target_failure_msg(model, target_model_name, target_model_package,
+                            include_path, reason):
+    target_package_string = ''
+    if target_model_package is not None:
+        target_package_string = "in package '{}' ".format(target_model_package)
+
+    source_path_string = ''
+    if include_path:
+        source_path_string = ' ({})'.format(model.get('original_file_path'))
+
+    return ("{} '{}'{} depends on model '{}' {}which {}"
+            .format(model.get('resource_type').title(),
+                    model.get('unique_id'),
+                    source_path_string,
+                    target_model_name,
+                    target_package_string,
+                    reason))
+
+
+def get_target_disabled_msg(model, target_model_name, target_model_package):
+    return _get_target_failure_msg(model, target_model_name,
+                                   target_model_package, include_path=True,
+                                   reason='is disabled')
+
+
+def get_target_not_found_msg(model, target_model_name, target_model_package):
+    return _get_target_failure_msg(model, target_model_name,
+                                   target_model_package, include_path=True,
+                                   reason='was not found')
+
+
+def get_target_not_found_or_disabled_msg(model, target_model_name,
+                                         target_model_package):
+    return _get_target_failure_msg(model, target_model_name,
+                                   target_model_package, include_path=False,
+                                   reason='was not found or is disabled')
+
+
+def ref_target_not_found(model, target_model_name, target_model_package):
+    msg = get_target_not_found_or_disabled_msg(model, target_model_name,
+                                               target_model_package)
+    raise_compiler_error(msg, model)
+
+
+def source_disabled_message(model, target_name, target_table_name):
+    return ("{} '{}' ({}) depends on source '{}.{}' which was not found"
+            .format(model.get('resource_type').title(),
+                    model.get('unique_id'),
+                    model.get('original_file_path'),
+                    target_name,
+                    target_table_name))
+
+
+def source_target_not_found(model, target_name, target_table_name):
+    msg = source_disabled_message(model, target_name, target_table_name)
+    raise_compiler_error(msg, model)
+
+
+def ref_disabled_dependency(model, target_model):
+    raise_compiler_error(
+        "Model '{}' depends on model '{}' which is disabled in "
+        "the project config".format(model.get('unique_id'),
+                                    target_model.get('unique_id')),
+        model)
+
+
+def dependency_not_found(model, target_model_name):
+    raise_compiler_error(
+        "'{}' depends on '{}' which is not in the graph!"
+        .format(model.get('unique_id'), target_model_name),
+        model)
+
+
+def macro_not_found(model, target_macro_id):
+    raise_compiler_error(
+        model,
+        "'{}' references macro '{}' which is not defined!"
+        .format(model.get('unique_id'), target_macro_id))
+
+
+def materialization_not_available(model, adapter_type):
+    from dbt.utils import get_materialization  # noqa
+    materialization = get_materialization(model)
+
+    raise_compiler_error(
+        "Materialization '{}' is not available for {}!"
+        .format(materialization, adapter_type),
+        model)
+
+
+def missing_materialization(model, adapter_type):
+    from dbt.utils import get_materialization  # noqa
+    materialization = get_materialization(model)
+
+    valid_types = "'default'"
+
+    if adapter_type != 'default':
+        valid_types = "'default' and '{}'".format(adapter_type)
+
+    raise_compiler_error(
+        "No materialization '{}' was found for adapter {}! (searched types {})"
+        .format(materialization, adapter_type, valid_types),
+        model)
+
+
+def bad_package_spec(repo, spec, error_message):
+    raise InternalException(
+        "Error checking out spec='{}' for repo {}\n{}".format(
+            spec, repo, error_message))
+
+
+def raise_cache_inconsistent(message):
+    raise InternalException('Cache inconsistency detected: {}'.format(message))
+
+
+def missing_config(model, name):
+    raise_compiler_error(
+        "Model '{}' does not define a required config parameter '{}'."
+        .format(model.get('unique_id'), name),
+        model)
+
+
+def missing_relation(relation, model=None):
+    raise_compiler_error(
+        "Relation {} not found!".format(relation),
+        model)
+
+
+def relation_wrong_type(relation, expected_type, model=None):
+    raise_compiler_error(
+        ('Trying to create {expected_type} {relation}, '
+         'but it currently exists as a {current_type}. Either '
+         'drop {relation} manually, or run dbt with '
+         '`--full-refresh` and dbt will drop it for you.')
+        .format(relation=relation,
+                current_type=relation.type,
+                expected_type=expected_type),
+        model)
+
+
+def package_not_found(package_name):
+    raise_dependency_error(
+        "Package {} was not found in the package index".format(package_name))
+
+
+def package_version_not_found(package_name, version_range, available_versions):
+    base_msg = ('Could not find a matching version for package {}\n'
+                '  Requested range: {}\n'
+                '  Available versions: {}')
+    raise_dependency_error(base_msg.format(package_name,
+                                           version_range,
+                                           available_versions))
+
+
+def invalid_materialization_argument(name, argument):
+    raise_compiler_error(
+        "materialization '{}' received unknown argument '{}'."
+        .format(name, argument))
+
+
+def system_error(operation_name):
+    raise_compiler_error(
+        "dbt encountered an error when attempting to {}. "
+        "If this error persists, please create an issue at: \n\n"
+        "https://github.com/fishtown-analytics/dbt"
+        .format(operation_name))
+
+
+class RegistryException(Exception):
+    pass
+
+
+def raise_dep_not_found(node, node_description, required_pkg):
+    raise_compiler_error(
+        'Error while parsing {}.\nThe required package "{}" was not found. '
+        'Is the package installed?\nHint: You may need to run '
+        '`dbt deps`.'.format(node_description, required_pkg), node=node)
+
+
+def multiple_matching_relations(kwargs, matches):
+    raise_compiler_error(
+        'get_relation returned more than one relation with the given args. '
+        'Please specify a database or schema to narrow down the result set.'
+        '\n{}\n\n{}'
+        .format(kwargs, matches))
+
+
+def get_relation_returned_multiple_results(kwargs, matches):
+    multiple_matching_relations(kwargs, matches)
+
+
+def approximate_relation_match(target, relation):
+    raise_compiler_error(
+        'When searching for a relation, dbt found an approximate match. '
+        'Instead of guessing \nwhich relation to use, dbt will move on. '
+        'Please delete {relation}, or rename it to be less ambiguous.'
+        '\nSearched for: {target}\nFound: {relation}'
+        .format(target=target,
+                relation=relation))
+
+
+def raise_duplicate_resource_name(node_1, node_2):
+    duped_name = node_1['name']
+
+    raise_compiler_error(
+        'dbt found two resources with the name "{}". Since these resources '
+        'have the same name,\ndbt will be unable to find the correct resource '
+        'when ref("{}") is used. To fix this,\nchange the name of one of '
+        'these resources:\n- {} ({})\n- {} ({})'.format(
+            duped_name,
+            duped_name,
+            node_1['unique_id'], node_1['original_file_path'],
+            node_2['unique_id'], node_2['original_file_path']))
+
+
+def raise_ambiguous_alias(node_1, node_2):
+    duped_name = "{}.{}".format(node_1['schema'], node_1['alias'])
+
+    raise_compiler_error(
+        'dbt found two resources with the database representation "{}".\ndbt '
+        'cannot create two resources with identical database representations. '
+        'To fix this,\nchange the "schema" or "alias" configuration of one of '
+        'these resources:\n- {} ({})\n- {} ({})'.format(
+            duped_name,
+            node_1['unique_id'], node_1['original_file_path'],
+            node_2['unique_id'], node_2['original_file_path']))
+
+
+def raise_ambiguous_catalog_match(unique_id, match_1, match_2):
+
+    def get_match_string(match):
+        return "{}.{}".format(
+            match.get('metadata', {}).get('schema'),
+            match.get('metadata', {}).get('name'))
+
+    raise_compiler_error(
+        'dbt found two relations in your warehouse with similar database '
+        'identifiers. dbt\nis unable to determine which of these relations '
+        'was created by the model "{unique_id}".\nIn order for dbt to '
+        'correctly generate the catalog, one of the following relations must '
+        'be deleted or renamed:\n\n - {match_1_s}\n - {match_2_s}'.format(
+            unique_id=unique_id,
+            match_1_s=get_match_string(match_1),
+            match_2_s=get_match_string(match_2),
+        ))
+
+
+def raise_patch_targets_not_found(patches):
+    patch_list = '\n\t'.join(
+        'model {} (referenced in path {})'.format(p.name, p.original_file_path)
+        for p in patches.values()
+    )
+    raise_compiler_error(
+        'dbt could not find models for the following patches:\n\t{}'.format(
+            patch_list
+        )
+    )
+
+
+def raise_duplicate_patch_name(name, patch_1, patch_2):
+    raise_compiler_error(
+        'dbt found two schema.yml entries for the same model named {}. The '
+        'first patch was specified in {} and the second in {}. Models and '
+        'their associated columns may only be described a single time.'.format(
+            name,
+            patch_1,
+            patch_2,
+        )
+    )
+
+
+def raise_invalid_schema_yml_version(path, issue):
+    raise_compiler_error(
+        'The schema file at {} is invalid because {}. Please consult the '
+        'documentation for more information on schema.yml syntax:\n\n'
+        'https://docs.getdbt.com/docs/schemayml-files'
+        .format(path, issue)
+    )
+
+
+def raise_unrecognized_credentials_type(typename, supported_types):
+    raise_compiler_error(
+        'Unrecognized credentials type "{}" - supported types are ({})'
+        .format(typename, ', '.join('"{}"'.format(t) for t in supported_types))
+    )
+
+
+def raise_not_implemented(msg):
+    raise NotImplementedException(
+        "ERROR: {}"
+        .format(msg))
+
+
+def warn_or_error(msg, node=None, log_fmt=None):
+    if dbt.flags.WARN_ERROR:
+        raise_compiler_error(msg, node)
+    else:
+        if log_fmt is not None:
+            msg = log_fmt.format(msg)
+        logger.warning(msg)
+
+
+def warn_or_raise(exc, log_fmt=None):
+    if dbt.flags.WARN_ERROR:
+        raise exc
+    else:
+        msg = str(exc)
+        if log_fmt is not None:
+            msg = log_fmt.format(msg)
+        logger.warning(msg)
+
+
+# Update this when a new function should be added to the
+# dbt context's `exceptions` key!
+CONTEXT_EXPORTS = {
+    fn.__name__: fn
+    for fn in
+    [
+        missing_config,
+        missing_materialization,
+        missing_relation,
+        raise_ambiguous_alias,
+        raise_ambiguous_catalog_match,
+        raise_cache_inconsistent,
+        raise_compiler_error,
+        raise_database_error,
+        raise_dep_not_found,
+        raise_dependency_error,
+        raise_duplicate_patch_name,
+        raise_duplicate_resource_name,
+        raise_invalid_schema_yml_version,
+        raise_not_implemented,
+        relation_wrong_type,
+    ]
+}
+
+
+def wrapper(model):
+    def wrap(func):
+        @functools.wraps(func)
+        def inner(*args, **kwargs):
+            try:
+                return func(*args, **kwargs)
+            except Exception:
+                exc_type, exc, exc_tb = sys.exc_info()
+                if hasattr(exc, 'node') and exc.node is None:
+                    exc.node = model
+                six.reraise(exc_type, exc, exc_tb)
+
+        return inner
+    return wrap
+
+
+def wrapped_exports(model):
+    wrap = wrapper(model)
+    return {
+        name: wrap(export) for name, export in CONTEXT_EXPORTS.items()
+    }

core/dbt/flags.py

@@ -0,0 +1,29 @@
+STRICT_MODE = False
+FULL_REFRESH = False
+USE_CACHE = True
+WARN_ERROR = False
+TEST_NEW_PARSER = False
+
+
+def reset():
+    global STRICT_MODE, FULL_REFRESH, USE_CACHE, WARN_ERROR, TEST_NEW_PARSER
+
+    STRICT_MODE = False
+    FULL_REFRESH = False
+    USE_CACHE = True
+    WARN_ERROR = False
+    TEST_NEW_PARSER = False
+
+
+def set_from_args(args):
+    global STRICT_MODE, FULL_REFRESH, USE_CACHE, WARN_ERROR, TEST_NEW_PARSER
+    USE_CACHE = getattr(args, 'use_cache', True)
+
+    FULL_REFRESH = getattr(args, 'full_refresh', False)
+    STRICT_MODE = getattr(args, 'strict', False)
+    WARN_ERROR = (
+        STRICT_MODE or
+        getattr(args, 'warn_error', False)
+    )
+
+    TEST_NEW_PARSER = getattr(args, 'test_new_parser', False)

core/dbt/graph/selector.py

@@ -0,0 +1,348 @@
+import networkx as nx
+from dbt.logger import GLOBAL_LOGGER as logger
+
+from dbt.utils import is_enabled, coalesce
+from dbt.node_types import NodeType
+import dbt.exceptions
+
+SELECTOR_PARENTS = '+'
+SELECTOR_CHILDREN = '+'
+SELECTOR_GLOB = '*'
+SELECTOR_CHILDREN_AND_ANCESTORS = '@'
+SELECTOR_DELIMITER = ':'
+
+
+class SelectionCriteria(object):
+    def __init__(self, node_spec):
+        self.raw = node_spec
+        self.select_children = False
+        self.select_parents = False
+        self.select_childrens_parents = False
+        self.selector_type = SELECTOR_FILTERS.FQN
+
+        if node_spec.startswith(SELECTOR_CHILDREN_AND_ANCESTORS):
+            self.select_childrens_parents = True
+            node_spec = node_spec[1:]
+
+        if node_spec.startswith(SELECTOR_PARENTS):
+            self.select_parents = True
+            node_spec = node_spec[1:]
+
+        if node_spec.endswith(SELECTOR_CHILDREN):
+            self.select_children = True
+            node_spec = node_spec[:-1]
+
+        if self.select_children and self.select_childrens_parents:
+            raise dbt.exceptions.RuntimeException(
+                'Invalid node spec {} - "@" prefix and "+" suffix are '
+                'incompatible'.format(self.raw)
+            )
+
+        if SELECTOR_DELIMITER in node_spec:
+            selector_parts = node_spec.split(SELECTOR_DELIMITER, 1)
+            self.selector_type, self.selector_value = selector_parts
+        else:
+            self.selector_value = node_spec
+
+
+class SELECTOR_FILTERS(object):
+    FQN = 'fqn'
+    TAG = 'tag'
+    SOURCE = 'source'
+
+
+def split_specs(node_specs):
+    specs = set()
+    for spec in node_specs:
+        parts = spec.split(" ")
+        specs.update(parts)
+
+    return specs
+
+
+def get_package_names(graph):
+    return set([node.split(".")[1] for node in graph.nodes()])
+
+
+def is_selected_node(real_node, node_selector):
+    for i, selector_part in enumerate(node_selector):
+
+        is_last = (i == len(node_selector) - 1)
+
+        # if we hit a GLOB, then this node is selected
+        if selector_part == SELECTOR_GLOB:
+            return True
+
+        # match package.node_name or package.dir.node_name
+        elif is_last and selector_part == real_node[-1]:
+            return True
+
+        elif len(real_node) <= i:
+            return False
+
+        elif real_node[i] == selector_part:
+            continue
+
+        else:
+            return False
+
+    # if we get all the way down here, then the node is a match
+    return True
+
+
+def _node_is_match(qualified_name, package_names, fqn):
+    """Determine if a qualfied name matches an fqn, given the set of package
+    names in the graph.
+
+    :param List[str] qualified_name: The components of the selector or node
+        name, split on '.'.
+    :param Set[str] package_names: The set of pacakge names in the graph.
+    :param List[str] fqn: The node's fully qualified name in the graph.
+    """
+    if len(qualified_name) == 1 and fqn[-1] == qualified_name[0]:
+        return True
+
+    if qualified_name[0] in package_names:
+        if is_selected_node(fqn, qualified_name):
+            return True
+
+    for package_name in package_names:
+        local_qualified_node_name = [package_name] + qualified_name
+        if is_selected_node(fqn, local_qualified_node_name):
+            return True
+
+    return False
+
+
+def warn_if_useless_spec(spec, nodes):
+    if len(nodes) > 0:
+        return
+
+    msg = (
+        "* Spec='{}' does not identify any models"
+        .format(spec['raw'])
+    )
+    dbt.exceptions.warn_or_error(msg, log_fmt='{} and was ignored\n')
+
+
+class NodeSelector(object):
+    def __init__(self, linker, manifest):
+        self.linker = linker
+        self.manifest = manifest
+
+    def _node_iterator(self, graph, exclude, include):
+        for node in graph.nodes():
+            real_node = self.manifest.nodes[node]
+            if include is not None and real_node.resource_type not in include:
+                continue
+            if exclude is not None and real_node.resource_type in exclude:
+                continue
+            yield node, real_node
+
+    def parsed_nodes(self, graph):
+        return self._node_iterator(
+            graph,
+            exclude=(NodeType.Source,),
+            include=None)
+
+    def source_nodes(self, graph):
+        return self._node_iterator(
+            graph,
+            exclude=None,
+            include=(NodeType.Source,))
+
+    def get_nodes_by_qualified_name(self, graph, qualified_name_selector):
+        """Yield all nodes in the graph that match the qualified_name_selector.
+
+        :param str qualified_name_selector: The selector or node name
+        """
+        qualified_name = qualified_name_selector.split(".")
+        package_names = get_package_names(graph)
+        for node, real_node in self.parsed_nodes(graph):
+            if _node_is_match(qualified_name, package_names, real_node.fqn):
+                yield node
+
+    def get_nodes_by_tag(self, graph, tag_name):
+        """ yields nodes from graph that have the specified tag """
+        for node, real_node in self.parsed_nodes(graph):
+            if tag_name in real_node.tags:
+                yield node
+
+    def get_nodes_by_source(self, graph, source_full_name):
+        """yields nodes from graph are the specified source."""
+        parts = source_full_name.split('.')
+        target_package = SELECTOR_GLOB
+        if len(parts) == 1:
+            target_source, target_table = parts[0], None
+        elif len(parts) == 2:
+            target_source, target_table = parts
+        elif len(parts) == 3:
+            target_package, target_source, target_table = parts
+        else:  # len(parts) > 3 or len(parts) == 0
+            msg = (
+                'Invalid source selector value "{}". Sources must be of the '
+                'form `${{source_name}}`, '
+                '`${{source_name}}.${{target_name}}`, or '
+                '`${{package_name}}.${{source_name}}.${{target_name}}'
+            ).format(source_full_name)
+            raise dbt.exceptions.RuntimeException(msg)
+
+        for node, real_node in self.source_nodes(graph):
+            if target_package not in (real_node.package_name, SELECTOR_GLOB):
+                continue
+            if target_source not in (real_node.source_name, SELECTOR_GLOB):
+                continue
+            if target_table in (None, real_node.name, SELECTOR_GLOB):
+                yield node
+
+    def select_childrens_parents(self, graph, selected):
+        ancestors_for = self.select_children(graph, selected) | selected
+        return self.select_parents(graph, ancestors_for) | ancestors_for
+
+    def select_children(self, graph, selected):
+        descendants = set()
+        for node in selected:
+            descendants.update(nx.descendants(graph, node))
+        return descendants
+
+    def select_parents(self, graph, selected):
+        ancestors = set()
+        for node in selected:
+            ancestors.update(nx.ancestors(graph, node))
+        return ancestors
+
+    def collect_models(self, graph, selected, spec):
+        additional = set()
+        if spec.select_childrens_parents:
+            additional.update(self.select_childrens_parents(graph, selected))
+        if spec.select_parents:
+            additional.update(self.select_parents(graph, selected))
+        if spec.select_children:
+            additional.update(self.select_children(graph, selected))
+        return additional
+
+    def collect_tests(self, graph, model_nodes):
+        test_nodes = set()
+        for node in model_nodes:
+            # include tests that depend on this node. if we aren't running
+            # tests, they'll be filtered out later.
+            child_tests = [n for n in graph.successors(node)
+                           if self.manifest.nodes[n].resource_type ==
+                           NodeType.Test]
+            test_nodes.update(child_tests)
+        return test_nodes
+
+    def get_nodes_from_spec(self, graph, spec):
+        filter_map = {
+            SELECTOR_FILTERS.FQN: self.get_nodes_by_qualified_name,
+            SELECTOR_FILTERS.TAG: self.get_nodes_by_tag,
+            SELECTOR_FILTERS.SOURCE: self.get_nodes_by_source,
+        }
+
+        filter_method = filter_map.get(spec.selector_type)
+
+        if filter_method is None:
+            valid_selectors = ", ".join(filter_map.keys())
+            logger.info("The '{}' selector specified in {} is invalid. Must "
+                        "be one of [{}]".format(
+                            spec.selector_type,
+                            spec.raw,
+                            valid_selectors))
+
+            return set()
+
+        collected = set(filter_method(graph, spec.selector_value))
+        collected.update(self.collect_models(graph, collected, spec))
+        collected.update(self.collect_tests(graph, collected))
+
+        return collected
+
+    def select_nodes(self, graph, raw_include_specs, raw_exclude_specs):
+        selected_nodes = set()
+
+        for raw_spec in split_specs(raw_include_specs):
+            spec = SelectionCriteria(raw_spec)
+            included_nodes = self.get_nodes_from_spec(graph, spec)
+            selected_nodes.update(included_nodes)
+
+        for raw_spec in split_specs(raw_exclude_specs):
+            spec = SelectionCriteria(raw_spec)
+            excluded_nodes = self.get_nodes_from_spec(graph, spec)
+            selected_nodes.difference_update(excluded_nodes)
+
+        return selected_nodes
+
+    def _is_graph_member(self, node_name):
+        node = self.manifest.nodes[node_name]
+        if node.resource_type == NodeType.Source:
+            return True
+        return not node.get('empty') and is_enabled(node)
+
+    def get_valid_nodes(self, graph):
+        return [
+            node_name for node_name in graph.nodes()
+            if self._is_graph_member(node_name)
+        ]
+
+    def _is_match(self, node_name, resource_types, tags, required):
+        node = self.manifest.nodes[node_name]
+        if node.resource_type not in resource_types:
+            return False
+        tags = set(tags)
+        if tags and not bool(set(node.tags) & tags):
+            # there are tags specified but none match
+            return False
+        for attr in required:
+            if not getattr(node, attr):
+                return False
+        return True
+
+    def get_selected(self, include, exclude, resource_types, tags, required):
+        graph = self.linker.graph
+
+        include = coalesce(include, ['fqn:*', 'source:*'])
+        exclude = coalesce(exclude, [])
+        tags = coalesce(tags, [])
+
+        to_run = self.get_valid_nodes(graph)
+        filtered_graph = graph.subgraph(to_run)
+        selected_nodes = self.select_nodes(filtered_graph, include, exclude)
+
+        filtered_nodes = set()
+        for node_name in selected_nodes:
+            if self._is_match(node_name, resource_types, tags, required):
+                filtered_nodes.add(node_name)
+
+        return filtered_nodes
+
+    def select(self, query):
+        include = query.get('include')
+        exclude = query.get('exclude')
+        resource_types = query.get('resource_types')
+        tags = query.get('tags')
+        required = query.get('required', ())
+        addin_ephemeral_nodes = query.get('addin_ephemeral_nodes', True)
+
+        selected = self.get_selected(include, exclude, resource_types, tags,
+                                     required)
+
+        # if you haven't selected any nodes, return that so we can give the
+        # nice "no models selected" message.
+        if not selected:
+            return selected
+
+        # we used to carefully go through all node ancestors and add those if
+        # they were ephemeral. Sadly, the algorithm we used ended up being
+        # O(n^2). Instead, since ephemeral nodes are almost free, just add all
+        # ephemeral nodes in the graph.
+        # someday at large enough scale we might want to prune it to only be
+        # ancestors of the selected nodes so we can skip the compile.
+        if addin_ephemeral_nodes:
+            addins = {
+                uid for uid, node in self.manifest.nodes.items()
+                if node.is_ephemeral_model
+            }
+        else:
+            addins = set()
+
+        return selected | addins

core/dbt/hooks.py

@@ -0,0 +1,41 @@
+
+import json
+from dbt.compat import to_string
+
+
+class ModelHookType:
+    PreHook = 'pre-hook'
+    PostHook = 'post-hook'
+    Both = [PreHook, PostHook]
+
+
+def _parse_hook_to_dict(hook_string):
+    try:
+        hook_dict = json.loads(hook_string)
+    except ValueError:
+        hook_dict = {"sql": hook_string}
+
+    if 'transaction' not in hook_dict:
+        hook_dict['transaction'] = True
+
+    return hook_dict
+
+
+def get_hook_dict(hook, index):
+    if isinstance(hook, dict):
+        hook_dict = hook
+    else:
+        hook_dict = _parse_hook_to_dict(to_string(hook))
+
+    hook_dict['index'] = index
+    return hook_dict
+
+
+def get_hooks(model, hook_key):
+    hooks = model.config.get(hook_key, [])
+
+    if not isinstance(hooks, (list, tuple)):
+        hooks = [hooks]
+
+    wrapped = [get_hook_dict(hook, i) for i, hook in enumerate(hooks)]
+    return wrapped

core/dbt/include/__init__.py

@@ -0,0 +1 @@
+__path__ = __import__('pkgutil').extend_path(__path__, __name__)

core/dbt/include/global_project/__init__.py

@@ -0,0 +1,11 @@
+import os
+
+PACKAGE_PATH = os.path.dirname(__file__)
+PROJECT_NAME = 'dbt'
+
+DOCS_INDEX_FILE_PATH = os.path.normpath(
+    os.path.join(PACKAGE_PATH, '..', "index.html"))
+
+
+# Adapter registration will add to this
+PACKAGES = {PROJECT_NAME: PACKAGE_PATH}

core/dbt/include/global_project/dbt_project.yml

@@ -0,0 +1,6 @@
+
+name: dbt
+version: 1.0
+
+docs-paths: ['docs']
+macro-paths: ["macros"]

core/dbt/include/global_project/docs/overview.md

@@ -0,0 +1,43 @@
+
+{% docs __overview__ %}
+
+### Welcome!
+
+Welcome to the auto-generated documentation for your dbt project!
+
+### Navigation
+
+You can use the `Project` and `Database` navigation tabs on the left side of the window to explore the models
+in your project.
+
+#### Project Tab
+The `Project` tab mirrors the directory structure of your dbt project. In this tab, you can see all of the
+models defined in your dbt project, as well as models imported from dbt packages.
+
+#### Database Tab
+The `Database` tab also exposes your models, but in a format that looks more like a database explorer. This view
+shows relations (tables and views) grouped into database schemas. Note that ephemeral models are _not_ shown
+in this interface, as they do not exist in the database.
+
+### Graph Exploration
+You can click the blue icon on the bottom-right corner of the page to view the lineage graph of your models.
+
+On model pages, you'll see the immediate parents and children of the model you're exploring. By clicking the `Expand`
+button at the top-right of this lineage pane, you'll be able to see all of the models that are used to build,
+or are built from, the model you're exploring.
+
+Once expanded, you'll be able to use the `--models` and `--exclude` model selection syntax to filter the
+models in the graph. For more information on model selection, check out the [dbt docs](https://docs.getdbt.com/docs/model-selection-syntax).
+
+Note that you can also right-click on models to interactively filter and explore the graph.
+
+---
+
+### More information
+
+- [What is dbt](https://docs.getdbt.com/docs/overview)?
+- Read the [dbt viewpoint](https://docs.getdbt.com/docs/viewpoint)
+- [Installation](https://docs.getdbt.com/docs/installation)
+- Join the [chat](https://slack.getdbt.com/) on Slack for live questions and support.
+
+{% enddocs %}

core/dbt/include/global_project/macros/adapters/common.sql

@@ -0,0 +1,269 @@
+{% macro adapter_macro(name) -%}
+{% set original_name = name %}
+  {% if '.' in name %}
+    {% set package_name, name = name.split(".", 1) %}
+  {% else %}
+    {% set package_name = none %}
+  {% endif %}
+
+  {% if package_name is none %}
+    {% set package_context = context %}
+  {% elif package_name in context %}
+    {% set package_context = context[package_name] %}
+  {% else %}
+    {% set error_msg %}
+        In adapter_macro: could not find package '{{package_name}}', called with '{{original_name}}'
+    {% endset %}
+    {{ exceptions.raise_compiler_error(error_msg | trim) }}
+  {% endif %}
+
+  {%- set separator = '__' -%}
+  {%- set search_name = adapter.type() + separator + name -%}
+  {%- set default_name = 'default' + separator + name -%}
+
+  {%- if package_context.get(search_name) is not none -%}
+    {{ return(package_context[search_name](*varargs, **kwargs)) }}
+  {%- else -%}
+    {{ return(package_context[default_name](*varargs, **kwargs)) }}
+  {%- endif -%}
+{%- endmacro %}
+
+{% macro get_columns_in_query(select_sql) -%}
+  {{ return(adapter_macro('get_columns_in_query', select_sql)) }}
+{% endmacro %}
+
+{% macro default__get_columns_in_query(select_sql) %}
+    {% call statement('get_columns_in_query', fetch_result=True, auto_begin=False) -%}
+        select * from (
+            {{ select_sql }}
+        ) as __dbt_sbq
+        where false
+        limit 0
+    {% endcall %}
+
+    {{ return(load_result('get_columns_in_query').table.columns | map(attribute='name') | list) }}
+{% endmacro %}
+
+{% macro create_schema(database_name, schema_name) -%}
+  {{ adapter_macro('create_schema', database_name, schema_name) }}
+{% endmacro %}
+
+{% macro default__create_schema(database_name, schema_name) -%}
+  {%- call statement('create_schema') -%}
+    create schema if not exists {{database_name}}.{{schema_name}}
+  {% endcall %}
+{% endmacro %}
+
+{% macro drop_schema(database_name, schema_name) -%}
+  {{ adapter_macro('drop_schema', database_name, schema_name) }}
+{% endmacro %}
+
+{% macro default__drop_schema(database_name, schema_name) -%}
+  {%- call statement('drop_schema') -%}
+    drop schema if exists {{database_name}}.{{schema_name}} cascade
+  {% endcall %}
+{% endmacro %}
+
+{% macro create_table_as(temporary, relation, sql) -%}
+  {{ adapter_macro('create_table_as', temporary, relation, sql) }}
+{%- endmacro %}
+
+{% macro default__create_table_as(temporary, relation, sql) -%}
+  create {% if temporary: -%}temporary{%- endif %} table
+    {{ relation.include(database=(not temporary), schema=(not temporary)) }}
+  as (
+    {{ sql }}
+  );
+{% endmacro %}
+
+{% macro create_view_as(relation, sql) -%}
+  {{ adapter_macro('create_view_as', relation, sql) }}
+{%- endmacro %}
+
+{% macro default__create_view_as(relation, sql) -%}
+  create view {{ relation }} as (
+    {{ sql }}
+  );
+{% endmacro %}
+
+
+{% macro get_catalog(information_schemas) -%}
+  {{ return(adapter_macro('get_catalog', information_schemas)) }}
+{%- endmacro %}
+
+{% macro default__get_catalog(information_schemas) -%}
+
+  {% set typename = adapter.type() %}
+  {% set msg -%}
+    get_catalog not implemented for {{ typename }}
+  {%- endset %}
+
+  {{ exceptions.raise_compiler_error(msg) }}
+{% endmacro %}
+
+
+{% macro get_columns_in_relation(relation) -%}
+  {{ return(adapter_macro('get_columns_in_relation', relation)) }}
+{% endmacro %}
+
+{% macro sql_convert_columns_in_relation(table) -%}
+  {% set columns = [] %}
+  {% for row in table %}
+    {% do columns.append(api.Column(*row)) %}
+  {% endfor %}
+  {{ return(columns) }}
+{% endmacro %}
+
+{% macro default__get_columns_in_relation(relation) -%}
+  {{ exceptions.raise_not_implemented(
+    'get_columns_in_relation macro not implemented for adapter '+adapter.type()) }}
+{% endmacro %}
+
+{% macro alter_column_type(relation, column_name, new_column_type) -%}
+  {{ return(adapter_macro('alter_column_type', relation, column_name, new_column_type)) }}
+{% endmacro %}
+
+{% macro default__alter_column_type(relation, column_name, new_column_type) -%}
+  {#
+    1. Create a new column (w/ temp name and correct type)
+    2. Copy data over to it
+    3. Drop the existing column (cascade!)
+    4. Rename the new column to existing column
+  #}
+  {%- set tmp_column = column_name + "__dbt_alter" -%}
+
+  {% call statement('alter_column_type') %}
+    alter table {{ relation }} add column {{ tmp_column }} {{ new_column_type }};
+    update {{ relation }} set {{ tmp_column }} = {{ column_name }};
+    alter table {{ relation }} drop column {{ column_name }} cascade;
+    alter table {{ relation }} rename column {{ tmp_column }} to {{ column_name }}
+  {% endcall %}
+
+{% endmacro %}
+
+
+{% macro drop_relation(relation) -%}
+  {{ return(adapter_macro('drop_relation', relation)) }}
+{% endmacro %}
+
+
+{% macro default__drop_relation(relation) -%}
+  {% call statement('drop_relation', auto_begin=False) -%}
+    drop {{ relation.type }} if exists {{ relation }} cascade
+  {%- endcall %}
+{% endmacro %}
+
+{% macro truncate_relation(relation) -%}
+  {{ return(adapter_macro('truncate_relation', relation)) }}
+{% endmacro %}
+
+
+{% macro default__truncate_relation(relation) -%}
+  {% call statement('truncate_relation') -%}
+    truncate table {{ relation }}
+  {%- endcall %}
+{% endmacro %}
+
+{% macro rename_relation(from_relation, to_relation) -%}
+  {{ return(adapter_macro('rename_relation', from_relation, to_relation)) }}
+{% endmacro %}
+
+{% macro default__rename_relation(from_relation, to_relation) -%}
+  {% set target_name = adapter.quote_as_configured(to_relation.identifier, 'identifier') %}
+  {% call statement('rename_relation') -%}
+    alter table {{ from_relation }} rename to {{ target_name }}
+  {%- endcall %}
+{% endmacro %}
+
+
+{% macro information_schema_name(database) %}
+  {{ return(adapter_macro('information_schema_name', database)) }}
+{% endmacro %}
+
+{% macro default__information_schema_name(database) -%}
+  {%- if database -%}
+    {{ adapter.quote_as_configured(database, 'database') }}.information_schema
+  {%- else -%}
+    information_schema
+  {%- endif -%}
+{%- endmacro %}
+
+
+{% macro list_schemas(database) -%}
+  {{ return(adapter_macro('list_schemas', database)) }}
+{% endmacro %}
+
+{% macro default__list_schemas(database) -%}
+  {% call statement('list_schemas', fetch_result=True, auto_begin=False) %}
+    select distinct schema_name
+    from {{ information_schema_name(database) }}.schemata
+    where catalog_name ilike '{{ database }}'
+  {% endcall %}
+  {{ return(load_result('list_schemas').table) }}
+{% endmacro %}
+
+
+{% macro check_schema_exists(information_schema, schema) -%}
+  {{ return(adapter_macro('check_schema_exists', information_schema, schema)) }}
+{% endmacro %}
+
+{% macro default__check_schema_exists(information_schema, schema) -%}
+  {% call statement('check_schema_exists', fetch_result=True, auto_begin=False) -%}
+        select count(*)
+        from {{ information_schema }}.schemata
+        where catalog_name='{{ information_schema.database }}'
+          and schema_name='{{ schema }}'
+  {%- endcall %}
+  {{ return(load_result('check_schema_exists').table) }}
+{% endmacro %}
+
+
+{% macro list_relations_without_caching(information_schema, schema) %}
+  {{ return(adapter_macro('list_relations_without_caching', information_schema, schema)) }}
+{% endmacro %}
+
+
+{% macro default__list_relations_without_caching(information_schema, schema) %}
+  {{ exceptions.raise_not_implemented(
+    'list_relations_without_caching macro not implemented for adapter '+adapter.type()) }}
+{% endmacro %}
+
+
+{% macro current_timestamp() -%}
+  {{ adapter_macro('current_timestamp') }}
+{%- endmacro %}
+
+
+{% macro default__current_timestamp() -%}
+  {{ exceptions.raise_not_implemented(
+    'current_timestamp macro not implemented for adapter '+adapter.type()) }}
+{%- endmacro %}
+
+
+{% macro collect_freshness(source, loaded_at_field) %}
+  {{ return(adapter_macro('collect_freshness', source, loaded_at_field))}}
+{% endmacro %}
+
+
+{% macro default__collect_freshness(source, loaded_at_field) %}
+  {% call statement('check_schema_exists', fetch_result=True, auto_begin=False) -%}
+    select
+      max({{ loaded_at_field }}) as max_loaded_at,
+      {{ current_timestamp() }} as snapshotted_at
+    from {{ source }}
+  {% endcall %}
+  {{ return(load_result('check_schema_exists').table) }}
+{% endmacro %}
+
+{% macro make_temp_relation(base_relation, suffix='__dbt_tmp') %}
+  {{ return(adapter_macro('make_temp_relation', base_relation, suffix))}}
+{% endmacro %}
+
+{% macro default__make_temp_relation(base_relation, suffix) %}
+    {% set tmp_identifier = base_relation.identifier ~ suffix %}
+    {% set tmp_relation = base_relation.incorporate(
+                                path={"identifier": tmp_identifier},
+                                table_name=tmp_identifier) -%}
+
+    {% do return(tmp_relation) %}
+{% endmacro %}

core/dbt/include/global_project/macros/core.sql

@@ -0,0 +1,30 @@
+{% macro statement(name=None, fetch_result=False, auto_begin=True) -%}
+  {%- if execute: -%}
+    {%- set sql = render(caller()) -%}
+
+    {%- if name == 'main' -%}
+      {{ log('Writing runtime SQL for node "{}"'.format(model['unique_id'])) }}
+      {{ write(sql) }}
+    {%- endif -%}
+
+    {%- set status, res = adapter.execute(sql, auto_begin=auto_begin, fetch=fetch_result) -%}
+    {%- if name is not none -%}
+      {{ store_result(name, status=status, agate_table=res) }}
+    {%- endif -%}
+
+  {%- endif -%}
+{%- endmacro %}
+
+{% macro noop_statement(name=None, status=None, res=None) -%}
+  {%- set sql = render(caller()) -%}
+
+  {%- if name == 'main' -%}
+    {{ log('Writing runtime SQL for node "{}"'.format(model['unique_id'])) }}
+    {{ write(sql) }}
+  {%- endif -%}
+
+  {%- if name is not none -%}
+    {{ store_result(name, status=status, agate_table=res) }}
+  {%- endif -%}
+
+{%- endmacro %}

core/dbt/include/global_project/macros/etc/datetime.sql

@@ -0,0 +1,60 @@
+
+{% macro convert_datetime(date_str, date_fmt) %}
+
+  {% set error_msg -%}
+      The provided partition date '{{ date_str }}' does not match the expected format '{{ date_fmt }}'
+  {%- endset %}
+
+  {% set res = try_or_compiler_error(error_msg, modules.datetime.datetime.strptime, date_str.strip(), date_fmt) %}
+  {{ return(res) }}
+
+{% endmacro %}
+
+{% macro dates_in_range(start_date_str, end_date_str=none, in_fmt="%Y%m%d", out_fmt="%Y%m%d") %}
+    {% set end_date_str = start_date_str if end_date_str is none else end_date_str %}
+
+    {% set start_date = convert_datetime(start_date_str, in_fmt) %}
+    {% set end_date = convert_datetime(end_date_str, in_fmt) %}
+
+    {% set day_count = (end_date - start_date).days %}
+    {% if day_count < 0 %}
+        {% set msg -%}
+            Partiton start date is after the end date ({{ start_date }}, {{ end_date }})
+        {%- endset %}
+
+        {{ exceptions.raise_compiler_error(msg, model) }}
+    {% endif %}
+
+    {% set date_list = [] %}
+    {% for i in range(0, day_count + 1) %}
+        {% set the_date = (modules.datetime.timedelta(days=i) + start_date) %}
+        {% if not out_fmt %}
+            {% set _ = date_list.append(the_date) %}
+        {% else %}
+            {% set _ = date_list.append(the_date.strftime(out_fmt)) %}
+        {% endif %}
+    {% endfor %}
+
+    {{ return(date_list) }}
+{% endmacro %}
+
+{% macro partition_range(raw_partition_date, date_fmt='%Y%m%d') %}
+    {% set partition_range = (raw_partition_date | string).split(",") %}
+
+    {% if (partition_range | length) == 1 %}
+      {% set start_date = partition_range[0] %}
+      {% set end_date = none %}
+    {% elif (partition_range | length) == 2 %}
+      {% set start_date = partition_range[0] %}
+      {% set end_date = partition_range[1] %}
+    {% else %}
+      {{ exceptions.raise_compiler_error("Invalid partition time. Expected format: {Start Date}[,{End Date}]. Got: " ~ raw_partition_date) }}
+    {% endif %}
+
+    {{ return(dates_in_range(start_date, end_date, in_fmt=date_fmt)) }}
+{% endmacro %}
+
+{% macro py_current_timestring() %}
+    {% set dt = modules.datetime.datetime.now() %}
+    {% do return(dt.strftime("%Y%m%d%H%M%S%f")) %}
+{% endmacro %}

core/dbt/include/global_project/macros/etc/get_custom_alias.sql

@@ -0,0 +1,27 @@
+
+{#
+    Renders a alias name given a custom alias name. If the custom
+    alias name is none, then the resulting alias is just the filename of the
+    model. If an alias override is specified, then that is used.
+
+    This macro can be overriden in projects to define different semantics
+    for rendering a alias name.
+
+    Arguments:
+    custom_alias_name: The custom alias name specified for a model, or none
+    node: The available node that an alias is being generated for, or none
+
+#}
+{% macro generate_alias_name(custom_alias_name=none, node=none) -%}
+
+    {%- if custom_alias_name is none -%}
+
+        {{ node.name }}
+
+    {%- else -%}
+
+        {{ custom_alias_name | trim }}
+
+    {%- endif -%}
+
+{%- endmacro %}

core/dbt/include/global_project/macros/etc/get_custom_schema.sql

@@ -0,0 +1,56 @@
+
+{#
+    Renders a schema name given a custom schema name. If the custom
+    schema name is none, then the resulting schema is just the "schema"
+    value in the specified target. If a schema override is specified, then
+    the resulting schema is the default schema concatenated with the
+    custom schema.
+
+    This macro can be overriden in projects to define different semantics
+    for rendering a schema name.
+
+    Arguments:
+    custom_schema_name: The custom schema name specified for a model, or none
+    node: The node the schema is being generated for
+
+#}
+{% macro generate_schema_name(custom_schema_name, node) -%}
+
+    {%- set default_schema = target.schema -%}
+    {%- if custom_schema_name is none -%}
+
+        {{ default_schema }}
+
+    {%- else -%}
+
+        {{ default_schema }}_{{ custom_schema_name | trim }}
+
+    {%- endif -%}
+
+{%- endmacro %}
+
+
+{#
+    Renders a schema name given a custom schema name. In production, this macro
+    will render out the overriden schema name for a model. Otherwise, the default
+    schema specified in the active target is used.
+
+    Arguments:
+    custom_schema_name: The custom schema name specified for a model, or none
+    node: The node the schema is being generated for
+
+#}
+{% macro generate_schema_name_for_env(custom_schema_name, node) -%}
+
+    {%- set default_schema = target.schema -%}
+    {%- if target.name == 'prod' and custom_schema_name is not none -%}
+
+        {{ custom_schema_name | trim }}
+
+    {%- else -%}
+
+        {{ default_schema }}
+
+    {%- endif -%}
+
+{%- endmacro %}

core/dbt/include/global_project/macros/etc/get_relation_comment.sql

@@ -0,0 +1,18 @@
+{% macro table_options() %}
+  {%- set raw_persist_docs = config.get('persist_docs', {}) -%}
+
+{%- endmacro -%}
+
+{% macro get_relation_comment(persist_docs, model) %}
+
+  {%- if persist_docs is not mapping -%}
+    {{ exceptions.raise_compiler_error("Invalid value provided for 'persist_docs'. Expected dict but got value: " ~ raw_persist_docs) }}
+  {% endif %}
+
+  {% if persist_docs.get('relation', false) %}
+    {{ return((model.description | tojson)[1:-1]) }}
+  {%- else -%}
+    {{ return(none) }}
+  {% endif %}
+
+{% endmacro %}

core/dbt/include/global_project/macros/etc/is_incremental.sql

@@ -0,0 +1,13 @@
+
+{% macro is_incremental() %}
+    {#-- do not run introspective queries in parsing #}
+    {% if not execute %}
+        {{ return(False) }}
+    {% else %}
+        {% set relation = adapter.get_relation(this.database, this.schema, this.table) %}
+        {{ return(relation is not none
+                  and relation.type == 'table'
+                  and model.config.materialized == 'incremental'
+                  and not flags.FULL_REFRESH) }}
+    {% endif %}
+{% endmacro %}

core/dbt/include/global_project/macros/etc/query.sql

@@ -0,0 +1,8 @@
+
+{% macro run_query(sql) %}
+  {% call statement("run_query_statement", fetch_result=true, auto_begin=false) %}
+    {{ sql }}
+  {% endcall %}
+
+  {% do return(load_result("run_query_statement").table) %}
+{% endmacro %}

core/dbt/include/global_project/macros/materializations/common/merge.sql

@@ -0,0 +1,71 @@
+
+
+{% macro get_merge_sql(target, source, unique_key, dest_columns) -%}
+  {{ adapter_macro('get_merge_sql', target, source, unique_key, dest_columns) }}
+{%- endmacro %}
+
+{% macro get_delete_insert_merge_sql(target, source, unique_key, dest_columns) -%}
+  {{ adapter_macro('get_delete_insert_merge_sql', target, source, unique_key, dest_columns) }}
+{%- endmacro %}
+
+
+{% macro common_get_merge_sql(target, source, unique_key, dest_columns) -%}
+    {%- set dest_cols_csv = dest_columns | map(attribute="name") | join(', ') -%}
+
+    merge into {{ target }} as DBT_INTERNAL_DEST
+    using {{ source }} as DBT_INTERNAL_SOURCE
+
+    {% if unique_key %}
+        on DBT_INTERNAL_SOURCE.{{ unique_key }} = DBT_INTERNAL_DEST.{{ unique_key }}
+    {% else %}
+        on FALSE
+    {% endif %}
+
+    {% if unique_key %}
+    when matched then update set
+        {% for column in dest_columns -%}
+            {{ column.name }} = DBT_INTERNAL_SOURCE.{{ column.name }}
+            {%- if not loop.last %}, {%- endif %}
+        {%- endfor %}
+    {% endif %}
+
+    when not matched then insert
+        ({{ dest_cols_csv }})
+    values
+        ({{ dest_cols_csv }})
+
+{%- endmacro %}
+
+{% macro default__get_merge_sql(target, source, unique_key, dest_columns) -%}
+    {% set typename = adapter.type() %}
+
+    {{ exceptions.raise_compiler_error(
+        'get_merge_sql is not implemented for {}'.format(typename)
+       )
+    }}
+
+{% endmacro %}
+
+
+{% macro common_get_delete_insert_merge_sql(target, source, unique_key, dest_columns) -%}
+    {%- set dest_cols_csv = dest_columns | map(attribute="name") | join(', ') -%}
+
+    {% if unique_key is not none %}
+    delete from {{ target }}
+    where ({{ unique_key }}) in (
+        select ({{ unique_key }})
+        from {{ source }}
+    );
+    {% endif %}
+
+    insert into {{ target }} ({{ dest_cols_csv }})
+    (
+        select {{ dest_cols_csv }}
+        from {{ source }}
+    );
+
+{%- endmacro %}
+
+{% macro default__get_delete_insert_merge_sql(target, source, unique_key, dest_columns) -%}
+    {{ common_get_delete_insert_merge_sql(target, source, unique_key, dest_columns) }}
+{% endmacro %}

core/dbt/include/global_project/macros/materializations/helpers.sql

@@ -0,0 +1,52 @@
+{% macro run_hooks(hooks, inside_transaction=True) %}
+  {% for hook in hooks | selectattr('transaction', 'equalto', inside_transaction)  %}
+    {% if not inside_transaction and loop.first %}
+      {% call statement(auto_begin=inside_transaction) %}
+        commit;
+      {% endcall %}
+    {% endif %}
+    {% call statement(auto_begin=inside_transaction) %}
+      {{ hook.get('sql') }}
+    {% endcall %}
+  {% endfor %}
+{% endmacro %}
+
+
+{% macro column_list(columns) %}
+  {%- for col in columns %}
+    {{ col.name }} {% if not loop.last %},{% endif %}
+  {% endfor -%}
+{% endmacro %}
+
+
+{% macro column_list_for_create_table(columns) %}
+  {%- for col in columns %}
+    {{ col.name }} {{ col.data_type }} {%- if not loop.last %},{% endif %}
+  {% endfor -%}
+{% endmacro %}
+
+
+{% macro make_hook_config(sql, inside_transaction) %}
+    {{ tojson({"sql": sql, "transaction": inside_transaction}) }}
+{% endmacro %}
+
+
+{% macro before_begin(sql) %}
+    {{ make_hook_config(sql, inside_transaction=False) }}
+{% endmacro %}
+
+
+{% macro in_transaction(sql) %}
+    {{ make_hook_config(sql, inside_transaction=True) }}
+{% endmacro %}
+
+
+{% macro after_commit(sql) %}
+    {{ make_hook_config(sql, inside_transaction=False) }}
+{% endmacro %}
+
+{% macro drop_relation_if_exists(relation) %}
+  {% if relation is not none %}
+    {{ adapter.drop_relation(relation) }}
+  {% endif %}
+{% endmacro %}

core/dbt/include/global_project/macros/materializations/incremental/incremental.sql

@@ -0,0 +1,82 @@
+{% macro dbt__incremental_delete(target_relation, tmp_relation) -%}
+
+  {%- set unique_key = config.require('unique_key') -%}
+
+  delete
+  from {{ target_relation }}
+  where ({{ unique_key }}) in (
+    select ({{ unique_key }})
+    from {{ tmp_relation.include(schema=False, database=False) }}
+  );
+
+{%- endmacro %}
+
+{% materialization incremental, default -%}
+  {%- set unique_key = config.get('unique_key') -%}
+
+  {%- set identifier = model['alias'] -%}
+  {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}
+  {%- set target_relation = api.Relation.create(identifier=identifier, schema=schema, database=database,  type='table') -%}
+  {%- set tmp_relation = make_temp_relation(target_relation) %}
+
+  {%- set full_refresh_mode = (flags.FULL_REFRESH == True) -%}
+
+  {%- set exists_as_table = (old_relation is not none and old_relation.is_table) -%}
+  {%- set exists_not_as_table = (old_relation is not none and not old_relation.is_table) -%}
+
+  {%- set should_drop = (full_refresh_mode or exists_not_as_table) -%}
+
+  -- setup
+  {% if old_relation is none -%}
+    -- noop
+  {%- elif should_drop -%}
+    {{ adapter.drop_relation(old_relation) }}
+    {%- set old_relation = none -%}
+  {%- endif %}
+
+  {{ run_hooks(pre_hooks, inside_transaction=False) }}
+
+  -- `BEGIN` happens here:
+  {{ run_hooks(pre_hooks, inside_transaction=True) }}
+
+  -- build model
+  {% if full_refresh_mode or old_relation is none -%}
+    {%- call statement('main') -%}
+      {{ create_table_as(False, target_relation, sql) }}
+    {%- endcall -%}
+  {%- else -%}
+     {%- call statement() -%}
+
+       {{ dbt.create_table_as(True, tmp_relation, sql) }}
+
+     {%- endcall -%}
+
+     {{ adapter.expand_target_column_types(from_relation=tmp_relation,
+                                           to_relation=target_relation) }}
+
+     {%- call statement('main') -%}
+       {% set dest_columns = adapter.get_columns_in_relation(target_relation) %}
+       {% set dest_cols_csv = dest_columns | map(attribute='quoted') | join(', ') %}
+
+       {% if unique_key is not none -%}
+
+         {{ dbt__incremental_delete(target_relation, tmp_relation) }}
+
+       {%- endif %}
+
+       insert into {{ target_relation }} ({{ dest_cols_csv }})
+       (
+         select {{ dest_cols_csv }}
+         from {{ tmp_relation }}
+       );
+     {% endcall %}
+  {%- endif %}
+
+  {{ run_hooks(post_hooks, inside_transaction=True) }}
+
+  -- `COMMIT` happens here
+  {{ adapter.commit() }}
+
+  {{ run_hooks(post_hooks, inside_transaction=False) }}
+
+{%- endmaterialization %}

core/dbt/include/global_project/macros/materializations/seed/seed.sql

@@ -0,0 +1,135 @@
+
+{% macro create_csv_table(model) -%}
+  {{ adapter_macro('create_csv_table', model) }}
+{%- endmacro %}
+
+{% macro reset_csv_table(model, full_refresh, old_relation) -%}
+  {{ adapter_macro('reset_csv_table', model, full_refresh, old_relation) }}
+{%- endmacro %}
+
+{% macro load_csv_rows(model) -%}
+  {{ adapter_macro('load_csv_rows', model) }}
+{%- endmacro %}
+
+{% macro default__create_csv_table(model) %}
+  {%- set agate_table = model['agate_table'] -%}
+  {%- set column_override = model['config'].get('column_types', {}) -%}
+
+  {% set sql %}
+    create table {{ this.render(False) }} (
+        {%- for col_name in agate_table.column_names -%}
+            {%- set inferred_type = adapter.convert_type(agate_table, loop.index0) -%}
+            {%- set type = column_override.get(col_name, inferred_type) -%}
+            {{ col_name | string }} {{ type }} {%- if not loop.last -%}, {%- endif -%}
+        {%- endfor -%}
+    )
+  {% endset %}
+
+  {% call statement('_') -%}
+    {{ sql }}
+  {%- endcall %}
+
+  {{ return(sql) }}
+{% endmacro %}
+
+
+{% macro default__reset_csv_table(model, full_refresh, old_relation) %}
+    {% set sql = "" %}
+    {% if full_refresh %}
+        {{ adapter.drop_relation(old_relation) }}
+        {% set sql = create_csv_table(model) %}
+    {% else %}
+        {{ adapter.truncate_relation(old_relation) }}
+        {% set sql = "truncate table " ~ old_relation %}
+    {% endif %}
+
+    {{ return(sql) }}
+{% endmacro %}
+
+
+{% macro basic_load_csv_rows(model, batch_size) %}
+    {% set agate_table = model['agate_table'] %}
+    {% set cols_sql = ", ".join(agate_table.column_names) %}
+    {% set bindings = [] %}
+
+    {% set statements = [] %}
+
+    {% for chunk in agate_table.rows | batch(batch_size) %}
+        {% set bindings = [] %}
+
+        {% for row in chunk %}
+            {% set _ = bindings.extend(row) %}
+        {% endfor %}
+
+        {% set sql %}
+            insert into {{ this.render(False) }} ({{ cols_sql }}) values
+            {% for row in chunk -%}
+                ({%- for column in agate_table.column_names -%}
+                    %s
+                    {%- if not loop.last%},{%- endif %}
+                {%- endfor -%})
+                {%- if not loop.last%},{%- endif %}
+            {%- endfor %}
+        {% endset %}
+
+        {% set _ = adapter.add_query(sql, bindings=bindings, abridge_sql_log=True) %}
+
+        {% if loop.index0 == 0 %}
+            {% set _ = statements.append(sql) %}
+        {% endif %}
+    {% endfor %}
+
+    {# Return SQL so we can render it out into the compiled files #}
+    {{ return(statements[0]) }}
+{% endmacro %}
+
+
+{% macro default__load_csv_rows(model) %}
+  {{ return(basic_load_csv_rows(model, 10000) )}}
+{% endmacro %}
+
+
+{% materialization seed, default %}
+
+  {%- set identifier = model['alias'] -%}
+  {%- set full_refresh_mode = (flags.FULL_REFRESH == True) -%}
+
+  {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}
+
+  {%- set exists_as_table = (old_relation is not none and old_relation.is_table) -%}
+  {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%}
+
+  {%- set csv_table = model["agate_table"] -%}
+
+  {{ run_hooks(pre_hooks, inside_transaction=False) }}
+
+  -- `BEGIN` happens here:
+  {{ run_hooks(pre_hooks, inside_transaction=True) }}
+
+  -- build model
+  {% set create_table_sql = "" %}
+  {% if exists_as_view %}
+    {{ exceptions.raise_compiler_error("Cannot seed to '{}', it is a view".format(old_relation)) }}
+  {% elif exists_as_table %}
+    {% set create_table_sql = reset_csv_table(model, full_refresh_mode, old_relation) %}
+  {% else %}
+    {% set create_table_sql = create_csv_table(model) %}
+  {% endif %}
+
+  {% set status = 'CREATE' if full_refresh_mode else 'INSERT' %}
+  {% set num_rows = (csv_table.rows | length) %}
+  {% set sql = load_csv_rows(model) %}
+
+  {% call noop_statement('main', status ~ ' ' ~ num_rows) %}
+    {{ create_table_sql }};
+    -- dbt seed --
+    {{ sql }}
+  {% endcall %}
+
+  {{ run_hooks(post_hooks, inside_transaction=True) }}
+
+  -- `COMMIT` happens here
+  {{ adapter.commit() }}
+
+  {{ run_hooks(post_hooks, inside_transaction=False) }}
+{% endmaterialization %}

core/dbt/include/global_project/macros/materializations/snapshot/snapshot.sql

@@ -0,0 +1,262 @@
+{#
+    Add new columns to the table if applicable
+#}
+{% macro create_columns(relation, columns) %}
+  {{ adapter_macro('create_columns', relation, columns) }}
+{% endmacro %}
+
+{% macro default__create_columns(relation, columns) %}
+  {% for column in columns %}
+    {% call statement() %}
+      alter table {{ relation }} add column "{{ column.name }}" {{ column.data_type }};
+    {% endcall %}
+  {% endfor %}
+{% endmacro %}
+
+
+{% macro post_snapshot(staging_relation) %}
+  {{ adapter_macro('post_snapshot', staging_relation) }}
+{% endmacro %}
+
+{% macro default__post_snapshot(staging_relation) %}
+    {# no-op #}
+{% endmacro %}
+
+
+{% macro snapshot_staging_table_inserts(strategy, source_sql, target_relation) -%}
+
+    with snapshot_query as (
+
+        {{ source_sql }}
+
+    ),
+
+    snapshotted_data as (
+
+        select *,
+            {{ strategy.unique_key }} as dbt_unique_key
+
+        from {{ target_relation }}
+
+    ),
+
+    source_data as (
+
+        select *,
+            {{ strategy.scd_id }} as dbt_scd_id,
+            {{ strategy.unique_key }} as dbt_unique_key,
+            {{ strategy.updated_at }} as dbt_updated_at,
+            {{ strategy.updated_at }} as dbt_valid_from,
+            nullif({{ strategy.updated_at }}, {{ strategy.updated_at }}) as dbt_valid_to
+
+        from snapshot_query
+    ),
+
+    insertions as (
+
+        select
+            'insert' as dbt_change_type,
+            source_data.*
+
+        from source_data
+        left outer join snapshotted_data on snapshotted_data.dbt_unique_key = source_data.dbt_unique_key
+        where snapshotted_data.dbt_unique_key is null
+           or (
+                snapshotted_data.dbt_unique_key is not null
+            and snapshotted_data.dbt_valid_to is null
+            and (
+                {{ strategy.row_changed }}
+            )
+        )
+
+    )
+
+    select * from insertions
+
+{%- endmacro %}
+
+
+{% macro snapshot_staging_table_updates(strategy, source_sql, target_relation) -%}
+
+    with snapshot_query as (
+
+        {{ source_sql }}
+
+    ),
+
+    snapshotted_data as (
+
+        select *,
+            {{ strategy.unique_key }} as dbt_unique_key
+
+        from {{ target_relation }}
+
+    ),
+
+    source_data as (
+
+        select
+            *,
+            {{ strategy.scd_id }} as dbt_scd_id,
+            {{ strategy.unique_key }} as dbt_unique_key,
+            {{ strategy.updated_at }} as dbt_updated_at,
+            {{ strategy.updated_at }} as dbt_valid_from
+
+        from snapshot_query
+    ),
+
+    updates as (
+
+        select
+            'update' as dbt_change_type,
+            snapshotted_data.dbt_scd_id,
+            source_data.dbt_valid_from as dbt_valid_to
+
+        from source_data
+        join snapshotted_data on snapshotted_data.dbt_unique_key = source_data.dbt_unique_key
+        where snapshotted_data.dbt_valid_to is null
+        and (
+            {{ strategy.row_changed }}
+        )
+
+    )
+
+    select * from updates
+
+{%- endmacro %}
+
+
+{% macro build_snapshot_table(strategy, sql) %}
+
+    select *,
+        {{ strategy.scd_id }} as dbt_scd_id,
+        {{ strategy.updated_at }} as dbt_updated_at,
+        {{ strategy.updated_at }} as dbt_valid_from,
+        nullif({{ strategy.updated_at }}, {{ strategy.updated_at }}) as dbt_valid_to
+    from (
+        {{ sql }}
+    ) sbq
+
+{% endmacro %}
+
+
+{% macro get_or_create_relation(database, schema, identifier, type) %}
+  {%- set target_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) %}
+
+  {% if target_relation %}
+    {% do return([true, target_relation]) %}
+  {% endif %}
+
+  {%- set new_relation = api.Relation.create(
+      database=database,
+      schema=schema,
+      identifier=identifier,
+      type=type
+  ) -%}
+  {% do return([false, new_relation]) %}
+{% endmacro %}
+
+{% macro build_snapshot_staging_table(strategy, sql, target_relation) %}
+    {% set tmp_relation = make_temp_relation(target_relation) %}
+
+    {% set inserts_select = snapshot_staging_table_inserts(strategy, sql, target_relation) %}
+    {% set updates_select = snapshot_staging_table_updates(strategy, sql, target_relation) %}
+
+    {% call statement('build_snapshot_staging_relation_inserts') %}
+        {{ create_table_as(True, tmp_relation, inserts_select) }}
+    {% endcall %}
+
+    {% call statement('build_snapshot_staging_relation_updates') %}
+        insert into {{ tmp_relation }} (dbt_change_type, dbt_scd_id, dbt_valid_to)
+        select dbt_change_type, dbt_scd_id, dbt_valid_to from (
+            {{ updates_select }}
+        ) dbt_sbq;
+    {% endcall %}
+
+    {% do return(tmp_relation) %}
+{% endmacro %}
+
+
+{% materialization snapshot, default %}
+  {%- set config = model['config'] -%}
+
+  {%- set target_database = config.get('target_database') -%}
+  {%- set target_schema = config.get('target_schema') -%}
+  {%- set target_table = model.get('alias', model.get('name')) -%}
+
+  {%- set strategy_name = config.get('strategy') -%}
+  {%- set unique_key = config.get('unique_key') %}
+
+  {% if not adapter.check_schema_exists(target_database, target_schema) %}
+    {% do create_schema(target_database, target_schema) %}
+  {% endif %}
+
+  {% set target_relation_exists, target_relation = get_or_create_relation(
+          database=target_database,
+          schema=target_schema,
+          identifier=target_table,
+          type='table') -%}
+
+  {%- if not target_relation.is_table -%}
+    {% do exceptions.relation_wrong_type(target_relation, 'table') %}
+  {%- endif -%}
+
+  {% set strategy_macro = strategy_dispatch(strategy_name) %}
+  {% set strategy = strategy_macro(model, "snapshotted_data", "source_data", config, target_relation_exists) %}
+
+  {% if not target_relation_exists %}
+
+      {% set build_sql = build_snapshot_table(strategy, model['injected_sql']) %}
+      {% call statement('main') -%}
+          {{ create_table_as(False, target_relation, build_sql) }}
+      {% endcall %}
+
+  {% else %}
+
+      {{ adapter.valid_snapshot_target(target_relation) }}
+
+      {% set staging_table = build_snapshot_staging_table(strategy, sql, target_relation) %}
+
+      -- this may no-op if the database does not require column expansion
+      {% do adapter.expand_target_column_types(from_relation=staging_table,
+                                               to_relation=target_relation) %}
+
+      {% set missing_columns = adapter.get_missing_columns(staging_table, target_relation)
+                                   | rejectattr('name', 'equalto', 'dbt_change_type')
+                                   | rejectattr('name', 'equalto', 'DBT_CHANGE_TYPE')
+                                   | rejectattr('name', 'equalto', 'dbt_unique_key')
+                                   | rejectattr('name', 'equalto', 'DBT_UNIQUE_KEY')
+                                   | list %}
+
+      {% do create_columns(target_relation, missing_columns) %}
+
+      {% set source_columns = adapter.get_columns_in_relation(staging_table)
+                                   | rejectattr('name', 'equalto', 'dbt_change_type')
+                                   | rejectattr('name', 'equalto', 'DBT_CHANGE_TYPE')
+                                   | rejectattr('name', 'equalto', 'dbt_unique_key')
+                                   | rejectattr('name', 'equalto', 'DBT_UNIQUE_KEY')
+                                   | list %}
+
+      {% set quoted_source_columns = [] %}
+      {% for column in source_columns %}
+        {% do quoted_source_columns.append(adapter.quote(column.name)) %}
+      {% endfor %}
+
+      {% call statement('main') %}
+          {{ snapshot_merge_sql(
+                target = target_relation,
+                source = staging_table,
+                insert_cols = quoted_source_columns
+             )
+          }}
+      {% endcall %}
+
+  {% endif %}
+
+  {{ adapter.commit() }}
+
+  {% if staging_table is defined %}
+      {% do post_snapshot(staging_table) %}
+  {% endif %}
+
+{% endmaterialization %}

core/dbt/include/global_project/macros/materializations/snapshot/snapshot_merge.sql

@@ -0,0 +1,27 @@
+
+{% macro snapshot_merge_sql(target, source, insert_cols) -%}
+  {{ adapter_macro('snapshot_merge_sql', target, source, insert_cols) }}
+{%- endmacro %}
+
+
+{% macro default__snapshot_merge_sql(target, source, insert_cols) -%}
+    {%- set insert_cols_csv = insert_cols | join(', ') -%}
+
+    merge into {{ target }} as DBT_INTERNAL_DEST
+    using {{ source }} as DBT_INTERNAL_SOURCE
+    on DBT_INTERNAL_SOURCE.dbt_scd_id = DBT_INTERNAL_DEST.dbt_scd_id
+
+    when matched
+     and DBT_INTERNAL_DEST.dbt_valid_to is null
+     and DBT_INTERNAL_SOURCE.dbt_change_type = 'update'
+        then update
+        set dbt_valid_to = DBT_INTERNAL_SOURCE.dbt_valid_to
+
+    when not matched
+     and DBT_INTERNAL_SOURCE.dbt_change_type = 'insert'
+        then insert ({{ insert_cols_csv }})
+        values ({{ insert_cols_csv }})
+    ;
+{% endmacro %}
+
+

core/dbt/include/global_project/macros/materializations/snapshot/strategies.sql

@@ -0,0 +1,118 @@
+{#
+    Dispatch strategies by name, optionally qualified to a package
+#}
+{% macro strategy_dispatch(name) -%}
+{% set original_name = name %}
+  {% if '.' in name %}
+    {% set package_name, name = name.split(".", 1) %}
+  {% else %}
+    {% set package_name = none %}
+  {% endif %}
+
+  {% if package_name is none %}
+    {% set package_context = context %}
+  {% elif package_name in context %}
+    {% set package_context = context[package_name] %}
+  {% else %}
+    {% set error_msg %}
+        Could not find package '{{package_name}}', called with '{{original_name}}'
+    {% endset %}
+    {{ exceptions.raise_compiler_error(error_msg | trim) }}
+  {% endif %}
+
+  {%- set search_name = 'snapshot_' ~ name ~ '_strategy' -%}
+
+  {% if search_name not in package_context %}
+    {% set error_msg %}
+        The specified strategy macro '{{name}}' was not found in package '{{ package_name }}'
+    {% endset %}
+    {{ exceptions.raise_compiler_error(error_msg | trim) }}
+  {% endif %}
+  {{ return(package_context[search_name]) }}
+{%- endmacro %}
+
+
+{#
+    Create SCD Hash SQL fields cross-db
+#}
+{% macro snapshot_hash_arguments(args) -%}
+  {{ adapter_macro('snapshot_hash_arguments', args) }}
+{%- endmacro %}
+
+
+{% macro default__snapshot_hash_arguments(args) -%}
+    md5({%- for arg in args -%}
+        coalesce(cast({{ arg }} as varchar ), '')
+        {% if not loop.last %} || '|' || {% endif %}
+    {%- endfor -%})
+{%- endmacro %}
+
+
+{#
+    Get the current time cross-db
+#}
+{% macro snapshot_get_time() -%}
+  {{ adapter_macro('snapshot_get_time') }}
+{%- endmacro %}
+
+{% macro default__snapshot_get_time() -%}
+  {{ current_timestamp() }}
+{%- endmacro %}
+
+
+{#
+    Core strategy definitions
+#}
+{% macro snapshot_timestamp_strategy(node, snapshotted_rel, current_rel, config, target_exists) %}
+    {% set primary_key = config['unique_key'] %}
+    {% set updated_at = config['updated_at'] %}
+
+    {% set row_changed_expr -%}
+        ({{ snapshotted_rel }}.{{ updated_at }} < {{ current_rel }}.{{ updated_at }})
+    {%- endset %}
+
+    {% set scd_id_expr = snapshot_hash_arguments([primary_key, updated_at]) %}
+
+    {% do return({
+        "unique_key": primary_key,
+        "updated_at": updated_at,
+        "row_changed": row_changed_expr,
+        "scd_id": scd_id_expr
+    }) %}
+{% endmacro %}
+
+
+{% macro snapshot_check_strategy(node, snapshotted_rel, current_rel, config, target_exists) %}
+    {% set check_cols_config = config['check_cols'] %}
+    {% set primary_key = config['unique_key'] %}
+    {% set updated_at = snapshot_get_time() %}
+
+    {% if check_cols_config == 'all' %}
+        {% set check_cols = get_columns_in_query(node['injected_sql']) %}
+    {% elif check_cols_config is iterable and (check_cols_config | length) > 0 %}
+        {% set check_cols = check_cols_config %}
+    {% else %}
+        {% do exceptions.raise_compiler_error("Invalid value for 'check_cols': " ~ check_cols_config) %}
+    {% endif %}
+
+    {% set row_changed_expr -%}
+        (
+        {% for col in check_cols %}
+            {{ snapshotted_rel }}.{{ col }} != {{ current_rel }}.{{ col }}
+            or
+            ({{ snapshotted_rel }}.{{ col }} is null) != ({{ current_rel }}.{{ col }} is null)
+            {%- if not loop.last %} or {% endif %}
+
+        {% endfor %}
+        )
+    {%- endset %}
+
+    {% set scd_id_expr = snapshot_hash_arguments([primary_key, updated_at]) %}
+
+    {% do return({
+        "unique_key": primary_key,
+        "updated_at": updated_at,
+        "row_changed": row_changed_expr,
+        "scd_id": scd_id_expr
+    }) %}
+{% endmacro %}

core/dbt/include/global_project/macros/materializations/table/table.sql

@@ -0,0 +1,59 @@
+{% materialization table, default %}
+  {%- set identifier = model['alias'] -%}
+  {%- set tmp_identifier = model['name'] + '__dbt_tmp' -%}
+  {%- set backup_identifier = model['name'] + '__dbt_backup' -%}
+
+  {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}
+  {%- set target_relation = api.Relation.create(identifier=identifier,
+                                                schema=schema,
+                                                database=database,
+                                                type='table') -%}
+  {%- set intermediate_relation = api.Relation.create(identifier=tmp_identifier,
+                                                      schema=schema,
+                                                      database=database,
+                                                      type='table') -%}
+
+  /*
+      See ../view/view.sql for more information about this relation.
+  */
+  {%- set backup_relation_type = 'table' if old_relation is none else old_relation.type -%}
+  {%- set backup_relation = api.Relation.create(identifier=backup_identifier,
+                                                schema=schema,
+                                                database=database,
+                                                type=backup_relation_type) -%}
+
+  {%- set exists_as_table = (old_relation is not none and old_relation.is_table) -%}
+  {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%}
+
+
+  -- drop the temp relations if they exists for some reason
+  {{ adapter.drop_relation(intermediate_relation) }}
+  {{ adapter.drop_relation(backup_relation) }}
+
+  {{ run_hooks(pre_hooks, inside_transaction=False) }}
+
+  -- `BEGIN` happens here:
+  {{ run_hooks(pre_hooks, inside_transaction=True) }}
+
+  -- build model
+  {% call statement('main') -%}
+    {{ create_table_as(False, intermediate_relation, sql) }}
+  {%- endcall %}
+
+  -- cleanup
+  {% if old_relation is not none %}
+      {{ adapter.rename_relation(target_relation, backup_relation) }}
+  {% endif %}
+
+  {{ adapter.rename_relation(intermediate_relation, target_relation) }}
+
+  {{ run_hooks(post_hooks, inside_transaction=True) }}
+
+  -- `COMMIT` happens here
+  {{ adapter.commit() }}
+
+  -- finally, drop the existing/backup relation after the commit
+  {{ drop_relation_if_exists(backup_relation) }}
+
+  {{ run_hooks(post_hooks, inside_transaction=False) }}
+{% endmaterialization %}

core/dbt/include/global_project/macros/materializations/view/create_or_replace_view.sql

@@ -0,0 +1,61 @@
+
+{% macro handle_existing_table(full_refresh, old_relation) %}
+    {{ adapter_macro("dbt.handle_existing_table", full_refresh, old_relation) }}
+{% endmacro %}
+
+{% macro default__handle_existing_table(full_refresh, old_relation) %}
+    {{ adapter.drop_relation(old_relation) }}
+{% endmacro %}
+
+{# /*
+       Core materialization implementation. BigQuery and Snowflake are similar
+       because both can use `create or replace view` where the resulting view schema
+       is not necessarily the same as the existing view. On Redshift, this would
+       result in: ERROR:  cannot change number of columns in view
+
+       This implementation is superior to the create_temp, swap_with_existing, drop_old
+       paradigm because transactions don't run DDL queries atomically on Snowflake. By using
+       `create or replace view`, the materialization becomes atomic in nature.
+    */
+#}
+
+{% macro create_or_replace_view(run_outside_transaction_hooks=True) %}
+  {%- set identifier = model['alias'] -%}
+
+  {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}
+
+  {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%}
+
+  {%- set target_relation = api.Relation.create(
+      identifier=identifier, schema=schema, database=database,
+      type='view') -%}
+
+  {% if run_outside_transaction_hooks %}
+      -- no transactions on BigQuery
+      {{ run_hooks(pre_hooks, inside_transaction=False) }}
+  {% endif %}
+
+  -- `BEGIN` happens here on Snowflake
+  {{ run_hooks(pre_hooks, inside_transaction=True) }}
+
+  -- If there's a table with the same name and we weren't told to full refresh,
+  -- that's an error. If we were told to full refresh, drop it. This behavior differs
+  -- for Snowflake and BigQuery, so multiple dispatch is used.
+  {%- if old_relation is not none and old_relation.is_table -%}
+    {{ handle_existing_table(flags.FULL_REFRESH, old_relation) }}
+  {%- endif -%}
+
+  -- build model
+  {% call statement('main') -%}
+    {{ create_view_as(target_relation, sql) }}
+  {%- endcall %}
+
+  {{ run_hooks(post_hooks, inside_transaction=True) }}
+
+  {{ adapter.commit() }}
+
+  {% if run_outside_transaction_hooks %}
+      -- No transactions on BigQuery
+      {{ run_hooks(post_hooks, inside_transaction=False) }}
+  {% endif %}
+{% endmacro %}

core/dbt/include/global_project/macros/materializations/view/view.sql

@@ -0,0 +1,62 @@
+{%- materialization view, default -%}
+
+  {%- set identifier = model['alias'] -%}
+  {%- set tmp_identifier = model['name'] + '__dbt_tmp' -%}
+  {%- set backup_identifier = model['name'] + '__dbt_backup' -%}
+
+  {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}
+  {%- set target_relation = api.Relation.create(identifier=identifier, schema=schema, database=database,
+                                                type='view') -%}
+  {%- set intermediate_relation = api.Relation.create(identifier=tmp_identifier,
+                                                      schema=schema, database=database, type='view') -%}
+
+  /*
+     This relation (probably) doesn't exist yet. If it does exist, it's a leftover from
+     a previous run, and we're going to try to drop it immediately. At the end of this
+     materialization, we're going to rename the "old_relation" to this identifier,
+     and then we're going to drop it. In order to make sure we run the correct one of:
+       - drop view ...
+       - drop table ...
+
+     We need to set the type of this relation to be the type of the old_relation, if it exists,
+     or else "view" as a sane default if it does not. Note that if the old_relation does not
+     exist, then there is nothing to move out of the way and subsequentally drop. In that case,
+     this relation will be effectively unused.
+  */
+  {%- set backup_relation_type = 'view' if old_relation is none else old_relation.type -%}
+  {%- set backup_relation = api.Relation.create(identifier=backup_identifier,
+                                                schema=schema, database=database,
+                                                type=backup_relation_type) -%}
+
+  {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%}
+
+  {{ run_hooks(pre_hooks, inside_transaction=False) }}
+
+  -- drop the temp relations if they exists for some reason
+  {{ adapter.drop_relation(intermediate_relation) }}
+  {{ adapter.drop_relation(backup_relation) }}
+
+  -- `BEGIN` happens here:
+  {{ run_hooks(pre_hooks, inside_transaction=True) }}
+
+  -- build model
+  {% call statement('main') -%}
+    {{ create_view_as(intermediate_relation, sql) }}
+  {%- endcall %}
+
+  -- cleanup
+  -- move the existing view out of the way
+  {% if old_relation is not none %}
+    {{ adapter.rename_relation(target_relation, backup_relation) }}
+  {% endif %}
+  {{ adapter.rename_relation(intermediate_relation, target_relation) }}
+
+  {{ run_hooks(post_hooks, inside_transaction=True) }}
+
+  {{ adapter.commit() }}
+
+  {{ drop_relation_if_exists(backup_relation) }}
+
+  {{ run_hooks(post_hooks, inside_transaction=False) }}
+
+{%- endmaterialization -%}

core/dbt/include/global_project/macros/schema_tests/accepted_values.sql

@@ -0,0 +1,33 @@
+
+{% macro test_accepted_values(model, values) %}
+
+{% set column_name = kwargs.get('column_name', kwargs.get('field')) %}
+
+with all_values as (
+
+    select distinct
+        {{ column_name }} as value_field
+
+    from {{ model }}
+
+),
+
+validation_errors as (
+
+    select
+        value_field
+
+    from all_values
+    where value_field not in (
+        {% for value in values -%}
+
+            '{{ value }}' {% if not loop.last -%} , {%- endif %}
+
+        {%- endfor %}
+    )
+)
+
+select count(*)
+from validation_errors
+
+{% endmacro %}

core/dbt/include/global_project/macros/schema_tests/not_null.sql

@@ -0,0 +1,11 @@
+
+{% macro test_not_null(model) %}
+
+{% set column_name = kwargs.get('column_name', kwargs.get('arg')) %}
+
+select count(*)
+from {{ model }}
+where {{ column_name }} is null
+
+{% endmacro %}
+

core/dbt/include/global_project/macros/schema_tests/relationships.sql

@@ -0,0 +1,18 @@
+
+{% macro test_relationships(model, to, field) %}
+
+{% set column_name = kwargs.get('column_name', kwargs.get('from')) %}
+
+
+select count(*)
+from (
+    select {{ column_name }} as id from {{ model }}
+) as child
+left join (
+    select {{ field }} as id from {{ to }}
+) as parent on parent.id = child.id
+where child.id is not null
+  and parent.id is null
+
+{% endmacro %}
+

core/dbt/include/global_project/macros/schema_tests/unique.sql

@@ -0,0 +1,19 @@
+
+{% macro test_unique(model) %}
+
+{% set column_name = kwargs.get('column_name', kwargs.get('arg')) %}
+
+select count(*)
+from (
+
+    select
+        {{ column_name }}
+
+    from {{ model }}
+    where {{ column_name }} is not null
+    group by {{ column_name }}
+    having count(*) > 1
+
+) validation_errors
+
+{% endmacro %}

core/dbt/linker.py

@@ -0,0 +1,272 @@
+import networkx as nx
+import threading
+
+from dbt.compat import PriorityQueue
+from dbt.node_types import NodeType
+
+
+GRAPH_SERIALIZE_BLACKLIST = [
+    'agate_table'
+]
+
+
+def from_file(graph_file):
+    linker = Linker()
+    linker.read_graph(graph_file)
+
+    return linker
+
+
+def is_blocking_dependency(node):
+    return node.resource_type == NodeType.Model
+
+
+class GraphQueue(object):
+    """A fancy queue that is backed by the dependency graph.
+    Note: this will mutate input!
+
+    This queue is thread-safe for `mark_done` calls, though you must ensure
+    that separate threads do not call `.empty()` or `__len__()` and `.get()` at
+    the same time, as there is an unlocked race!
+    """
+    def __init__(self, graph, manifest):
+        self.graph = graph
+        self.manifest = manifest
+        # store the queue as a priority queue.
+        self.inner = PriorityQueue()
+        # things that have been popped off the queue but not finished
+        # and worker thread reservations
+        self.in_progress = set()
+        # things that are in the queue
+        self.queued = set()
+        # this lock controls most things
+        self.lock = threading.Lock()
+        # store the 'score' of each node as a number. Lower is higher priority.
+        self._scores = self._calculate_scores()
+        # populate the initial queue
+        self._find_new_additions()
+
+    def get_node(self, node_id):
+        return self.manifest.nodes[node_id]
+
+    def _include_in_cost(self, node_id):
+        node = self.get_node(node_id)
+        if not is_blocking_dependency(node):
+            return False
+        if node.get_materialization() == 'ephemeral':
+            return False
+        return True
+
+    def _calculate_scores(self):
+        """Calculate the 'value' of each node in the graph based on how many
+        blocking descendants it has. We use this score for the internal
+        priority queue's ordering, so the quality of this metric is important.
+
+        The score is stored as a negative number because the internal
+        PriorityQueue picks lowest values first.
+
+        We could do this in one pass over the graph instead of len(self.graph)
+        passes but this is easy. For large graphs this may hurt performance.
+
+        This operates on the graph, so it would require a lock if called from
+        outside __init__.
+
+        :return Dict[str, int]: The score dict, mapping unique IDs to integer
+            scores. Lower scores are higher priority.
+        """
+        scores = {}
+        for node in self.graph.nodes():
+            score = -1 * len([
+                d for d in nx.descendants(self.graph, node)
+                if self._include_in_cost(d)
+            ])
+            scores[node] = score
+        return scores
+
+    def get(self, block=True, timeout=None):
+        """Get a node off the inner priority queue. By default, this blocks.
+
+        This takes the lock, but only for part of it.
+
+        :param bool block: If True, block until the inner queue has data
+        :param Optional[float] timeout: If set, block for timeout seconds
+            waiting for data.
+        :return ParsedNode: The node as present in the manifest.
+
+        See `queue.PriorityQueue` for more information on `get()` behavior and
+        exceptions.
+        """
+        _, node_id = self.inner.get(block=block, timeout=timeout)
+        with self.lock:
+            self._mark_in_progress(node_id)
+        return self.get_node(node_id)
+
+    def __len__(self):
+        """The length of the queue is the number of tasks left for the queue to
+        give out, regardless of where they are. Incomplete tasks are not part
+        of the length.
+
+        This takes the lock.
+        """
+        with self.lock:
+            return len(self.graph) - len(self.in_progress)
+
+    def empty(self):
+        """The graph queue is 'empty' if it all remaining nodes in the graph
+        are in progress.
+
+        This takes the lock.
+        """
+        return len(self) == 0
+
+    def _already_known(self, node):
+        """Decide if a node is already known (either handed out as a task, or
+        in the queue).
+
+        Callers must hold the lock.
+
+        :param str node: The node ID to check
+        :returns bool: If the node is in progress/queued.
+        """
+        return node in self.in_progress or node in self.queued
+
+    def _find_new_additions(self):
+        """Find any nodes in the graph that need to be added to the internal
+        queue and add them.
+
+        Callers must hold the lock.
+        """
+        for node, in_degree in dict(self.graph.in_degree()).items():
+            if not self._already_known(node) and in_degree == 0:
+                self.inner.put((self._scores[node], node))
+                self.queued.add(node)
+
+    def mark_done(self, node_id):
+        """Given a node's unique ID, mark it as done.
+
+        This method takes the lock.
+
+        :param str node_id: The node ID to mark as complete.
+        """
+        with self.lock:
+            self.in_progress.remove(node_id)
+            self.graph.remove_node(node_id)
+            self._find_new_additions()
+            self.inner.task_done()
+
+    def _mark_in_progress(self, node_id):
+        """Mark the node as 'in progress'.
+
+        Callers must hold the lock.
+
+        :param str node_id: The node ID to mark as in progress.
+        """
+        self.queued.remove(node_id)
+        self.in_progress.add(node_id)
+
+    def join(self):
+        """Join the queue. Blocks until all tasks are marked as done.
+
+        Make sure not to call this before the queue reports that it is empty.
+        """
+        self.inner.join()
+
+
+def _subset_graph(graph, include_nodes):
+    """Create and return a new graph that is a shallow copy of graph but with
+    only the nodes in include_nodes. Transitive edges across removed nodes are
+    preserved as explicit new edges.
+    """
+    new_graph = nx.algorithms.transitive_closure(graph)
+
+    include_nodes = set(include_nodes)
+
+    for node in graph.nodes():
+        if node not in include_nodes:
+            new_graph.remove_node(node)
+
+    for node in include_nodes:
+        if node not in new_graph:
+            raise RuntimeError(
+                "Couldn't find model '{}' -- does it exist or is "
+                "it disabled?".format(node)
+            )
+    return new_graph
+
+
+class Linker(object):
+    def __init__(self, data=None):
+        if data is None:
+            data = {}
+        self.graph = nx.DiGraph(**data)
+
+    def edges(self):
+        return self.graph.edges()
+
+    def nodes(self):
+        return self.graph.nodes()
+
+    def find_cycles(self):
+        # There's a networkx find_cycle function, but there's a bug in the
+        # nx 1.11 release that prevents us from using it. We should use that
+        # function when we upgrade to 2.X. More info:
+        #     https://github.com/networkx/networkx/pull/2473
+        cycles = list(nx.simple_cycles(self.graph))
+
+        if len(cycles) > 0:
+            cycle_nodes = cycles[0]
+            cycle_nodes.append(cycle_nodes[0])
+            return " --> ".join(cycle_nodes)
+
+        return None
+
+    def as_graph_queue(self, manifest, limit_to=None):
+        """Returns a queue over nodes in the graph that tracks progress of
+        dependecies.
+        """
+        if limit_to is None:
+            graph_nodes = self.graph.nodes()
+        else:
+            graph_nodes = limit_to
+
+        new_graph = _subset_graph(self.graph, graph_nodes)
+        return GraphQueue(new_graph, manifest)
+
+    def get_dependent_nodes(self, node):
+        return nx.descendants(self.graph, node)
+
+    def dependency(self, node1, node2):
+        "indicate that node1 depends on node2"
+        self.graph.add_node(node1)
+        self.graph.add_node(node2)
+        self.graph.add_edge(node2, node1)
+
+    def add_node(self, node):
+        self.graph.add_node(node)
+
+    def remove_node(self, node):
+        children = nx.descendants(self.graph, node)
+        self.graph.remove_node(node)
+        return children
+
+    def write_graph(self, outfile, manifest):
+        """Write the graph to a gpickle file. Before doing so, serialize and
+        include all nodes in their corresponding graph entries.
+        """
+        out_graph = _updated_graph(self.graph, manifest)
+        nx.write_gpickle(out_graph, outfile)
+
+    def read_graph(self, infile):
+        self.graph = nx.read_gpickle(infile)
+
+
+def _updated_graph(graph, manifest):
+    graph = graph.copy()
+    for node_id in graph.nodes():
+        # serialize() removes the agate table
+        data = manifest.nodes[node_id].serialize()
+        for key in GRAPH_SERIALIZE_BLACKLIST:
+            if key in data:
+                del data[key]
+        graph.add_node(node_id, **data)
+    return graph

core/dbt/links.py

@@ -0,0 +1,3 @@
+ProfileConfigDocs = 'https://docs.getdbt.com/docs/configure-your-profile'
+SnowflakeQuotingDocs = 'https://docs.getdbt.com/v0.10/docs/configuring-quoting'
+IncrementalDocs = 'https://docs.getdbt.com/docs/configuring-incremental-models'

core/dbt/loader.py

@@ -0,0 +1,265 @@
+import os
+import itertools
+
+from dbt.include.global_project import PACKAGES
+import dbt.exceptions
+import dbt.flags
+
+from dbt.node_types import NodeType
+from dbt.contracts.graph.manifest import Manifest
+from dbt.utils import timestring
+
+from dbt.parser import MacroParser, ModelParser, SeedParser, AnalysisParser, \
+    DocumentationParser, DataTestParser, HookParser, SchemaParser, \
+    ParserUtils, SnapshotParser
+
+from dbt.contracts.project import ProjectList
+
+
+class GraphLoader(object):
+    def __init__(self, root_project, all_projects):
+        self.root_project = root_project
+        self.all_projects = all_projects
+        self.nodes = {}
+        self.docs = {}
+        self.macros = {}
+        self.tests = {}
+        self.patches = {}
+        self.disabled = []
+        self.macro_manifest = None
+
+    def _load_sql_nodes(self, parser_type, resource_type, relative_dirs_attr,
+                        **kwargs):
+        parser = parser_type(self.root_project, self.all_projects,
+                             self.macro_manifest)
+
+        for project_name, project in self.all_projects.items():
+            parse_results = parser.load_and_parse(
+                package_name=project_name,
+                root_dir=project.project_root,
+                relative_dirs=getattr(project, relative_dirs_attr),
+                resource_type=resource_type,
+                **kwargs
+            )
+            self.nodes.update(parse_results.parsed)
+            self.disabled.extend(parse_results.disabled)
+
+    def _load_macros(self, internal_manifest=None):
+        # skip any projects in the internal manifest
+        all_projects = self.all_projects.copy()
+        if internal_manifest is not None:
+            for name in internal_project_names():
+                all_projects.pop(name, None)
+            self.macros.update(internal_manifest.macros)
+
+        # give the macroparser all projects but then only load what we haven't
+        # loaded already
+        parser = MacroParser(self.root_project, self.all_projects)
+        for project_name, project in all_projects.items():
+            self.macros.update(parser.load_and_parse(
+                package_name=project_name,
+                root_dir=project.project_root,
+                relative_dirs=project.macro_paths,
+                resource_type=NodeType.Macro,
+            ))
+
+    def _load_seeds(self):
+        parser = SeedParser(self.root_project, self.all_projects,
+                            self.macro_manifest)
+        for project_name, project in self.all_projects.items():
+            self.nodes.update(parser.load_and_parse(
+                package_name=project_name,
+                root_dir=project.project_root,
+                relative_dirs=project.data_paths,
+            ))
+
+    def _load_nodes(self):
+        self._load_sql_nodes(ModelParser, NodeType.Model, 'source_paths')
+        self._load_sql_nodes(SnapshotParser, NodeType.Snapshot,
+                             'snapshot_paths')
+        self._load_sql_nodes(AnalysisParser, NodeType.Analysis,
+                             'analysis_paths')
+        self._load_sql_nodes(DataTestParser, NodeType.Test, 'test_paths',
+                             tags=['data'])
+
+        hook_parser = HookParser(self.root_project, self.all_projects,
+                                 self.macro_manifest)
+        self.nodes.update(hook_parser.load_and_parse())
+
+        self._load_seeds()
+
+    def _load_docs(self):
+        parser = DocumentationParser(self.root_project, self.all_projects)
+        for project_name, project in self.all_projects.items():
+            self.docs.update(parser.load_and_parse(
+                package_name=project_name,
+                root_dir=project.project_root,
+                relative_dirs=project.docs_paths
+            ))
+
+    def _load_schema_tests(self):
+        parser = SchemaParser(self.root_project, self.all_projects,
+                              self.macro_manifest)
+        for project_name, project in self.all_projects.items():
+            tests, patches, sources = parser.load_and_parse(
+                package_name=project_name,
+                root_dir=project.project_root,
+                relative_dirs=project.source_paths
+            )
+
+            for unique_id, test in tests.items():
+                if unique_id in self.tests:
+                    dbt.exceptions.raise_duplicate_resource_name(
+                        test, self.tests[unique_id],
+                    )
+                self.tests[unique_id] = test
+
+            for unique_id, source in sources.items():
+                if unique_id in self.nodes:
+                    dbt.exceptions.raise_duplicate_resource_name(
+                        source, self.nodes[unique_id],
+                    )
+                self.nodes[unique_id] = source
+
+            for name, patch in patches.items():
+                if name in self.patches:
+                    dbt.exceptions.raise_duplicate_patch_name(
+                        name, patch, self.patches[name]
+                    )
+                self.patches[name] = patch
+
+    def load(self, internal_manifest=None):
+        self._load_macros(internal_manifest=internal_manifest)
+        # make a manifest with just the macros to get the context
+        self.macro_manifest = Manifest(macros=self.macros, nodes={}, docs={},
+                                       generated_at=timestring(), disabled=[])
+        self._load_nodes()
+        self._load_docs()
+        self._load_schema_tests()
+
+    def create_manifest(self):
+        manifest = Manifest(
+            nodes=self.nodes,
+            macros=self.macros,
+            docs=self.docs,
+            generated_at=timestring(),
+            config=self.root_project,
+            disabled=self.disabled
+        )
+        manifest.add_nodes(self.tests)
+        manifest.patch_nodes(self.patches)
+        manifest = ParserUtils.process_sources(manifest, self.root_project)
+        manifest = ParserUtils.process_refs(manifest,
+                                            self.root_project.project_name)
+        manifest = ParserUtils.process_docs(manifest, self.root_project)
+        return manifest
+
+    @classmethod
+    def _load_from_projects(cls, root_config, projects, internal_manifest):
+        if dbt.flags.STRICT_MODE:
+            ProjectList(**projects)
+
+        loader = cls(root_config, projects)
+        loader.load(internal_manifest=internal_manifest)
+        return loader.create_manifest()
+
+    @classmethod
+    def load_all(cls, root_config, internal_manifest=None):
+        projects = load_all_projects(root_config)
+        manifest = cls._load_from_projects(root_config, projects,
+                                           internal_manifest)
+        _check_manifest(manifest, root_config)
+        return manifest
+
+    @classmethod
+    def load_internal(cls, root_config):
+        projects = load_internal_projects(root_config)
+        return cls._load_from_projects(root_config, projects, None)
+
+
+def _check_resource_uniqueness(manifest):
+    names_resources = {}
+    alias_resources = {}
+
+    for resource, node in manifest.nodes.items():
+        if node.resource_type not in NodeType.refable():
+            continue
+
+        name = node.name
+        alias = "{}.{}".format(node.schema, node.alias)
+
+        existing_node = names_resources.get(name)
+        if existing_node is not None:
+            dbt.exceptions.raise_duplicate_resource_name(
+                existing_node, node
+            )
+
+        existing_alias = alias_resources.get(alias)
+        if existing_alias is not None:
+            dbt.exceptions.raise_ambiguous_alias(
+                existing_alias, node
+            )
+
+        names_resources[name] = node
+        alias_resources[alias] = node
+
+
+def _warn_for_unused_resource_config_paths(manifest, config):
+    resource_fqns = manifest.get_resource_fqns()
+    disabled_fqns = [n.fqn for n in manifest.disabled]
+    config.warn_for_unused_resource_config_paths(resource_fqns, disabled_fqns)
+
+
+def _check_manifest(manifest, config):
+    _check_resource_uniqueness(manifest)
+    _warn_for_unused_resource_config_paths(manifest, config)
+
+
+def internal_project_names():
+    return iter(PACKAGES.values())
+
+
+def _load_projects(config, paths):
+    for path in paths:
+        try:
+            project = config.new_project(path)
+        except dbt.exceptions.DbtProjectError as e:
+            raise dbt.exceptions.DbtProjectError(
+                'Failed to read package at {}: {}'
+                .format(path, e)
+            )
+        else:
+            yield project.project_name, project
+
+
+def _project_directories(config):
+    root = os.path.join(config.project_root, config.modules_path)
+
+    dependencies = []
+    if os.path.exists(root):
+        dependencies = os.listdir(root)
+
+    for name in dependencies:
+        full_obj = os.path.join(root, name)
+
+        if not os.path.isdir(full_obj) or name.startswith('__'):
+            # exclude non-dirs and dirs that start with __
+            # the latter could be something like __pycache__
+            # for the global dbt modules dir
+            continue
+
+        yield full_obj
+
+
+def load_all_projects(config):
+    all_projects = {config.project_name: config}
+    project_paths = itertools.chain(
+        internal_project_names(),
+        _project_directories(config)
+    )
+    all_projects.update(_load_projects(config, project_paths))
+    return all_projects
+
+
+def load_internal_projects(config):
+    return dict(_load_projects(config, internal_project_names()))

core/dbt/logger.py

@@ -0,0 +1,229 @@
+import dbt.compat
+import dbt.flags
+import logging
+import logging.handlers
+import os
+import sys
+
+import colorama
+
+# Colorama needs some help on windows because we're using logger.info
+# intead of print(). If the Windows env doesn't have a TERM var set,
+# then we should override the logging stream to use the colorama
+# converter. If the TERM var is set (as with Git Bash), then it's safe
+# to send escape characters and no log handler injection is needed.
+colorama_stdout = sys.stdout
+colorama_wrap = True
+
+colorama.init(wrap=colorama_wrap)
+
+DEBUG = logging.DEBUG
+NOTICE = 15
+INFO = logging.INFO
+WARNING = logging.WARNING
+ERROR = logging.ERROR
+CRITICAL = logging.CRITICAL
+
+logging.addLevelName(NOTICE, 'NOTICE')
+
+
+class Logger(logging.Logger):
+    def notice(self, msg, *args, **kwargs):
+        if self.isEnabledFor(NOTICE):
+            self._log(NOTICE, msg, args, **kwargs)
+
+
+logging.setLoggerClass(Logger)
+
+
+if sys.platform == 'win32' and not os.environ.get('TERM'):
+    colorama_wrap = False
+    colorama_stdout = colorama.AnsiToWin32(sys.stdout).stream
+
+elif sys.platform == 'win32':
+    colorama_wrap = False
+
+colorama.init(wrap=colorama_wrap)
+
+# create a global console logger for dbt
+stdout_handler = logging.StreamHandler(colorama_stdout)
+stdout_handler.setFormatter(logging.Formatter('%(message)s'))
+stdout_handler.setLevel(NOTICE)
+
+stderr_handler = logging.StreamHandler(sys.stderr)
+stderr_handler.setFormatter(logging.Formatter('%(message)s'))
+stderr_handler.setLevel(WARNING)
+
+
+logger = logging.getLogger('dbt')
+logger.addHandler(stdout_handler)
+logger.setLevel(DEBUG)
+logging.getLogger().setLevel(CRITICAL)
+
+# Quiet these down in the logs
+logging.getLogger('botocore').setLevel(INFO)
+logging.getLogger('requests').setLevel(INFO)
+logging.getLogger('urllib3').setLevel(INFO)
+logging.getLogger('google').setLevel(INFO)
+logging.getLogger('snowflake.connector').setLevel(INFO)
+logging.getLogger('parsedatetime').setLevel(INFO)
+# we never want to seek werkzeug logs
+logging.getLogger('werkzeug').setLevel(CRITICAL)
+
+# provide this for the cache.
+CACHE_LOGGER = logging.getLogger('dbt.cache')
+# add a dummy handler to avoid `No handlers could be found for logger`
+nothing_handler = logging.StreamHandler()
+nothing_handler.setLevel(CRITICAL)
+CACHE_LOGGER.addHandler(nothing_handler)
+# provide this for RPC connection logging
+RPC_LOGGER = logging.getLogger('dbt.rpc')
+
+
+# Redirect warnings through our logging setup
+# They will be logged to a file below
+logging.captureWarnings(True)
+dbt.compat.suppress_warnings()
+
+initialized = False
+
+
+def _swap_handler(logger, old, new):
+    if old in logger.handlers:
+        logger.handlers.remove(old)
+    if new not in logger.handlers:
+        logger.addHandler(new)
+
+
+def log_to_stderr(logger):
+    _swap_handler(logger, stdout_handler, stderr_handler)
+
+
+def log_to_stdout(logger):
+    _swap_handler(logger, stderr_handler, stdout_handler)
+
+
+def make_log_dir_if_missing(log_dir):
+    import dbt.clients.system
+    dbt.clients.system.make_directory(log_dir)
+
+
+class ColorFilter(logging.Filter):
+    def filter(self, record):
+        subbed = dbt.compat.to_string(record.msg)
+        for escape_sequence in dbt.ui.colors.COLORS.values():
+            subbed = subbed.replace(escape_sequence, '')
+        record.msg = subbed
+
+        return True
+
+
+def default_formatter():
+    return logging.Formatter('%(asctime)-18s (%(threadName)s): %(message)s')
+
+
+def initialize_logger(debug_mode=False, path=None):
+    global initialized, logger, stdout_handler, stderr_handler
+
+    if initialized:
+        return
+
+    if debug_mode:
+        # we'll only use one of these, but just set both up
+        stdout_handler.setFormatter(default_formatter())
+        stdout_handler.setLevel(DEBUG)
+        stderr_handler.setFormatter(default_formatter())
+        stderr_handler.setLevel(DEBUG)
+
+    if path is not None:
+        make_log_dir_if_missing(path)
+        log_path = os.path.join(path, 'dbt.log')
+
+        # log to directory as well
+        logdir_handler = logging.handlers.TimedRotatingFileHandler(
+            filename=log_path,
+            when='d',
+            interval=1,
+            backupCount=7,
+        )
+
+        color_filter = ColorFilter()
+        logdir_handler.addFilter(color_filter)
+
+        logdir_handler.setFormatter(default_formatter())
+        logdir_handler.setLevel(DEBUG)
+
+        logger.addHandler(logdir_handler)
+
+        # Log Python warnings to file
+        warning_logger = logging.getLogger('py.warnings')
+        warning_logger.addHandler(logdir_handler)
+        warning_logger.setLevel(DEBUG)
+
+    initialized = True
+
+
+def logger_initialized():
+    return initialized
+
+
+def log_cache_events(flag):
+    """Set the cache logger to propagate its messages based on the given flag.
+    """
+    CACHE_LOGGER.propagate = flag
+
+
+GLOBAL_LOGGER = logger
+
+
+class QueueFormatter(logging.Formatter):
+    def formatMessage(self, record):
+        superself = super(QueueFormatter, self)
+        if hasattr(superself, 'formatMessage'):
+            # python 3.x
+            return superself.formatMessage(record)
+
+        # python 2.x, handling weird unicode things
+        try:
+            return self._fmt % record.__dict__
+        except UnicodeDecodeError:
+            try:
+                record.name = record.name.decode('utf-8')
+                return self._fmt % record.__dict__
+            except UnicodeDecodeError as e:
+                raise e
+
+    def format(self, record):
+        record.message = record.getMessage()
+        record.asctime = self.formatTime(record, self.datefmt)
+        formatted = self.formatMessage(record)
+
+        output = {
+            'message': formatted,
+            'timestamp': record.asctime,
+            'levelname': record.levelname,
+            'level': record.levelno,
+        }
+        if record.exc_info:
+            if not record.exc_text:
+                record.exc_text = self.formatException(record.exc_info)
+            output['exc_info'] = record.exc_text
+        return output
+
+
+class QueueLogHandler(logging.Handler):
+    def __init__(self, queue):
+        super(QueueLogHandler, self).__init__()
+        self.queue = queue
+
+    def emit(self, record):
+        msg = self.format(record)
+        self.queue.put_nowait(['log', msg])
+
+
+def add_queue_handler(queue):
+    """Add a queue log handler to the global logger."""
+    handler = QueueLogHandler(queue)
+    handler.setFormatter(QueueFormatter())
+    handler.setLevel(DEBUG)
+    GLOBAL_LOGGER.addHandler(handler)

core/dbt/main.py

@@ -0,0 +1,811 @@
+from dbt.logger import initialize_logger, GLOBAL_LOGGER as logger, \
+    logger_initialized, log_cache_events
+
+import argparse
+import os.path
+import sys
+import traceback
+from contextlib import contextmanager
+
+import dbt.version
+import dbt.flags as flags
+import dbt.task.run as run_task
+import dbt.task.compile as compile_task
+import dbt.task.debug as debug_task
+import dbt.task.clean as clean_task
+import dbt.task.deps as deps_task
+import dbt.task.init as init_task
+import dbt.task.seed as seed_task
+import dbt.task.test as test_task
+import dbt.task.snapshot as snapshot_task
+import dbt.task.generate as generate_task
+import dbt.task.serve as serve_task
+import dbt.task.freshness as freshness_task
+import dbt.task.run_operation as run_operation_task
+from dbt.task.list import ListTask
+from dbt.task.migrate import MigrationTask
+from dbt.task.rpc_server import RPCServerTask
+from dbt.adapters.factory import reset_adapters
+
+import dbt.tracking
+import dbt.ui.printer
+import dbt.compat
+import dbt.deprecations
+import dbt.profiler
+
+from dbt.utils import ExitCodes
+from dbt.config import UserConfig, PROFILES_DIR
+from dbt.exceptions import RuntimeException
+
+
+PROFILES_HELP_MESSAGE = """
+For more information on configuring profiles, please consult the dbt docs:
+
+https://docs.getdbt.com/docs/configure-your-profile
+"""
+
+
+class DBTVersion(argparse.Action):
+    """This is very very similar to the builtin argparse._Version action,
+    except it just calls dbt.version.get_version_information().
+    """
+    def __init__(self,
+                 option_strings,
+                 version=None,
+                 dest=argparse.SUPPRESS,
+                 default=argparse.SUPPRESS,
+                 help="show program's version number and exit"):
+        super(DBTVersion, self).__init__(
+            option_strings=option_strings,
+            dest=dest,
+            default=default,
+            nargs=0,
+            help=help)
+
+    def __call__(self, parser, namespace, values, option_string=None):
+        formatter = parser._get_formatter()
+        formatter.add_text(dbt.version.get_version_information())
+        parser.exit(message=formatter.format_help())
+
+
+class DBTArgumentParser(argparse.ArgumentParser):
+    def __init__(self, *args, **kwargs):
+        super(DBTArgumentParser, self).__init__(*args, **kwargs)
+        self.register('action', 'dbtversion', DBTVersion)
+
+
+def main(args=None):
+    if args is None:
+        args = sys.argv[1:]
+
+    try:
+        results, succeeded = handle_and_check(args)
+        if succeeded:
+            exit_code = ExitCodes.Success
+        else:
+            exit_code = ExitCodes.ModelError
+
+    except KeyboardInterrupt:
+        logger.info("ctrl-c")
+        exit_code = ExitCodes.UnhandledError
+
+    # This can be thrown by eg. argparse
+    except SystemExit as e:
+        exit_code = e.code
+
+    except BaseException as e:
+        logger.warn("Encountered an error:")
+        logger.warn(str(e))
+
+        if logger_initialized():
+            logger.debug(traceback.format_exc())
+        elif not isinstance(e, RuntimeException):
+            # if it did not come from dbt proper and the logger is not
+            # initialized (so there's no safe path to log to), log the stack
+            # trace at error level.
+            logger.error(traceback.format_exc())
+        exit_code = ExitCodes.UnhandledError
+
+    _python2_compatibility_message()
+
+    sys.exit(exit_code)
+
+
+# here for backwards compatibility
+def handle(args):
+    res, success = handle_and_check(args)
+    return res
+
+
+def initialize_config_values(parsed):
+    """Given the parsed args, initialize the dbt tracking code.
+
+    It would be nice to re-use this profile later on instead of parsing it
+    twice, but dbt's intialization is not structured in a way that makes that
+    easy.
+    """
+    try:
+        cfg = UserConfig.from_directory(parsed.profiles_dir)
+    except RuntimeException:
+        cfg = UserConfig.from_dict(None)
+
+    cfg.set_values(parsed.profiles_dir)
+
+
+def handle_and_check(args):
+    parsed = parse_args(args)
+    profiler_enabled = False
+
+    if parsed.record_timing_info:
+        profiler_enabled = True
+
+    with dbt.profiler.profiler(
+        enable=profiler_enabled,
+        outfile=parsed.record_timing_info
+    ):
+
+        initialize_config_values(parsed)
+
+        reset_adapters()
+
+        task, res = run_from_args(parsed)
+        success = task.interpret_results(res)
+
+        return res, success
+
+
+@contextmanager
+def track_run(task):
+    dbt.tracking.track_invocation_start(config=task.config, args=task.args)
+    try:
+        yield
+        dbt.tracking.track_invocation_end(
+            config=task.config, args=task.args, result_type="ok"
+        )
+    except (dbt.exceptions.NotImplementedException,
+            dbt.exceptions.FailedToConnectException) as e:
+        logger.error('ERROR: {}'.format(e))
+        dbt.tracking.track_invocation_end(
+            config=task.config, args=task.args, result_type="error"
+        )
+    except Exception:
+        dbt.tracking.track_invocation_end(
+            config=task.config, args=task.args, result_type="error"
+        )
+        raise
+    finally:
+        dbt.tracking.flush()
+
+
+_PYTHON_27_WARNING = '''
+Python 2.7 will reach the end of its life on January 1st, 2020.
+Please upgrade your Python as Python 2.7 won't be maintained after that date.
+A future version of dbt will drop support for Python 2.7.
+'''.strip()
+
+
+def _python2_compatibility_message():
+    if dbt.compat.WHICH_PYTHON != 2:
+        return
+
+    logger.critical(
+        dbt.ui.printer.red('DEPRECATION: ') + _PYTHON_27_WARNING
+    )
+
+
+def run_from_args(parsed):
+    log_cache_events(getattr(parsed, 'log_cache_events', False))
+    flags.set_from_args(parsed)
+
+    parsed.cls.pre_init_hook()
+    logger.info("Running with dbt{}".format(dbt.version.installed))
+
+    # this will convert DbtConfigErrors into RuntimeExceptions
+    task = parsed.cls.from_args(args=parsed)
+    logger.debug("running dbt with arguments %s", parsed)
+
+    log_path = None
+    if task.config is not None:
+        log_path = getattr(task.config, 'log_path', None)
+    initialize_logger(parsed.debug, log_path)
+    logger.debug("Tracking: {}".format(dbt.tracking.active_user.state()))
+
+    results = None
+
+    with track_run(task):
+        results = task.run()
+
+    return task, results
+
+
+def _build_base_subparser():
+    base_subparser = argparse.ArgumentParser(add_help=False)
+
+    base_subparser.add_argument(
+        '--project-dir',
+        default=None,
+        type=str,
+        help="""
+        Which directory to look in for the dbt_project.yml file.
+        Default is the current working directory and its parents.
+        """
+    )
+
+    base_subparser.add_argument(
+        '--profiles-dir',
+        default=PROFILES_DIR,
+        type=str,
+        help="""
+        Which directory to look in for the profiles.yml file. Default = {}
+        """.format(PROFILES_DIR)
+    )
+
+    base_subparser.add_argument(
+        '--profile',
+        required=False,
+        type=str,
+        help="""
+        Which profile to load. Overrides setting in dbt_project.yml.
+        """
+    )
+
+    base_subparser.add_argument(
+        '--target',
+        default=None,
+        type=str,
+        help='Which target to load for the given profile'
+    )
+
+    base_subparser.add_argument(
+        '--vars',
+        type=str,
+        default='{}',
+        help="""
+            Supply variables to the project. This argument overrides
+            variables defined in your dbt_project.yml file. This argument
+            should be a YAML string, eg. '{my_variable: my_value}'"""
+    )
+
+    # if set, log all cache events. This is extremely verbose!
+    base_subparser.add_argument(
+        '--log-cache-events',
+        action='store_true',
+        help=argparse.SUPPRESS,
+    )
+
+    base_subparser.add_argument(
+        '--bypass-cache',
+        action='store_false',
+        dest='use_cache',
+        help='If set, bypass the adapter-level cache of database state',
+    )
+    return base_subparser
+
+
+def _build_docs_subparser(subparsers, base_subparser):
+    docs_sub = subparsers.add_parser(
+        'docs',
+        parents=[base_subparser],
+        help="Generate or serve the documentation "
+        "website for your project.")
+    return docs_sub
+
+
+def _build_source_subparser(subparsers, base_subparser):
+    source_sub = subparsers.add_parser(
+        'source',
+        parents=[base_subparser],
+        help="Manage your project's sources")
+    return source_sub
+
+
+def _build_init_subparser(subparsers, base_subparser):
+    sub = subparsers.add_parser(
+        'init',
+        parents=[base_subparser],
+        help="Initialize a new DBT project.")
+    sub.add_argument('project_name', type=str, help='Name of the new project')
+    sub.set_defaults(cls=init_task.InitTask, which='init')
+    return sub
+
+
+def _build_clean_subparser(subparsers, base_subparser):
+    sub = subparsers.add_parser(
+        'clean',
+        parents=[base_subparser],
+        help="Delete all folders in the clean-targets list"
+        "\n(usually the dbt_modules and target directories.)")
+    sub.set_defaults(cls=clean_task.CleanTask, which='clean')
+    return sub
+
+
+def _build_debug_subparser(subparsers, base_subparser):
+    sub = subparsers.add_parser(
+        'debug',
+        parents=[base_subparser],
+        help="Show some helpful information about dbt for debugging."
+        "\nNot to be confused with the --debug option which increases "
+        "verbosity.")
+    sub.add_argument(
+        '--config-dir',
+        action='store_true',
+        help="""
+        If specified, DBT will show path information for this project
+        """
+    )
+    sub.set_defaults(cls=debug_task.DebugTask, which='debug')
+    return sub
+
+
+def _build_deps_subparser(subparsers, base_subparser):
+    sub = subparsers.add_parser(
+        'deps',
+        parents=[base_subparser],
+        help="Pull the most recent version of the dependencies "
+        "listed in packages.yml")
+    sub.set_defaults(cls=deps_task.DepsTask, which='deps')
+    return sub
+
+
+def _build_snapshot_subparser(subparsers, base_subparser, which='snapshot'):
+    if which == 'archive':
+        helpmsg = (
+            'DEPRECATED: This command is deprecated and will\n'
+            'be removed in a future release. Use dbt snapshot instead.'
+        )
+    else:
+        helpmsg = 'Execute snapshots defined in your project'
+
+    sub = subparsers.add_parser(
+        which,
+        parents=[base_subparser],
+        help=helpmsg)
+    sub.add_argument(
+        '--threads',
+        type=int,
+        required=False,
+        help="""
+        Specify number of threads to use while snapshotting tables. Overrides
+        settings in profiles.yml.
+        """
+    )
+    sub.set_defaults(cls=snapshot_task.SnapshotTask, which=which)
+    return sub
+
+
+def _build_run_subparser(subparsers, base_subparser):
+    run_sub = subparsers.add_parser(
+        'run',
+        parents=[base_subparser],
+        help="Compile SQL and execute against the current "
+        "target database.")
+    run_sub.set_defaults(cls=run_task.RunTask, which='run')
+    return run_sub
+
+
+def _build_compile_subparser(subparsers, base_subparser):
+    sub = subparsers.add_parser(
+        'compile',
+        parents=[base_subparser],
+        help="Generates executable SQL from source model, test, and"
+        "analysis files. \nCompiled SQL files are written to the target/"
+        "directory.")
+    sub.set_defaults(cls=compile_task.CompileTask, which='compile')
+    return sub
+
+
+def _build_docs_generate_subparser(subparsers, base_subparser):
+    # it might look like docs_sub is the correct parents entry, but that
+    # will cause weird errors about 'conflicting option strings'.
+    generate_sub = subparsers.add_parser('generate', parents=[base_subparser])
+    generate_sub.set_defaults(cls=generate_task.GenerateTask,
+                              which='generate')
+    generate_sub.add_argument(
+        '--no-compile',
+        action='store_false',
+        dest='compile',
+        help='Do not run "dbt compile" as part of docs generation'
+    )
+    return generate_sub
+
+
+def _add_selection_arguments(*subparsers, **kwargs):
+    models_name = kwargs.get('models_name', 'models')
+    for sub in subparsers:
+        sub.add_argument(
+            '-{}'.format(models_name[0]),
+            '--{}'.format(models_name),
+            dest='models',
+            required=False,
+            nargs='+',
+            help="""
+            Specify the models to include.
+            """
+        )
+        sub.add_argument(
+            '--exclude',
+            required=False,
+            nargs='+',
+            help="""
+            Specify the models to exclude.
+            """
+        )
+
+
+def _add_table_mutability_arguments(*subparsers):
+    for sub in subparsers:
+        sub.add_argument(
+            '--full-refresh',
+            action='store_true',
+            help="""
+            If specified, DBT will drop incremental models and
+            fully-recalculate the incremental table from the model definition.
+            """)
+
+
+def _add_common_arguments(*subparsers):
+    for sub in subparsers:
+        sub.add_argument(
+            '--threads',
+            type=int,
+            required=False,
+            help="""
+            Specify number of threads to use while executing models. Overrides
+            settings in profiles.yml.
+            """
+        )
+        sub.add_argument(
+            '--no-version-check',
+            dest='version_check',
+            action='store_false',
+            help="""
+            If set, skip ensuring dbt's version matches the one specified in
+            the dbt_project.yml file ('require-dbt-version')
+            """)
+
+
+def _build_seed_subparser(subparsers, base_subparser):
+    seed_sub = subparsers.add_parser(
+        'seed',
+        parents=[base_subparser],
+        help="Load data from csv files into your data warehouse.")
+    seed_sub.add_argument(
+        '--full-refresh',
+        action='store_true',
+        help='Drop existing seed tables and recreate them'
+    )
+    seed_sub.add_argument(
+        '--show',
+        action='store_true',
+        help='Show a sample of the loaded data in the terminal'
+    )
+    seed_sub.set_defaults(cls=seed_task.SeedTask, which='seed')
+    return seed_sub
+
+
+def _build_docs_serve_subparser(subparsers, base_subparser):
+    serve_sub = subparsers.add_parser('serve', parents=[base_subparser])
+    serve_sub.add_argument(
+        '--port',
+        default=8080,
+        type=int,
+        help='Specify the port number for the docs server.'
+    )
+    serve_sub.set_defaults(cls=serve_task.ServeTask, which='serve')
+    return serve_sub
+
+
+def _build_test_subparser(subparsers, base_subparser):
+    sub = subparsers.add_parser(
+        'test',
+        parents=[base_subparser],
+        help="Runs tests on data in deployed models."
+        "Run this after `dbt run`")
+    sub.add_argument(
+        '--data',
+        action='store_true',
+        help='Run data tests defined in "tests" directory.'
+    )
+    sub.add_argument(
+        '--schema',
+        action='store_true',
+        help='Run constraint validations from schema.yml files'
+    )
+
+    sub.set_defaults(cls=test_task.TestTask, which='test')
+    return sub
+
+
+def _build_source_snapshot_freshness_subparser(subparsers, base_subparser):
+    sub = subparsers.add_parser(
+        'snapshot-freshness',
+        parents=[base_subparser],
+        help="Snapshots the current freshness of the project's sources",
+    )
+    sub.add_argument(
+        '-s',
+        '--select',
+        required=False,
+        nargs='+',
+        help="""
+        Specify the sources to snapshot freshness
+        """,
+        dest='selected'
+    )
+    sub.add_argument(
+        '-o',
+        '--output',
+        required=False,
+        help="""
+        Specify the output path for the json report. By default, outputs to
+        target/sources.json
+        """
+    )
+    sub.add_argument(
+        '--threads',
+        type=int,
+        required=False,
+        help="""
+        Specify number of threads to use. Overrides settings in profiles.yml
+        """
+    )
+    sub.set_defaults(cls=freshness_task.FreshnessTask,
+                     which='snapshot-freshness')
+    return sub
+
+
+def _build_rpc_subparser(subparsers, base_subparser):
+    sub = subparsers.add_parser(
+        'rpc',
+        parents=[base_subparser],
+        help='Start a json-rpc server',
+    )
+    sub.add_argument(
+        '--host',
+        default='0.0.0.0',
+        help='Specify the host to listen on for the rpc server.'
+    )
+    sub.add_argument(
+        '--port',
+        default=8580,
+        type=int,
+        help='Specify the port number for the rpc server.'
+    )
+    sub.set_defaults(cls=RPCServerTask, which='rpc')
+    # the rpc task does a 'compile', so we need these attributes to exist, but
+    # we don't want users to be allowed to set them.
+    sub.set_defaults(models=None, exclude=None)
+    return sub
+
+
+def _build_list_subparser(subparsers, base_subparser):
+    sub = subparsers.add_parser(
+        'list',
+        parents=[base_subparser],
+        help='List the resources in your project'
+    )
+    sub.set_defaults(cls=ListTask, which='list')
+    resource_values = list(ListTask.ALL_RESOURCE_VALUES) + ['default', 'all']
+    sub.add_argument('--resource-type',
+                     choices=resource_values,
+                     action='append',
+                     default=[],
+                     dest='resource_types')
+    sub.add_argument('--output',
+                     choices=['json', 'name', 'path', 'selector'],
+                     default='selector')
+    sub.add_argument(
+        '-s',
+        '--select',
+        required=False,
+        nargs='+',
+        metavar='SELECTOR',
+        help="Specify the nodes to select.",
+    )
+    sub.add_argument(
+        '-m',
+        '--models',
+        required=False,
+        nargs='+',
+        metavar='SELECTOR',
+        help="Specify the models to select and set the resource-type to "
+              "'model'. Mutually exclusive with '--select' (or '-s') and "
+              "'--resource-type'",
+    )
+    sub.add_argument(
+        '--exclude',
+        required=False,
+        nargs='+',
+        metavar='SELECTOR',
+        help="Specify the models to exclude."
+    )
+    # in python 3.x you can use the 'aliases' kwarg, but in python 2.7 you get
+    # to do this
+    subparsers._name_parser_map['ls'] = sub
+    return sub
+
+
+def _build_run_operation_subparser(subparsers, base_subparser):
+    sub = subparsers.add_parser(
+        'run-operation',
+        parents=[base_subparser],
+        help="""
+            (beta) Run the named macro with any supplied arguments. This
+            subcommand is unstable and subject to change in a future release
+            of dbt. Please use it with caution"""
+    )
+    sub.add_argument(
+        'macro',
+        help="""
+            Specify the macro to invoke. dbt will call this macro with the
+            supplied arguments and then exit"""
+    )
+    sub.add_argument(
+        '--args',
+        type=str,
+        default='{}',
+        help="""
+            Supply arguments to the macro. This dictionary will be mapped
+            to the keyword arguments defined in the selected macro. This
+            argument should be a YAML string, eg. '{my_variable: my_value}'"""
+    )
+    sub.set_defaults(cls=run_operation_task.RunOperationTask,
+                     which='run-operation')
+    return sub
+
+
+def _build_snapshot_migrate_subparser(subparsers, base_subparser):
+    sub = subparsers.add_parser(
+        'snapshot-migrate',
+        parents=[base_subparser],
+        help='Run the snapshot migration script'
+    )
+    sub.add_argument(
+        '--from-archive',
+        action='store_true',
+        help=('This flag is required for the 0.14.0 archive to snapshot '
+              'migration')
+    )
+    sub.add_argument(
+        '--apply-files',
+        action='store_true',
+        dest='write_files',
+        help='If set, write .sql files to disk instead of logging them'
+    )
+    sub.add_argument(
+        '--apply-database',
+        action='store_true',
+        dest='migrate_database',
+        help='If set, perform just the database migration'
+    )
+    sub.add_argument(
+        '--apply',
+        action='store_true',
+        help='If set, implies --apply-database --apply-files'
+    )
+    sub.set_defaults(cls=MigrationTask, which='migration')
+
+
+def parse_args(args):
+    p = DBTArgumentParser(
+        prog='dbt',
+        formatter_class=argparse.RawTextHelpFormatter,
+        description="An ELT tool for managing your SQL "
+        "transformations and data models."
+        "\nFor more documentation on these commands, visit: "
+        "docs.getdbt.com",
+        epilog="Specify one of these sub-commands and you can "
+        "find more help from there.")
+
+    p.add_argument(
+        '--version',
+        action='dbtversion',
+        help="Show version information")
+
+    p.add_argument(
+        '-r',
+        '--record-timing-info',
+        default=None,
+        type=str,
+        help="""
+        When this option is passed, dbt will output low-level timing
+        stats to the specified file. Example:
+        `--record-timing-info output.profile`
+        """
+    )
+
+    p.add_argument(
+        '-d',
+        '--debug',
+        action='store_true',
+        help='''Display debug logging during dbt execution. Useful for
+        debugging and making bug reports.''')
+
+    p.add_argument(
+        '-S',
+        '--strict',
+        action='store_true',
+        help='''Run schema validations at runtime. This will surface
+        bugs in dbt, but may incur a performance penalty.''')
+
+    p.add_argument(
+        '--warn-error',
+        action='store_true',
+        help='''If dbt would normally warn, instead raise an exception.
+        Examples include --models that selects nothing, deprecations,
+        configurations with no associated models, invalid test configurations,
+        and missing sources/refs in tests''')
+
+    # if set, run dbt in single-threaded mode: thread count is ignored, and
+    # calls go through `map` instead of the thread pool. This is useful for
+    # getting performance information about aspects of dbt that normally run in
+    # a thread, as the profiler ignores child threads. Users should really
+    # never use this.
+    p.add_argument(
+        '--single-threaded',
+        action='store_true',
+        help=argparse.SUPPRESS,
+    )
+
+    # if set, extract all models and blocks with the jinja block extractor, and
+    # verify that we don't fail anywhere the actual jinja parser passes. The
+    # reverse (passing files that ends up failing jinja) is fine.
+    p.add_argument(
+        '--test-new-parser',
+        action='store_true',
+        help=argparse.SUPPRESS
+    )
+
+    subs = p.add_subparsers(title="Available sub-commands")
+
+    base_subparser = _build_base_subparser()
+
+    # make the subcommands that have their own subcommands
+    docs_sub = _build_docs_subparser(subs, base_subparser)
+    docs_subs = docs_sub.add_subparsers(title="Available sub-commands")
+    source_sub = _build_source_subparser(subs, base_subparser)
+    source_subs = source_sub.add_subparsers(title="Available sub-commands")
+
+    _build_init_subparser(subs, base_subparser)
+    _build_clean_subparser(subs, base_subparser)
+    _build_debug_subparser(subs, base_subparser)
+    _build_deps_subparser(subs, base_subparser)
+    _build_list_subparser(subs, base_subparser)
+    _build_snapshot_migrate_subparser(subs, base_subparser)
+
+    snapshot_sub = _build_snapshot_subparser(subs, base_subparser)
+    archive_sub = _build_snapshot_subparser(subs, base_subparser, 'archive')
+    rpc_sub = _build_rpc_subparser(subs, base_subparser)
+    run_sub = _build_run_subparser(subs, base_subparser)
+    compile_sub = _build_compile_subparser(subs, base_subparser)
+    generate_sub = _build_docs_generate_subparser(docs_subs, base_subparser)
+    test_sub = _build_test_subparser(subs, base_subparser)
+    # --threads, --no-version-check
+    _add_common_arguments(run_sub, compile_sub, generate_sub, test_sub,
+                          rpc_sub)
+    # --models, --exclude
+    _add_selection_arguments(run_sub, compile_sub, generate_sub, test_sub,
+                             archive_sub)
+    _add_selection_arguments(snapshot_sub, models_name='select')
+    # --full-refresh
+    _add_table_mutability_arguments(run_sub, compile_sub)
+
+    _build_seed_subparser(subs, base_subparser)
+    _build_docs_serve_subparser(docs_subs, base_subparser)
+    _build_source_snapshot_freshness_subparser(source_subs, base_subparser)
+    _build_run_operation_subparser(subs, base_subparser)
+
+    if len(args) == 0:
+        p.print_help()
+        sys.exit(1)
+
+    parsed = p.parse_args(args)
+    parsed.profiles_dir = os.path.expanduser(parsed.profiles_dir)
+
+    if not hasattr(parsed, 'which'):
+        # the user did not provide a valid subcommand. trigger the help message
+        # and exit with a error
+        p.print_help()
+        p.exit(1)
+
+    return parsed