Rm plugins/redshift, plugins/snowflake

.bumpversion.cfg

@@ -34,17 +34,9 @@ first_value = 1
 
 [bumpversion:file:plugins/postgres/setup.py]
 
-[bumpversion:file:plugins/redshift/setup.py]
-
-[bumpversion:file:plugins/snowflake/setup.py]
-
 [bumpversion:file:plugins/bigquery/setup.py]
 
 [bumpversion:file:plugins/postgres/dbt/adapters/postgres/__version__.py]
 
-[bumpversion:file:plugins/redshift/dbt/adapters/redshift/__version__.py]
-
-[bumpversion:file:plugins/snowflake/dbt/adapters/snowflake/__version__.py]
-
 [bumpversion:file:plugins/bigquery/dbt/adapters/bigquery/__version__.py]
 

.github/dependabot.yml

@@ -21,16 +21,6 @@ updates:
     schedule:
       interval: "daily"
     rebase-strategy: "disabled"
-  - package-ecosystem: "pip"
-    directory: "/plugins/redshift"
-    schedule:
-      interval: "daily"
-    rebase-strategy: "disabled"
-  - package-ecosystem: "pip"
-    directory: "/plugins/snowflake"
-    schedule:
-      interval: "daily"
-    rebase-strategy: "disabled"
 
   # docker dependencies
   - package-ecosystem: "docker"

.github/scripts/integration-test-matrix.js

@@ -1,7 +1,7 @@
 module.exports = ({ context }) => {
   const defaultPythonVersion = "3.8";
   const supportedPythonVersions = ["3.6", "3.7", "3.8", "3.9"];
-  const supportedAdapters = ["snowflake", "postgres", "bigquery", "redshift"];
+  const supportedAdapters = ["postgres", "bigquery"];
 
   // if PR, generate matrix based on files changed and PR labels
   if (context.eventName.includes("pull_request")) {

.github/workflows/integration.yml

@@ -91,16 +91,9 @@ jobs:
               - 'core/**'
               - 'plugins/postgres/**'
               - 'dev-requirements.txt'
-            snowflake:
-              - 'core/**'
-              - 'plugins/snowflake/**'
             bigquery:
               - 'core/**'
               - 'plugins/bigquery/**'
-            redshift:
-              - 'core/**'
-              - 'plugins/redshift/**'
-              - 'plugins/postgres/**'
 
       - name: Generate integration test matrix
         id: generate-matrix
@@ -191,33 +184,6 @@ jobs:
         if: matrix.adapter == 'postgres'
         run: tox
 
-      - name: Run tox (redshift)
-        if: matrix.adapter == 'redshift'
-        env:
-          REDSHIFT_TEST_DBNAME: ${{ secrets.REDSHIFT_TEST_DBNAME }}
-          REDSHIFT_TEST_PASS: ${{ secrets.REDSHIFT_TEST_PASS }}
-          REDSHIFT_TEST_USER: ${{ secrets.REDSHIFT_TEST_USER }}
-          REDSHIFT_TEST_PORT: ${{ secrets.REDSHIFT_TEST_PORT }}
-          REDSHIFT_TEST_HOST: ${{ secrets.REDSHIFT_TEST_HOST }}
-        run: tox
-
-      - name: Run tox (snowflake)
-        if: matrix.adapter == 'snowflake'
-        env:
-          SNOWFLAKE_TEST_ACCOUNT: ${{ secrets.SNOWFLAKE_TEST_ACCOUNT }}
-          SNOWFLAKE_TEST_PASSWORD: ${{ secrets.SNOWFLAKE_TEST_PASSWORD }}
-          SNOWFLAKE_TEST_USER: ${{ secrets.SNOWFLAKE_TEST_USER }}
-          SNOWFLAKE_TEST_WAREHOUSE: ${{ secrets.SNOWFLAKE_TEST_WAREHOUSE }}
-          SNOWFLAKE_TEST_OAUTH_REFRESH_TOKEN: ${{ secrets.SNOWFLAKE_TEST_OAUTH_REFRESH_TOKEN }}
-          SNOWFLAKE_TEST_OAUTH_CLIENT_ID: ${{ secrets.SNOWFLAKE_TEST_OAUTH_CLIENT_ID }}
-          SNOWFLAKE_TEST_OAUTH_CLIENT_SECRET: ${{ secrets.SNOWFLAKE_TEST_OAUTH_CLIENT_SECRET }}
-          SNOWFLAKE_TEST_ALT_DATABASE: ${{ secrets.SNOWFLAKE_TEST_ALT_DATABASE }}
-          SNOWFLAKE_TEST_ALT_WAREHOUSE: ${{ secrets.SNOWFLAKE_TEST_ALT_WAREHOUSE }}
-          SNOWFLAKE_TEST_DATABASE: ${{ secrets.SNOWFLAKE_TEST_DATABASE }}
-          SNOWFLAKE_TEST_QUOTED_DATABASE: ${{ secrets.SNOWFLAKE_TEST_QUOTED_DATABASE }}
-          SNOWFLAKE_TEST_ROLE: ${{ secrets.SNOWFLAKE_TEST_ROLE }}
-        run: tox
-
       - name: Run tox (bigquery)
         if: matrix.adapter == 'bigquery'
         env:

ARCHITECTURE.md

@@ -26,7 +26,7 @@ This is the docs website code. It comes from the dbt-docs repository, and is gen
 
 ## Adapters
 
-dbt uses an adapter-plugin pattern to extend support to different databases, warehouses, query engines, etc. The four core adapters that are in the main repository, contained within the [`plugins`](plugins) subdirectory, are: Postgres Redshift, Snowflake and BigQuery. Other warehouses use adapter plugins defined in separate repositories (e.g. [dbt-spark](https://github.com/dbt-labs/dbt-spark), [dbt-presto](https://github.com/dbt-labs/dbt-presto)).
+dbt uses an adapter-plugin pattern to extend support to different databases, warehouses, query engines, etc. For testing and development purposes, the dbt-postgres plugin lives alongside the dbt-core codebase, in the [`plugins`](plugins) subdirectory. Like other adapter plugins, it is a self-contained codebase and package that builds on top of dbt-core.
 
 Each adapter is a mix of python, Jinja2, and SQL. The adapter code also makes heavy use of Jinja2 to wrap modular chunks of SQL functionality, define default implementations, and allow plugins to override it. 
 

Makefile

@@ -44,22 +44,6 @@ integration-postgres: .env ## Runs postgres integration tests with py38.
 integration-postgres-fail-fast: .env ## Runs postgres integration tests with py38 in "fail fast" mode.
 	$(DOCKER_CMD) tox -e py38-postgres -- -x -nauto
 
-.PHONY: integration-redshift
-integration-redshift: .env ## Runs redshift integration tests with py38.
-	$(DOCKER_CMD) tox -e py38-redshift -- -nauto
-
-.PHONY: integration-redshift-fail-fast
-integration-redshift-fail-fast: .env ## Runs redshift integration tests with py38 in "fail fast" mode.
-	$(DOCKER_CMD) tox -e py38-redshift -- -x -nauto
-
-.PHONY: integration-snowflake
-integration-snowflake: .env ## Runs snowflake integration tests with py38.
-	$(DOCKER_CMD) tox -e py38-snowflake -- -nauto
-
-.PHONY: integration-snowflake-fail-fast
-integration-snowflake-fail-fast: .env ## Runs snowflake integration tests with py38 in "fail fast" mode.
-	$(DOCKER_CMD) tox -e py38-snowflake -- -x -nauto
-
 .PHONY: integration-bigquery
 integration-bigquery: .env ## Runs bigquery integration tests with py38.
 	$(DOCKER_CMD) tox -e py38-bigquery -- -nauto

editable-requirements.txt

@@ -1,5 +1,3 @@
 -e ./core
 -e ./plugins/postgres
--e ./plugins/redshift
--e ./plugins/snowflake
 -e ./plugins/bigquery

plugins/bigquery/README.md

@@ -1,32 +0,0 @@
-<p align="center">
-  <img src="https://raw.githubusercontent.com/dbt-labs/dbt/6c6649f9129d5d108aa3b0526f634cd8f3a9d1ed/etc/dbt-logo-full.svg" alt="dbt logo" width="500"/>
-</p>
-
-**[dbt](https://www.getdbt.com/)** (data build tool) enables data analysts and engineers to transform their data using the same practices that software engineers use to build applications.
-
-dbt is the T in ELT. Organize, cleanse, denormalize, filter, rename, and pre-aggregate the raw data in your warehouse so that it's ready for analysis.
-
-## dbt-bigquery
-
-The `dbt-bigquery` package contains all of the code required to make dbt operate on a BigQuery database. For
-more information on using dbt with BigQuery, consult [the docs](https://docs.getdbt.com/docs/profile-bigquery).
-
-
-## Find out more
-
--   Check out the [Introduction to dbt](https://docs.getdbt.com/docs/introduction/).
--   Read the [dbt Viewpoint](https://docs.getdbt.com/docs/about/viewpoint/).
-
-## Join thousands of analysts in the dbt community
-
--   Join the [chat](http://community.getdbt.com/) on Slack.
--   Find community posts on [dbt Discourse](https://discourse.getdbt.com).
-
-## Reporting bugs and contributing code
-
--   Want to report a bug or request a feature? Let us know on [Slack](http://community.getdbt.com/), or open [an issue](https://github.com/dbt-labs/dbt/issues/new).
--   Want to help us build dbt? Check out the [Contributing Getting Started Guide](https://github.com/dbt-labs/dbt/blob/HEAD/CONTRIBUTING.md)
-
-## Code of Conduct
-
-Everyone interacting in the dbt project's codebases, issue trackers, chat rooms, and mailing lists is expected to follow the [dbt Code of Conduct](https://community.getdbt.com/code-of-conduct).

plugins/bigquery/dbt/adapters/bigquery/__init__.py

@@ -1,13 +0,0 @@
-from dbt.adapters.bigquery.connections import BigQueryConnectionManager  # noqa
-from dbt.adapters.bigquery.connections import BigQueryCredentials
-from dbt.adapters.bigquery.relation import BigQueryRelation  # noqa
-from dbt.adapters.bigquery.column import BigQueryColumn  # noqa
-from dbt.adapters.bigquery.impl import BigQueryAdapter, GrantTarget  # noqa
-
-from dbt.adapters.base import AdapterPlugin
-from dbt.include import bigquery
-
-Plugin = AdapterPlugin(
-    adapter=BigQueryAdapter,
-    credentials=BigQueryCredentials,
-    include_path=bigquery.PACKAGE_PATH)

plugins/bigquery/dbt/adapters/bigquery/__version__.py

@@ -1 +0,0 @@
-version = '0.21.0rc1'

plugins/bigquery/dbt/adapters/bigquery/column.py

@@ -1,127 +0,0 @@
-from dataclasses import dataclass
-from typing import Optional, List, TypeVar, Iterable, Type
-
-from dbt.adapters.base.column import Column
-
-from google.cloud.bigquery import SchemaField
-
-Self = TypeVar('Self', bound='BigQueryColumn')
-
-
-@dataclass(init=False)
-class BigQueryColumn(Column):
-    TYPE_LABELS = {
-        'STRING': 'STRING',
-        'TIMESTAMP': 'TIMESTAMP',
-        'FLOAT': 'FLOAT64',
-        'INTEGER': 'INT64',
-        'RECORD': 'RECORD',
-    }
-    fields: List[Self]
-    mode: str
-
-    def __init__(
-        self,
-        column: str,
-        dtype: str,
-        fields: Optional[Iterable[SchemaField]] = None,
-        mode: str = 'NULLABLE',
-    ) -> None:
-        super().__init__(column, dtype)
-
-        if fields is None:
-            fields = []
-
-        self.fields = self.wrap_subfields(fields)
-        self.mode = mode
-
-    @classmethod
-    def wrap_subfields(
-        cls: Type[Self], fields: Iterable[SchemaField]
-    ) -> List[Self]:
-        return [cls.create_from_field(field) for field in fields]
-
-    @classmethod
-    def create_from_field(cls: Type[Self], field: SchemaField) -> Self:
-        return cls(
-            field.name,
-            cls.translate_type(field.field_type),
-            field.fields,
-            field.mode,
-        )
-
-    @classmethod
-    def _flatten_recursive(
-        cls: Type[Self], col: Self, prefix: Optional[str] = None
-    ) -> List[Self]:
-        if prefix is None:
-            prefix = []
-
-        if len(col.fields) == 0:
-            prefixed_name = ".".join(prefix + [col.column])
-            new_col = cls(prefixed_name, col.dtype, col.fields, col.mode)
-            return [new_col]
-
-        new_fields = []
-        for field in col.fields:
-            new_prefix = prefix + [col.column]
-            new_fields.extend(cls._flatten_recursive(field, new_prefix))
-
-        return new_fields
-
-    def flatten(self):
-        return self._flatten_recursive(self)
-
-    @property
-    def quoted(self):
-        return '`{}`'.format(self.column)
-
-    def literal(self, value):
-        return "cast({} as {})".format(value, self.dtype)
-
-    @property
-    def data_type(self) -> str:
-        if self.dtype.upper() == 'RECORD':
-            subcols = [
-                "{} {}".format(col.name, col.data_type) for col in self.fields
-            ]
-            field_type = 'STRUCT<{}>'.format(", ".join(subcols))
-
-        else:
-            field_type = self.dtype
-
-        if self.mode.upper() == 'REPEATED':
-            return 'ARRAY<{}>'.format(field_type)
-
-        else:
-            return field_type
-
-    def is_string(self) -> bool:
-        return self.dtype.lower() == 'string'
-
-    def is_integer(self) -> bool:
-        return self.dtype.lower() == 'int64'
-
-    def is_numeric(self) -> bool:
-        return self.dtype.lower() == 'numeric'
-
-    def is_float(self):
-        return self.dtype.lower() == 'float64'
-
-    def can_expand_to(self: Self, other_column: Self) -> bool:
-        """returns True if both columns are strings"""
-        return self.is_string() and other_column.is_string()
-
-    def __repr__(self) -> str:
-        return "<BigQueryColumn {} ({}, {})>".format(self.name, self.data_type,
-                                                     self.mode)
-
-    def column_to_bq_schema(self) -> SchemaField:
-        """Convert a column to a bigquery schema object.
-        """
-        kwargs = {}
-        if len(self.fields) > 0:
-            fields = [field.column_to_bq_schema() for field in self.fields]
-            kwargs = {"fields": fields}
-
-        return SchemaField(self.name, self.dtype, self.mode, **kwargs)

plugins/bigquery/dbt/adapters/bigquery/connections.py

@@ -1,638 +0,0 @@
-import json
-import re
-from contextlib import contextmanager
-from dataclasses import dataclass
-from functools import lru_cache
-import agate
-from requests.exceptions import ConnectionError
-from typing import Optional, Any, Dict, Tuple
-
-import google.auth
-import google.auth.exceptions
-import google.cloud.bigquery
-import google.cloud.exceptions
-from google.api_core import retry, client_info
-from google.auth import impersonated_credentials
-from google.oauth2 import (
-    credentials as GoogleCredentials,
-    service_account as GoogleServiceAccountCredentials
-)
-
-from dbt.utils import format_bytes, format_rows_number
-from dbt.clients import agate_helper, gcloud
-from dbt.tracking import active_user
-from dbt.contracts.connection import ConnectionState, AdapterResponse
-from dbt.exceptions import (
-    FailedToConnectException, RuntimeException, DatabaseException
-)
-from dbt.adapters.base import BaseConnectionManager, Credentials
-from dbt.logger import GLOBAL_LOGGER as logger
-from dbt.version import __version__ as dbt_version
-
-from dbt.dataclass_schema import StrEnum
-
-
-BQ_QUERY_JOB_SPLIT = '-----Query Job SQL Follows-----'
-
-WRITE_TRUNCATE = google.cloud.bigquery.job.WriteDisposition.WRITE_TRUNCATE
-
-REOPENABLE_ERRORS = (
-    ConnectionResetError,
-    ConnectionError,
-)
-
-RETRYABLE_ERRORS = (
-    google.cloud.exceptions.ServerError,
-    google.cloud.exceptions.BadRequest,
-    ConnectionResetError,
-    ConnectionError,
-)
-
-
-@lru_cache()
-def get_bigquery_defaults(scopes=None) -> Tuple[Any, Optional[str]]:
-    """
-    Returns (credentials, project_id)
-
-    project_id is returned available from the environment; otherwise None
-    """
-    # Cached, because the underlying implementation shells out, taking ~1s
-    return google.auth.default(scopes=scopes)
-
-
-class Priority(StrEnum):
-    Interactive = 'interactive'
-    Batch = 'batch'
-
-
-class BigQueryConnectionMethod(StrEnum):
-    OAUTH = 'oauth'
-    SERVICE_ACCOUNT = 'service-account'
-    SERVICE_ACCOUNT_JSON = 'service-account-json'
-    OAUTH_SECRETS = 'oauth-secrets'
-
-
-@dataclass
-class BigQueryAdapterResponse(AdapterResponse):
-    bytes_processed: Optional[int] = None
-
-
-@dataclass
-class BigQueryCredentials(Credentials):
-    method: BigQueryConnectionMethod
-    # BigQuery allows an empty database / project, where it defers to the
-    # environment for the project
-    database: Optional[str]
-    execution_project: Optional[str] = None
-    timeout_seconds: Optional[int] = 300
-    location: Optional[str] = None
-    priority: Optional[Priority] = None
-    retries: Optional[int] = 1
-    maximum_bytes_billed: Optional[int] = None
-    impersonate_service_account: Optional[str] = None
-
-    # Keyfile json creds
-    keyfile: Optional[str] = None
-    keyfile_json: Optional[Dict[str, Any]] = None
-
-    # oauth-secrets
-    token: Optional[str] = None
-    refresh_token: Optional[str] = None
-    client_id: Optional[str] = None
-    client_secret: Optional[str] = None
-    token_uri: Optional[str] = None
-
-    _ALIASES = {
-        'project': 'database',
-        'dataset': 'schema',
-        'target_project': 'target_database',
-        'target_dataset': 'target_schema',
-    }
-
-    @property
-    def type(self):
-        return 'bigquery'
-
-    @property
-    def unique_field(self):
-        return self.database
-
-    def _connection_keys(self):
-        return ('method', 'database', 'schema', 'location', 'priority',
-                'timeout_seconds', 'maximum_bytes_billed')
-
-    @classmethod
-    def __pre_deserialize__(cls, d: Dict[Any, Any]) -> Dict[Any, Any]:
-        # We need to inject the correct value of the database (aka project) at
-        # this stage, ref
-        # https://github.com/dbt-labs/dbt/pull/2908#discussion_r532927436.
-
-        # `database` is an alias of `project` in BigQuery
-        if 'database' not in d:
-            _, database = get_bigquery_defaults()
-            d['database'] = database
-        # `execution_project` default to dataset/project
-        if 'execution_project' not in d:
-            d['execution_project'] = d['database']
-        return d
-
-
-class BigQueryConnectionManager(BaseConnectionManager):
-    TYPE = 'bigquery'
-
-    SCOPE = ('https://www.googleapis.com/auth/bigquery',
-             'https://www.googleapis.com/auth/cloud-platform',
-             'https://www.googleapis.com/auth/drive')
-
-    QUERY_TIMEOUT = 300
-    RETRIES = 1
-    DEFAULT_INITIAL_DELAY = 1.0  # Seconds
-    DEFAULT_MAXIMUM_DELAY = 1.0  # Seconds
-
-    @classmethod
-    def handle_error(cls, error, message):
-        error_msg = "\n".join([item['message'] for item in error.errors])
-        raise DatabaseException(error_msg)
-
-    def clear_transaction(self):
-        pass
-
-    @contextmanager
-    def exception_handler(self, sql):
-        try:
-            yield
-
-        except google.cloud.exceptions.BadRequest as e:
-            message = "Bad request while running query"
-            self.handle_error(e, message)
-
-        except google.cloud.exceptions.Forbidden as e:
-            message = "Access denied while running query"
-            self.handle_error(e, message)
-
-        except google.auth.exceptions.RefreshError as e:
-            message = "Unable to generate access token, if you're using " \
-                      "impersonate_service_account, make sure your " \
-                      'initial account has the "roles/' \
-                      'iam.serviceAccountTokenCreator" role on the ' \
-                      'account you are trying to impersonate.\n\n' \
-                      f'{str(e)}'
-            raise RuntimeException(message)
-
-        except Exception as e:
-            logger.debug("Unhandled error while running:\n{}".format(sql))
-            logger.debug(e)
-            if isinstance(e, RuntimeException):
-                # during a sql query, an internal to dbt exception was raised.
-                # this sounds a lot like a signal handler and probably has
-                # useful information, so raise it without modification.
-                raise
-            exc_message = str(e)
-            # the google bigquery library likes to add the query log, which we
-            # don't want to log. Hopefully they never change this!
-            if BQ_QUERY_JOB_SPLIT in exc_message:
-                exc_message = exc_message.split(BQ_QUERY_JOB_SPLIT)[0].strip()
-            raise RuntimeException(exc_message)
-
-    def cancel_open(self) -> None:
-        pass
-
-    @classmethod
-    def close(cls, connection):
-        connection.state = ConnectionState.CLOSED
-
-        return connection
-
-    def begin(self):
-        pass
-
-    def commit(self):
-        pass
-
-    @classmethod
-    def get_bigquery_credentials(cls, profile_credentials):
-        method = profile_credentials.method
-        creds = GoogleServiceAccountCredentials.Credentials
-
-        if method == BigQueryConnectionMethod.OAUTH:
-            credentials, _ = get_bigquery_defaults(scopes=cls.SCOPE)
-            return credentials
-
-        elif method == BigQueryConnectionMethod.SERVICE_ACCOUNT:
-            keyfile = profile_credentials.keyfile
-            return creds.from_service_account_file(keyfile, scopes=cls.SCOPE)
-
-        elif method == BigQueryConnectionMethod.SERVICE_ACCOUNT_JSON:
-            details = profile_credentials.keyfile_json
-            return creds.from_service_account_info(details, scopes=cls.SCOPE)
-
-        elif method == BigQueryConnectionMethod.OAUTH_SECRETS:
-            return GoogleCredentials.Credentials(
-                token=profile_credentials.token,
-                refresh_token=profile_credentials.refresh_token,
-                client_id=profile_credentials.client_id,
-                client_secret=profile_credentials.client_secret,
-                token_uri=profile_credentials.token_uri,
-                scopes=cls.SCOPE
-            )
-
-        error = ('Invalid `method` in profile: "{}"'.format(method))
-        raise FailedToConnectException(error)
-
-    @classmethod
-    def get_impersonated_bigquery_credentials(cls, profile_credentials):
-        source_credentials = cls.get_bigquery_credentials(profile_credentials)
-        return impersonated_credentials.Credentials(
-            source_credentials=source_credentials,
-            target_principal=profile_credentials.impersonate_service_account,
-            target_scopes=list(cls.SCOPE),
-            lifetime=profile_credentials.timeout_seconds,
-        )
-
-    @classmethod
-    def get_bigquery_client(cls, profile_credentials):
-        if profile_credentials.impersonate_service_account:
-            creds =\
-                cls.get_impersonated_bigquery_credentials(profile_credentials)
-        else:
-            creds = cls.get_bigquery_credentials(profile_credentials)
-        execution_project = profile_credentials.execution_project
-        location = getattr(profile_credentials, 'location', None)
-
-        info = client_info.ClientInfo(user_agent=f'dbt-{dbt_version}')
-        return google.cloud.bigquery.Client(
-            execution_project,
-            creds,
-            location=location,
-            client_info=info,
-        )
-
-    @classmethod
-    def open(cls, connection):
-        if connection.state == 'open':
-            logger.debug('Connection is already open, skipping open.')
-            return connection
-
-        try:
-            handle = cls.get_bigquery_client(connection.credentials)
-
-        except google.auth.exceptions.DefaultCredentialsError:
-            logger.info("Please log into GCP to continue")
-            gcloud.setup_default_credentials()
-
-            handle = cls.get_bigquery_client(connection.credentials)
-
-        except Exception as e:
-            logger.debug("Got an error when attempting to create a bigquery "
-                         "client: '{}'".format(e))
-
-            connection.handle = None
-            connection.state = 'fail'
-
-            raise FailedToConnectException(str(e))
-
-        connection.handle = handle
-        connection.state = 'open'
-        return connection
-
-    @classmethod
-    def get_timeout(cls, conn):
-        credentials = conn.credentials
-        return credentials.timeout_seconds
-
-    @classmethod
-    def get_retries(cls, conn) -> int:
-        credentials = conn.credentials
-        if credentials.retries is not None:
-            return credentials.retries
-        else:
-            return 1
-
-    @classmethod
-    def get_table_from_response(cls, resp):
-        column_names = [field.name for field in resp.schema]
-        return agate_helper.table_from_data_flat(resp, column_names)
-
-    def raw_execute(self, sql, fetch=False, *, use_legacy_sql=False):
-        conn = self.get_thread_connection()
-        client = conn.handle
-
-        logger.debug('On {}: {}', conn.name, sql)
-
-        if self.profile.query_comment and self.profile.query_comment.job_label:
-            query_comment = self.query_header.comment.query_comment
-            labels = self._labels_from_query_comment(query_comment)
-        else:
-            labels = {}
-
-        if active_user:
-            labels['dbt_invocation_id'] = active_user.invocation_id
-
-        job_params = {'use_legacy_sql': use_legacy_sql, 'labels': labels}
-
-        priority = conn.credentials.priority
-        if priority == Priority.Batch:
-            job_params['priority'] = google.cloud.bigquery.QueryPriority.BATCH
-        else:
-            job_params[
-                'priority'] = google.cloud.bigquery.QueryPriority.INTERACTIVE
-
-        maximum_bytes_billed = conn.credentials.maximum_bytes_billed
-        if maximum_bytes_billed is not None and maximum_bytes_billed != 0:
-            job_params['maximum_bytes_billed'] = maximum_bytes_billed
-
-        def fn():
-            return self._query_and_results(client, sql, conn, job_params)
-
-        query_job, iterator = self._retry_and_handle(msg=sql, conn=conn, fn=fn)
-
-        return query_job, iterator
-
-    def execute(
-        self, sql, auto_begin=False, fetch=None
-    ) -> Tuple[BigQueryAdapterResponse, agate.Table]:
-        sql = self._add_query_comment(sql)
-        # auto_begin is ignored on bigquery, and only included for consistency
-        query_job, iterator = self.raw_execute(sql, fetch=fetch)
-
-        if fetch:
-            table = self.get_table_from_response(iterator)
-        else:
-            table = agate_helper.empty_table()
-
-        message = 'OK'
-        code = None
-        num_rows = None
-        bytes_processed = None
-
-        if query_job.statement_type == 'CREATE_VIEW':
-            code = 'CREATE VIEW'
-
-        elif query_job.statement_type == 'CREATE_TABLE_AS_SELECT':
-            conn = self.get_thread_connection()
-            client = conn.handle
-            query_table = client.get_table(query_job.destination)
-            code = 'CREATE TABLE'
-            num_rows = query_table.num_rows
-            bytes_processed = query_job.total_bytes_processed
-            message = '{} ({} rows, {} processed)'.format(
-                code,
-                format_rows_number(num_rows),
-                format_bytes(bytes_processed)
-            )
-
-        elif query_job.statement_type == 'SCRIPT':
-            code = 'SCRIPT'
-            bytes_processed = query_job.total_bytes_processed
-            message = f'{code} ({format_bytes(bytes_processed)} processed)'
-
-        elif query_job.statement_type in ['INSERT', 'DELETE', 'MERGE']:
-            code = query_job.statement_type
-            num_rows = query_job.num_dml_affected_rows
-            bytes_processed = query_job.total_bytes_processed
-            message = '{} ({} rows, {} processed)'.format(
-                code,
-                format_rows_number(num_rows),
-                format_bytes(bytes_processed),
-            )
-
-        response = BigQueryAdapterResponse(
-            _message=message,
-            rows_affected=num_rows,
-            code=code,
-            bytes_processed=bytes_processed
-        )
-
-        return response, table
-
-    def get_partitions_metadata(self, table):
-        def standard_to_legacy(table):
-            return table.project + ':' + table.dataset + '.' + table.identifier
-
-        legacy_sql = 'SELECT * FROM ['\
-            + standard_to_legacy(table) + '$__PARTITIONS_SUMMARY__]'
-
-        sql = self._add_query_comment(legacy_sql)
-        # auto_begin is ignored on bigquery, and only included for consistency
-        _, iterator =\
-            self.raw_execute(sql, fetch='fetch_result', use_legacy_sql=True)
-        return self.get_table_from_response(iterator)
-
-    def create_bigquery_table(self, database, schema, table_name, callback,
-                              sql):
-        """Create a bigquery table. The caller must supply a callback
-        that takes one argument, a `google.cloud.bigquery.Table`, and mutates
-        it.
-        """
-        conn = self.get_thread_connection()
-        client = conn.handle
-
-        view_ref = self.table_ref(database, schema, table_name, conn)
-        view = google.cloud.bigquery.Table(view_ref)
-        callback(view)
-
-        def fn():
-            return client.create_table(view)
-        self._retry_and_handle(msg=sql, conn=conn, fn=fn)
-
-    def create_view(self, database, schema, table_name, sql):
-        def callback(table):
-            table.view_query = sql
-            table.view_use_legacy_sql = False
-
-        self.create_bigquery_table(database, schema, table_name, callback, sql)
-
-    def create_table(self, database, schema, table_name, sql):
-        conn = self.get_thread_connection()
-        client = conn.handle
-
-        table_ref = self.table_ref(database, schema, table_name, conn)
-        job_params = {'destination': table_ref,
-                      'write_disposition': WRITE_TRUNCATE}
-
-        timeout = self.get_timeout(conn)
-
-        def fn():
-            return self._query_and_results(client, sql, conn, job_params,
-                                           timeout=timeout)
-        self._retry_and_handle(msg=sql, conn=conn, fn=fn)
-
-    def create_date_partitioned_table(self, database, schema, table_name):
-        def callback(table):
-            table.partitioning_type = 'DAY'
-
-        self.create_bigquery_table(database, schema, table_name, callback,
-                                   'CREATE DAY PARTITIONED TABLE')
-
-    def copy_bq_table(self, source, destination, write_disposition):
-        conn = self.get_thread_connection()
-        client = conn.handle
-
-# -------------------------------------------------------------------------------
-#  BigQuery allows to use copy API using two different formats:
-#  1. client.copy_table(source_table_id, destination_table_id)
-#     where source_table_id = "your-project.source_dataset.source_table"
-#  2. client.copy_table(source_table_ids, destination_table_id)
-#     where source_table_ids = ["your-project.your_dataset.your_table_name", ...]
-#  Let's use uniform function call and always pass list there
-# -------------------------------------------------------------------------------
-        if type(source) is not list:
-            source = [source]
-
-        source_ref_array = [self.table_ref(
-            src_table.database, src_table.schema, src_table.table, conn)
-            for src_table in source]
-        destination_ref = self.table_ref(
-            destination.database, destination.schema, destination.table, conn)
-
-        logger.debug(
-            'Copying table(s) "{}" to "{}" with disposition: "{}"',
-            ', '.join(source_ref.path for source_ref in source_ref_array),
-            destination_ref.path, write_disposition)
-
-        def copy_and_results():
-            job_config = google.cloud.bigquery.CopyJobConfig(
-                write_disposition=write_disposition)
-            copy_job = client.copy_table(
-                source_ref_array, destination_ref, job_config=job_config)
-            iterator = copy_job.result(timeout=self.get_timeout(conn))
-            return copy_job, iterator
-
-        self._retry_and_handle(
-            msg='copy table "{}" to "{}"'.format(
-                ', '.join(source_ref.path for source_ref in source_ref_array),
-                destination_ref.path),
-            conn=conn, fn=copy_and_results)
-
-    @staticmethod
-    def dataset(database, schema, conn):
-        dataset_ref = conn.handle.dataset(schema, database)
-        return google.cloud.bigquery.Dataset(dataset_ref)
-
-    @staticmethod
-    def dataset_from_id(dataset_id):
-        return google.cloud.bigquery.Dataset.from_string(dataset_id)
-
-    def table_ref(self, database, schema, table_name, conn):
-        dataset = self.dataset(database, schema, conn)
-        return dataset.table(table_name)
-
-    def get_bq_table(self, database, schema, identifier):
-        """Get a bigquery table for a schema/model."""
-        conn = self.get_thread_connection()
-        table_ref = self.table_ref(database, schema, identifier, conn)
-        return conn.handle.get_table(table_ref)
-
-    def drop_dataset(self, database, schema):
-        conn = self.get_thread_connection()
-        dataset = self.dataset(database, schema, conn)
-        client = conn.handle
-
-        def fn():
-            return client.delete_dataset(
-                dataset, delete_contents=True, not_found_ok=True)
-
-        self._retry_and_handle(
-            msg='drop dataset', conn=conn, fn=fn)
-
-    def create_dataset(self, database, schema):
-        conn = self.get_thread_connection()
-        client = conn.handle
-        dataset = self.dataset(database, schema, conn)
-
-        def fn():
-            return client.create_dataset(dataset, exists_ok=True)
-        self._retry_and_handle(msg='create dataset', conn=conn, fn=fn)
-
-    def _query_and_results(self, client, sql, conn, job_params, timeout=None):
-        """Query the client and wait for results."""
-        # Cannot reuse job_config if destination is set and ddl is used
-        job_config = google.cloud.bigquery.QueryJobConfig(**job_params)
-        query_job = client.query(sql, job_config=job_config)
-        iterator = query_job.result(timeout=timeout)
-
-        return query_job, iterator
-
-    def _retry_and_handle(self, msg, conn, fn):
-        """retry a function call within the context of exception_handler."""
-        def reopen_conn_on_error(error):
-            if isinstance(error, REOPENABLE_ERRORS):
-                logger.warning('Reopening connection after {!r}', error)
-                self.close(conn)
-                self.open(conn)
-                return
-
-        with self.exception_handler(msg):
-            return retry.retry_target(
-                target=fn,
-                predicate=_ErrorCounter(self.get_retries(conn)).count_error,
-                sleep_generator=self._retry_generator(),
-                deadline=None,
-                on_error=reopen_conn_on_error)
-
-    def _retry_generator(self):
-        """Generates retry intervals that exponentially back off."""
-        return retry.exponential_sleep_generator(
-            initial=self.DEFAULT_INITIAL_DELAY,
-            maximum=self.DEFAULT_MAXIMUM_DELAY)
-
-    def _labels_from_query_comment(self, comment: str) -> Dict:
-        try:
-            comment_labels = json.loads(comment)
-        except (TypeError, ValueError):
-            return {'query_comment': _sanitize_label(comment)}
-        return {
-            _sanitize_label(key): _sanitize_label(str(value))
-            for key, value in comment_labels.items()
-        }
-
-
-class _ErrorCounter(object):
-    """Counts errors seen up to a threshold then raises the next error."""
-
-    def __init__(self, retries):
-        self.retries = retries
-        self.error_count = 0
-
-    def count_error(self, error):
-        if self.retries == 0:
-            return False  # Don't log
-        self.error_count += 1
-        if _is_retryable(error) and self.error_count <= self.retries:
-            logger.debug(
-                'Retry attempt {} of {} after error: {}',
-                self.error_count, self.retries, repr(error))
-            return True
-        else:
-            return False
-
-
-def _is_retryable(error):
-    """Return true for errors that are unlikely to occur again if retried."""
-    if isinstance(error, RETRYABLE_ERRORS):
-        return True
-    elif isinstance(error, google.api_core.exceptions.Forbidden) and any(
-            e['reason'] == 'rateLimitExceeded' for e in error.errors):
-        return True
-    return False
-
-
-_SANITIZE_LABEL_PATTERN = re.compile(r"[^a-z0-9_-]")
-
-_VALIDATE_LABEL_LENGTH_LIMIT = 63
-
-
-def _sanitize_label(value: str) -> str:
-    """Return a legal value for a BigQuery label."""
-    value = value.strip().lower()
-    value = _SANITIZE_LABEL_PATTERN.sub("_", value)
-    value_length = len(value)
-    if value_length > _VALIDATE_LABEL_LENGTH_LIMIT:
-        error_msg = (
-            f"Job label length {value_length} is greater than length limit: "
-            f"{_VALIDATE_LABEL_LENGTH_LIMIT}\n"
-            f"Current sanitized label: {value}"
-        )
-        raise RuntimeException(error_msg)
-    else:
-        return value

plugins/bigquery/dbt/adapters/bigquery/impl.py

@@ -1,870 +0,0 @@
-from dataclasses import dataclass
-from typing import Dict, List, Optional, Any, Set, Union
-from dbt.dataclass_schema import dbtClassMixin, ValidationError
-
-import dbt.deprecations
-import dbt.exceptions
-import dbt.clients.gcloud
-import dbt.clients.agate_helper
-
-from dbt import ui
-from dbt.adapters.base import (
-    BaseAdapter, available, RelationType, SchemaSearchMap, AdapterConfig
-)
-from dbt.adapters.bigquery.relation import BigQueryRelation
-from dbt.adapters.bigquery import BigQueryColumn
-from dbt.adapters.bigquery import BigQueryConnectionManager
-from dbt.contracts.graph.manifest import Manifest
-from dbt.logger import GLOBAL_LOGGER as logger, print_timestamped_line
-from dbt.utils import filter_null_values
-
-import google.auth
-import google.api_core
-import google.oauth2
-import google.cloud.exceptions
-import google.cloud.bigquery
-
-from google.cloud.bigquery import AccessEntry, SchemaField
-
-import time
-import agate
-import json
-
-# Write dispositions for bigquery.
-WRITE_APPEND = google.cloud.bigquery.job.WriteDisposition.WRITE_APPEND
-WRITE_TRUNCATE = google.cloud.bigquery.job.WriteDisposition.WRITE_TRUNCATE
-
-
-def sql_escape(string):
-    if not isinstance(string, str):
-        dbt.exceptions.raise_compiler_exception(
-            f'cannot escape a non-string: {string}'
-        )
-
-    return json.dumps(string)[1:-1]
-
-
-@dataclass
-class PartitionConfig(dbtClassMixin):
-    field: str
-    data_type: str = 'date'
-    granularity: str = 'day'
-    range: Optional[Dict[str, Any]] = None
-
-    def render(self, alias: Optional[str] = None):
-        column: str = self.field
-        if alias:
-            column = f'{alias}.{self.field}'
-
-        if self.data_type.lower() == 'int64' or (
-            self.data_type.lower() == 'date' and
-            self.granularity.lower() == 'day'
-        ):
-            return column
-        else:
-            return f'{self.data_type}_trunc({column}, {self.granularity})'
-
-    @classmethod
-    def parse(cls, raw_partition_by) -> Optional['PartitionConfig']:
-        if raw_partition_by is None:
-            return None
-        try:
-            cls.validate(raw_partition_by)
-            return cls.from_dict(raw_partition_by)
-        except ValidationError as exc:
-            msg = dbt.exceptions.validator_error_message(exc)
-            dbt.exceptions.raise_compiler_error(
-                f'Could not parse partition config: {msg}'
-            )
-        except TypeError:
-            dbt.exceptions.raise_compiler_error(
-                f'Invalid partition_by config:\n'
-                f'  Got: {raw_partition_by}\n'
-                f'  Expected a dictionary with "field" and "data_type" keys'
-            )
-
-
-@dataclass
-class GrantTarget(dbtClassMixin):
-    dataset: str
-    project: str
-
-    def render(self):
-        return f'{self.project}.{self.dataset}'
-
-
-def _stub_relation(*args, **kwargs):
-    return BigQueryRelation.create(
-        database='',
-        schema='',
-        identifier='',
-        quote_policy={},
-        type=BigQueryRelation.Table
-    )
-
-
-@dataclass
-class BigqueryConfig(AdapterConfig):
-    cluster_by: Optional[Union[List[str], str]] = None
-    partition_by: Optional[Dict[str, Any]] = None
-    kms_key_name: Optional[str] = None
-    labels: Optional[Dict[str, str]] = None
-    partitions: Optional[List[str]] = None
-    grant_access_to: Optional[List[Dict[str, str]]] = None
-    hours_to_expiration: Optional[int] = None
-    require_partition_filter: Optional[bool] = None
-    partition_expiration_days: Optional[int] = None
-    merge_update_columns: Optional[str] = None
-
-
-class BigQueryAdapter(BaseAdapter):
-
-    RELATION_TYPES = {
-        'TABLE': RelationType.Table,
-        'VIEW': RelationType.View,
-        'EXTERNAL': RelationType.External
-    }
-
-    Relation = BigQueryRelation
-    Column = BigQueryColumn
-    ConnectionManager = BigQueryConnectionManager
-
-    AdapterSpecificConfigs = BigqueryConfig
-
-    ###
-    # Implementations of abstract methods
-    ###
-
-    @classmethod
-    def date_function(cls) -> str:
-        return 'CURRENT_TIMESTAMP()'
-
-    @classmethod
-    def is_cancelable(cls) -> bool:
-        return False
-
-    def drop_relation(self, relation: BigQueryRelation) -> None:
-        is_cached = self._schema_is_cached(relation.database, relation.schema)
-        if is_cached:
-            self.cache_dropped(relation)
-
-        conn = self.connections.get_thread_connection()
-        client = conn.handle
-
-        dataset = self.connections.dataset(relation.database, relation.schema,
-                                           conn)
-        relation_object = dataset.table(relation.identifier)
-        client.delete_table(relation_object)
-
-    def truncate_relation(self, relation: BigQueryRelation) -> None:
-        raise dbt.exceptions.NotImplementedException(
-            '`truncate` is not implemented for this adapter!'
-        )
-
-    def rename_relation(
-        self, from_relation: BigQueryRelation, to_relation: BigQueryRelation
-    ) -> None:
-
-        conn = self.connections.get_thread_connection()
-        client = conn.handle
-
-        from_table_ref = self.connections.table_ref(from_relation.database,
-                                                    from_relation.schema,
-                                                    from_relation.identifier,
-                                                    conn)
-        from_table = client.get_table(from_table_ref)
-        if from_table.table_type == "VIEW" or \
-                from_relation.type == RelationType.View or \
-                to_relation.type == RelationType.View:
-            raise dbt.exceptions.RuntimeException(
-                'Renaming of views is not currently supported in BigQuery'
-            )
-
-        to_table_ref = self.connections.table_ref(to_relation.database,
-                                                  to_relation.schema,
-                                                  to_relation.identifier,
-                                                  conn)
-
-        self.cache_renamed(from_relation, to_relation)
-        client.copy_table(from_table_ref, to_table_ref)
-        client.delete_table(from_table_ref)
-
-    @available
-    def list_schemas(self, database: str) -> List[str]:
-        # the database string we get here is potentially quoted. Strip that off
-        # for the API call.
-        database = database.strip('`')
-        conn = self.connections.get_thread_connection()
-        client = conn.handle
-
-        def query_schemas():
-            # this is similar to how we have to deal with listing tables
-            all_datasets = client.list_datasets(project=database,
-                                                max_results=10000)
-            return [ds.dataset_id for ds in all_datasets]
-
-        return self.connections._retry_and_handle(
-            msg='list dataset', conn=conn, fn=query_schemas)
-
-    @available.parse(lambda *a, **k: False)
-    def check_schema_exists(self, database: str, schema: str) -> bool:
-        conn = self.connections.get_thread_connection()
-        client = conn.handle
-
-        bigquery_dataset = self.connections.dataset(
-            database, schema, conn
-        )
-        # try to do things with the dataset. If it doesn't exist it will 404.
-        # we have to do it this way to handle underscore-prefixed datasets,
-        # which appear in neither the information_schema.schemata view nor the
-        # list_datasets method.
-        try:
-            next(iter(client.list_tables(bigquery_dataset, max_results=1)))
-        except StopIteration:
-            pass
-        except google.api_core.exceptions.NotFound:
-            # the schema does not exist
-            return False
-        return True
-
-    def get_columns_in_relation(
-        self, relation: BigQueryRelation
-    ) -> List[BigQueryColumn]:
-        try:
-            table = self.connections.get_bq_table(
-                database=relation.database,
-                schema=relation.schema,
-                identifier=relation.identifier
-            )
-            return self._get_dbt_columns_from_bq_table(table)
-
-        except (ValueError, google.cloud.exceptions.NotFound) as e:
-            logger.debug("get_columns_in_relation error: {}".format(e))
-            return []
-
-    def expand_column_types(
-        self, goal: BigQueryRelation, current: BigQueryRelation
-    ) -> None:
-        # This is a no-op on BigQuery
-        pass
-
-    def expand_target_column_types(
-        self, from_relation: BigQueryRelation, to_relation: BigQueryRelation
-    ) -> None:
-        # This is a no-op on BigQuery
-        pass
-
-    @available.parse_list
-    def list_relations_without_caching(
-        self, schema_relation: BigQueryRelation
-    ) -> List[BigQueryRelation]:
-        connection = self.connections.get_thread_connection()
-        client = connection.handle
-
-        bigquery_dataset = self.connections.dataset(
-            schema_relation.database, schema_relation.schema, connection
-        )
-
-        all_tables = client.list_tables(
-            bigquery_dataset,
-            # BigQuery paginates tables by alphabetizing them, and using
-            # the name of the last table on a page as the key for the
-            # next page. If that key table gets dropped before we run
-            # list_relations, then this will 404. So, we avoid this
-            # situation by making the page size sufficiently large.
-            # see: https://github.com/dbt-labs/dbt/issues/726
-            # TODO: cache the list of relations up front, and then we
-            #       won't need to do this
-            max_results=100000)
-
-        # This will 404 if the dataset does not exist. This behavior mirrors
-        # the implementation of list_relations for other adapters
-        try:
-            return [self._bq_table_to_relation(table) for table in all_tables]
-        except google.api_core.exceptions.NotFound:
-            return []
-
-    def get_relation(
-        self, database: str, schema: str, identifier: str
-    ) -> BigQueryRelation:
-        if self._schema_is_cached(database, schema):
-            # if it's in the cache, use the parent's model of going through
-            # the relations cache and picking out the relation
-            return super().get_relation(
-                database=database,
-                schema=schema,
-                identifier=identifier
-            )
-
-        try:
-            table = self.connections.get_bq_table(database, schema, identifier)
-        except google.api_core.exceptions.NotFound:
-            table = None
-        return self._bq_table_to_relation(table)
-
-    def create_schema(self, relation: BigQueryRelation) -> None:
-        database = relation.database
-        schema = relation.schema
-        logger.debug('Creating schema "{}.{}".', database, schema)
-        self.connections.create_dataset(database, schema)
-
-    def drop_schema(self, relation: BigQueryRelation) -> None:
-        database = relation.database
-        schema = relation.schema
-        logger.debug('Dropping schema "{}.{}".', database, schema)
-        self.connections.drop_dataset(database, schema)
-        self.cache.drop_schema(database, schema)
-
-    @classmethod
-    def quote(cls, identifier: str) -> str:
-        return '`{}`'.format(identifier)
-
-    @classmethod
-    def convert_text_type(cls, agate_table: agate.Table, col_idx: int) -> str:
-        return "string"
-
-    @classmethod
-    def convert_number_type(
-        cls, agate_table: agate.Table, col_idx: int
-    ) -> str:
-        decimals = agate_table.aggregate(agate.MaxPrecision(col_idx))
-        return "float64" if decimals else "int64"
-
-    @classmethod
-    def convert_boolean_type(
-        cls, agate_table: agate.Table, col_idx: int
-    ) -> str:
-        return "bool"
-
-    @classmethod
-    def convert_datetime_type(
-        cls, agate_table: agate.Table, col_idx: int
-    ) -> str:
-        return "datetime"
-
-    @classmethod
-    def convert_date_type(cls, agate_table: agate.Table, col_idx: int) -> str:
-        return "date"
-
-    @classmethod
-    def convert_time_type(cls, agate_table: agate.Table, col_idx: int) -> str:
-        return "time"
-
-    ###
-    # Implementation details
-    ###
-    def _make_match_kwargs(
-        self, database: str, schema: str, identifier: str
-    ) -> Dict[str, str]:
-        return filter_null_values({
-            'database': database,
-            'identifier': identifier,
-            'schema': schema,
-        })
-
-    def _get_dbt_columns_from_bq_table(self, table) -> List[BigQueryColumn]:
-        "Translates BQ SchemaField dicts into dbt BigQueryColumn objects"
-
-        columns = []
-        for col in table.schema:
-            # BigQuery returns type labels that are not valid type specifiers
-            dtype = self.Column.translate_type(col.field_type)
-            column = self.Column(
-                col.name, dtype, col.fields, col.mode)
-            columns.append(column)
-
-        return columns
-
-    def _agate_to_schema(
-        self, agate_table: agate.Table, column_override: Dict[str, str]
-    ) -> List[SchemaField]:
-        """Convert agate.Table with column names to a list of bigquery schemas.
-        """
-        bq_schema = []
-        for idx, col_name in enumerate(agate_table.column_names):
-            inferred_type = self.convert_agate_type(agate_table, idx)
-            type_ = column_override.get(col_name, inferred_type)
-            bq_schema.append(SchemaField(col_name, type_))
-        return bq_schema
-
-    def _materialize_as_view(self, model: Dict[str, Any]) -> str:
-        model_database = model.get('database')
-        model_schema = model.get('schema')
-        model_alias = model.get('alias')
-        model_sql = model.get('compiled_sql')
-
-        logger.debug("Model SQL ({}):\n{}".format(model_alias, model_sql))
-        self.connections.create_view(
-            database=model_database,
-            schema=model_schema,
-            table_name=model_alias,
-            sql=model_sql
-        )
-        return "CREATE VIEW"
-
-    def _materialize_as_table(
-        self,
-        model: Dict[str, Any],
-        model_sql: str,
-        decorator: Optional[str] = None,
-    ) -> str:
-        model_database = model.get('database')
-        model_schema = model.get('schema')
-        model_alias = model.get('alias')
-
-        if decorator is None:
-            table_name = model_alias
-        else:
-            table_name = "{}${}".format(model_alias, decorator)
-
-        logger.debug("Model SQL ({}):\n{}".format(table_name, model_sql))
-        self.connections.create_table(
-            database=model_database,
-            schema=model_schema,
-            table_name=table_name,
-            sql=model_sql
-        )
-
-        return "CREATE TABLE"
-
-    @available.parse(lambda *a, **k: '')
-    def copy_table(self, source, destination, materialization):
-        if materialization == 'incremental':
-            write_disposition = WRITE_APPEND
-        elif materialization == 'table':
-            write_disposition = WRITE_TRUNCATE
-        else:
-            dbt.exceptions.raise_compiler_error(
-                'Copy table materialization must be "copy" or "table", but '
-                f"config.get('copy_materialization', 'table') was "
-                f'{materialization}')
-
-        self.connections.copy_bq_table(
-            source, destination, write_disposition)
-
-        return "COPY TABLE with materialization: {}".format(materialization)
-
-    @classmethod
-    def poll_until_job_completes(cls, job, timeout):
-        retry_count = timeout
-
-        while retry_count > 0 and job.state != 'DONE':
-            retry_count -= 1
-            time.sleep(1)
-            job.reload()
-
-        if job.state != 'DONE':
-            raise dbt.exceptions.RuntimeException("BigQuery Timeout Exceeded")
-
-        elif job.error_result:
-            message = '\n'.join(
-                error['message'].strip() for error in job.errors
-            )
-            raise dbt.exceptions.RuntimeException(message)
-
-    def _bq_table_to_relation(self, bq_table):
-        if bq_table is None:
-            return None
-
-        return self.Relation.create(
-            database=bq_table.project,
-            schema=bq_table.dataset_id,
-            identifier=bq_table.table_id,
-            quote_policy={
-                'schema': True,
-                'identifier': True
-            },
-            type=self.RELATION_TYPES.get(
-                bq_table.table_type, RelationType.External
-            ),
-        )
-
-    @classmethod
-    def warning_on_hooks(hook_type):
-        msg = "{} is not supported in bigquery and will be ignored"
-        print_timestamped_line(
-            msg.format(hook_type), ui.COLOR_FG_YELLOW
-        )
-
-    @available
-    def add_query(self, sql, auto_begin=True, bindings=None,
-                  abridge_sql_log=False):
-        if self.nice_connection_name() in ['on-run-start', 'on-run-end']:
-            self.warning_on_hooks(self.nice_connection_name())
-        else:
-            raise dbt.exceptions.NotImplementedException(
-                '`add_query` is not implemented for this adapter!')
-
-    ###
-    # Special bigquery adapter methods
-    ###
-    @available.parse_none
-    def make_date_partitioned_table(self, relation):
-        return self.connections.create_date_partitioned_table(
-            database=relation.database,
-            schema=relation.schema,
-            table_name=relation.identifier
-        )
-
-    @available.parse(lambda *a, **k: '')
-    def execute_model(self, model, materialization, sql_override=None,
-                      decorator=None):
-
-        if sql_override is None:
-            sql_override = model.get('compiled_sql')
-
-        if materialization == 'view':
-            res = self._materialize_as_view(model)
-        elif materialization == 'table':
-            res = self._materialize_as_table(model, sql_override, decorator)
-        else:
-            msg = "Invalid relation type: '{}'".format(materialization)
-            raise dbt.exceptions.RuntimeException(msg, model)
-
-        return res
-
-    def _partitions_match(
-        self, table, conf_partition: Optional[PartitionConfig]
-    ) -> bool:
-        """
-        Check if the actual and configured partitions for a table are a match.
-        BigQuery tables can be replaced if:
-        - Both tables are not partitioned, OR
-        - Both tables are partitioned using the exact same configs
-
-        If there is a mismatch, then the table cannot be replaced directly.
-        """
-        is_partitioned = (table.range_partitioning or table.time_partitioning)
-
-        if not is_partitioned and not conf_partition:
-            return True
-        elif conf_partition and table.time_partitioning is not None:
-            table_field = table.time_partitioning.field.lower()
-            table_granularity = table.partitioning_type.lower()
-            return table_field == conf_partition.field.lower() \
-                and table_granularity == conf_partition.granularity.lower()
-        elif conf_partition and table.range_partitioning is not None:
-            dest_part = table.range_partitioning
-            conf_part = conf_partition.range or {}
-
-            return dest_part.field == conf_partition.field \
-                and dest_part.range_.start == conf_part.get('start') \
-                and dest_part.range_.end == conf_part.get('end') \
-                and dest_part.range_.interval == conf_part.get('interval')
-        else:
-            return False
-
-    def _clusters_match(self, table, conf_cluster) -> bool:
-        """
-        Check if the actual and configured clustering columns for a table
-        are a match. BigQuery tables can be replaced if clustering columns
-        match exactly.
-        """
-        if isinstance(conf_cluster, str):
-            conf_cluster = [conf_cluster]
-
-        return table.clustering_fields == conf_cluster
-
-    @available.parse(lambda *a, **k: True)
-    def is_replaceable(
-        self,
-        relation,
-        conf_partition: Optional[PartitionConfig],
-        conf_cluster
-    ) -> bool:
-        """
-        Check if a given partition and clustering column spec for a table
-        can replace an existing relation in the database. BigQuery does not
-        allow tables to be replaced with another table that has a different
-        partitioning spec. This method returns True if the given config spec is
-        identical to that of the existing table.
-        """
-        if not relation:
-            return True
-
-        try:
-            table = self.connections.get_bq_table(
-                database=relation.database,
-                schema=relation.schema,
-                identifier=relation.identifier
-            )
-        except google.cloud.exceptions.NotFound:
-            return True
-
-        return all((
-            self._partitions_match(table, conf_partition),
-            self._clusters_match(table, conf_cluster)
-        ))
-
-    @available
-    def parse_partition_by(
-        self, raw_partition_by: Any
-    ) -> Optional[PartitionConfig]:
-        """
-        dbt v0.16.0 expects `partition_by` to be a dictionary where previously
-        it was a string. Check the type of `partition_by`, raise error
-        or warning if string, and attempt to convert to dict.
-        """
-        return PartitionConfig.parse(raw_partition_by)
-
-    def get_table_ref_from_relation(self, conn, relation):
-        return self.connections.table_ref(relation.database,
-                                          relation.schema,
-                                          relation.identifier,
-                                          conn)
-
-    def _update_column_dict(self, bq_column_dict, dbt_columns, parent=''):
-        """
-        Helper function to recursively traverse the schema of a table in the
-        update_column_descriptions function below.
-
-        bq_column_dict should be a dict as obtained by the to_api_repr()
-        function of a SchemaField object.
-        """
-        if parent:
-            dotted_column_name = '{}.{}'.format(parent, bq_column_dict['name'])
-        else:
-            dotted_column_name = bq_column_dict['name']
-
-        if dotted_column_name in dbt_columns:
-            column_config = dbt_columns[dotted_column_name]
-            bq_column_dict['description'] = column_config.get('description')
-            if column_config.get('policy_tags'):
-                bq_column_dict['policyTags'] = {
-                    'names': column_config.get('policy_tags')
-                }
-
-        new_fields = []
-        for child_col_dict in bq_column_dict.get('fields', list()):
-            new_child_column_dict = self._update_column_dict(
-                child_col_dict,
-                dbt_columns,
-                parent=dotted_column_name
-            )
-            new_fields.append(new_child_column_dict)
-
-        bq_column_dict['fields'] = new_fields
-
-        return bq_column_dict
-
-    @available.parse_none
-    def update_columns(self, relation, columns):
-        if len(columns) == 0:
-            return
-
-        conn = self.connections.get_thread_connection()
-        table_ref = self.get_table_ref_from_relation(conn, relation)
-        table = conn.handle.get_table(table_ref)
-
-        new_schema = []
-        for bq_column in table.schema:
-            bq_column_dict = bq_column.to_api_repr()
-            new_bq_column_dict = self._update_column_dict(
-                bq_column_dict,
-                columns
-            )
-            new_schema.append(SchemaField.from_api_repr(new_bq_column_dict))
-
-        new_table = google.cloud.bigquery.Table(table_ref, schema=new_schema)
-        conn.handle.update_table(new_table, ['schema'])
-
-    @available.parse_none
-    def update_table_description(
-        self, database: str, schema: str, identifier: str, description: str
-    ):
-        conn = self.connections.get_thread_connection()
-        client = conn.handle
-
-        table_ref = self.connections.table_ref(
-            database,
-            schema,
-            identifier,
-            conn
-        )
-        table = client.get_table(table_ref)
-        table.description = description
-        client.update_table(table, ['description'])
-
-    @available.parse_none
-    def alter_table_add_columns(self, relation, columns):
-
-        logger.debug('Adding columns ({}) to table {}".'.format(
-                     columns, relation))
-
-        conn = self.connections.get_thread_connection()
-        client = conn.handle
-
-        table_ref = self.connections.table_ref(relation.database,
-                                               relation.schema,
-                                               relation.identifier, conn)
-        table = client.get_table(table_ref)
-
-        new_columns = [col.column_to_bq_schema() for col in columns]
-        new_schema = table.schema + new_columns
-
-        new_table = google.cloud.bigquery.Table(table_ref, schema=new_schema)
-        client.update_table(new_table, ['schema'])
-
-    @available.parse_none
-    def load_dataframe(self, database, schema, table_name, agate_table,
-                       column_override):
-        bq_schema = self._agate_to_schema(agate_table, column_override)
-        conn = self.connections.get_thread_connection()
-        client = conn.handle
-
-        table = self.connections.table_ref(database, schema, table_name, conn)
-
-        load_config = google.cloud.bigquery.LoadJobConfig()
-        load_config.skip_leading_rows = 1
-        load_config.schema = bq_schema
-
-        with open(agate_table.original_abspath, "rb") as f:
-            job = client.load_table_from_file(f, table, rewind=True,
-                                              job_config=load_config)
-
-        timeout = self.connections.get_timeout(conn)
-        with self.connections.exception_handler("LOAD TABLE"):
-            self.poll_until_job_completes(job, timeout)
-
-    @classmethod
-    def _catalog_filter_table(
-        cls, table: agate.Table, manifest: Manifest
-    ) -> agate.Table:
-        table = table.rename(column_names={
-            col.name: col.name.replace('__', ':') for col in table.columns
-        })
-        return super()._catalog_filter_table(table, manifest)
-
-    def _get_catalog_schemas(self, manifest: Manifest) -> SchemaSearchMap:
-        candidates = super()._get_catalog_schemas(manifest)
-        db_schemas: Dict[str, Set[str]] = {}
-        result = SchemaSearchMap()
-
-        for candidate, schemas in candidates.items():
-            database = candidate.database
-            if database not in db_schemas:
-                db_schemas[database] = set(self.list_schemas(database))
-            if candidate.schema in db_schemas[database]:
-                result[candidate] = schemas
-            else:
-                logger.debug(
-                    'Skipping catalog for {}.{} - schema does not exist'
-                    .format(database, candidate.schema)
-                )
-        return result
-
-    @available.parse(lambda *a, **k: {})
-    def get_common_options(
-        self, config: Dict[str, Any], node: Dict[str, Any], temporary: bool = False
-    ) -> Dict[str, Any]:
-        opts = {}
-
-        if (config.get('hours_to_expiration') is not None) and (not temporary):
-            expiration = (
-                'TIMESTAMP_ADD(CURRENT_TIMESTAMP(), INTERVAL '
-                '{} hour)').format(config.get('hours_to_expiration'))
-            opts['expiration_timestamp'] = expiration
-
-        if config.persist_relation_docs() and 'description' in node:
-            description = sql_escape(node['description'])
-            opts['description'] = '"""{}"""'.format(description)
-
-        if config.get('labels'):
-            labels = config.get('labels', {})
-            opts['labels'] = list(labels.items())
-
-        return opts
-
-    @available.parse(lambda *a, **k: {})
-    def get_table_options(
-        self, config: Dict[str, Any], node: Dict[str, Any], temporary: bool
-    ) -> Dict[str, Any]:
-        opts = self.get_common_options(config, node, temporary)
-
-        if temporary:
-            expiration = 'TIMESTAMP_ADD(CURRENT_TIMESTAMP(), INTERVAL 12 hour)'
-            opts['expiration_timestamp'] = expiration
-
-        if config.get('kms_key_name') is not None:
-            opts['kms_key_name'] = "'{}'".format(config.get('kms_key_name'))
-
-        if config.get('require_partition_filter'):
-            opts['require_partition_filter'] = config.get(
-                'require_partition_filter')
-
-        if config.get('partition_expiration_days') is not None:
-            opts['partition_expiration_days'] = config.get(
-                'partition_expiration_days')
-
-        return opts
-
-    @available.parse(lambda *a, **k: {})
-    def get_view_options(
-        self, config: Dict[str, Any], node: Dict[str, Any]
-    ) -> Dict[str, Any]:
-        opts = self.get_common_options(config, node)
-        return opts
-
-    @available.parse_none
-    def grant_access_to(self, entity, entity_type, role, grant_target_dict):
-        """
-        Given an entity, grants it access to a permissioned dataset.
-        """
-        conn = self.connections.get_thread_connection()
-        client = conn.handle
-
-        GrantTarget.validate(grant_target_dict)
-        grant_target = GrantTarget.from_dict(grant_target_dict)
-        dataset = client.get_dataset(
-            self.connections.dataset_from_id(grant_target.render())
-        )
-
-        if entity_type == 'view':
-            entity = self.connections.table_ref(
-                entity.database,
-                entity.schema,
-                entity.identifier,
-                conn).to_api_repr()
-
-        access_entry = AccessEntry(role, entity_type, entity)
-        access_entries = dataset.access_entries
-
-        if access_entry in access_entries:
-            logger.debug(f"Access entry {access_entry} "
-                         f"already exists in dataset")
-            return
-
-        access_entries.append(AccessEntry(role, entity_type, entity))
-        dataset.access_entries = access_entries
-        client.update_dataset(dataset, ['access_entries'])
-
-    def get_rows_different_sql(
-        self,
-        relation_a: BigQueryRelation,
-        relation_b: BigQueryRelation,
-        column_names: Optional[List[str]] = None,
-        except_operator='EXCEPT DISTINCT'
-    ) -> str:
-        return super().get_rows_different_sql(
-            relation_a=relation_a,
-            relation_b=relation_b,
-            column_names=column_names,
-            except_operator=except_operator,
-        )
-
-    def timestamp_add_sql(
-        self, add_to: str, number: int = 1, interval: str = 'hour'
-    ) -> str:
-        return f'timestamp_add({add_to}, interval {number} {interval})'
-
-    def string_add_sql(
-        self, add_to: str, value: str, location='append',
-    ) -> str:
-        if location == 'append':
-            return f"concat({add_to}, '{value}')"
-        elif location == 'prepend':
-            return f"concat('{value}', {add_to})"
-        else:
-            raise dbt.exceptions.RuntimeException(
-                f'Got an unexpected location value of "{location}"'
-            )

plugins/bigquery/dbt/adapters/bigquery/relation.py

@@ -1,80 +0,0 @@
-from dataclasses import dataclass
-from typing import Optional
-
-from dbt.adapters.base.relation import (
-    BaseRelation, ComponentName, InformationSchema
-)
-from dbt.utils import filter_null_values
-from typing import TypeVar
-
-
-Self = TypeVar('Self', bound='BigQueryRelation')
-
-
-@dataclass(frozen=True, eq=False, repr=False)
-class BigQueryRelation(BaseRelation):
-    quote_character: str = '`'
-
-    def matches(
-        self,
-        database: Optional[str] = None,
-        schema: Optional[str] = None,
-        identifier: Optional[str] = None,
-    ) -> bool:
-        search = filter_null_values({
-            ComponentName.Database: database,
-            ComponentName.Schema: schema,
-            ComponentName.Identifier: identifier
-        })
-
-        if not search:
-            # nothing was passed in
-            pass
-
-        for k, v in search.items():
-            if not self._is_exactish_match(k, v):
-                return False
-
-        return True
-
-    @property
-    def project(self):
-        return self.database
-
-    @property
-    def dataset(self):
-        return self.schema
-
-    def information_schema(
-        self, identifier: Optional[str] = None
-    ) -> 'BigQueryInformationSchema':
-        return BigQueryInformationSchema.from_relation(self, identifier)
-
-
-@dataclass(frozen=True, eq=False, repr=False)
-class BigQueryInformationSchema(InformationSchema):
-    quote_character: str = '`'
-
-    @classmethod
-    def get_include_policy(cls, relation, information_schema_view):
-        schema = True
-        if information_schema_view in ('SCHEMATA', 'SCHEMATA_OPTIONS', None):
-            schema = False
-
-        identifier = True
-        if information_schema_view == '__TABLES__':
-            identifier = False
-
-        return relation.include_policy.replace(
-            schema=schema,
-            identifier=identifier,
-        )
-
-    def replace(self, **kwargs):
-        if 'information_schema_view' in kwargs:
-            view = kwargs['information_schema_view']
-            # we also need to update the include policy, unless the caller did
-            # in which case it's their problem
-            if 'include_policy' not in kwargs:
-                kwargs['include_policy'] = self.get_include_policy(self, view)
-        return super().replace(**kwargs)

plugins/bigquery/dbt/include/bigquery/__init__.py

@@ -1,2 +0,0 @@
-import os
-PACKAGE_PATH = os.path.dirname(__file__)

plugins/bigquery/dbt/include/bigquery/dbt_project.yml

@@ -1,5 +0,0 @@
-config-version: 2
-name: dbt_bigquery
-version: 1.0
-
-macro-paths: ["macros"]

plugins/bigquery/dbt/include/bigquery/macros/adapters.sql

@@ -1,196 +0,0 @@
-
-{% macro partition_by(partition_config) -%}
-    {%- if partition_config is none -%}
-      {% do return('') %}
-    {%- elif partition_config.data_type | lower in ('date','timestamp','datetime') -%}
-        partition by {{ partition_config.render() }}
-    {%- elif partition_config.data_type | lower in ('int64') -%}
-        {%- set range = partition_config.range -%}
-        partition by range_bucket(
-            {{ partition_config.field }},
-            generate_array({{ range.start}}, {{ range.end }}, {{ range.interval }})
-        )
-    {%- endif -%}
-{%- endmacro -%}
-
-{% macro cluster_by(raw_cluster_by) %}
-  {%- if raw_cluster_by is not none -%}
-  cluster by {% if raw_cluster_by is string -%}
-    {% set raw_cluster_by = [raw_cluster_by] %}
-  {%- endif -%}
-  {%- for cluster in raw_cluster_by -%}
-    {{ cluster }}
-    {%- if not loop.last -%}, {% endif -%}
-  {%- endfor -%}
-
-  {% endif %}
-
-{%- endmacro -%}
-
-{% macro bigquery_options(opts) %}
-  {% set options -%}
-    OPTIONS({% for opt_key, opt_val in opts.items() %}
-      {{ opt_key }}={{ opt_val }}{{ "," if not loop.last }}
-    {% endfor %})
-  {%- endset %}
-  {%- do return(options) -%}
-{%- endmacro -%}
-
-{% macro bigquery_table_options(config, node, temporary) %}
-  {% set opts = adapter.get_table_options(config, node, temporary) %}
-  {%- do return(bigquery_options(opts)) -%}
-{%- endmacro -%}
-
-{% macro bigquery__create_table_as(temporary, relation, sql) -%}
-  {%- set raw_partition_by = config.get('partition_by', none) -%}
-  {%- set raw_cluster_by = config.get('cluster_by', none) -%}
-  {%- set sql_header = config.get('sql_header', none) -%}
-
-  {%- set partition_config = adapter.parse_partition_by(raw_partition_by) -%}
-
-  {{ sql_header if sql_header is not none }}
-
-  create or replace table {{ relation }}
-  {{ partition_by(partition_config) }}
-  {{ cluster_by(raw_cluster_by) }}
-  {{ bigquery_table_options(config, model, temporary) }}
-  as (
-    {{ sql }}
-  );
-
-{%- endmacro -%}
-
-{% macro bigquery_view_options(config, node) %}
-  {% set opts = adapter.get_view_options(config, node) %}
-  {%- do return(bigquery_options(opts)) -%}
-{%- endmacro -%}
-
-{% macro bigquery__create_view_as(relation, sql) -%}
-  {%- set sql_header = config.get('sql_header', none) -%}
-
-  {{ sql_header if sql_header is not none }}
-
-  create or replace view {{ relation }}
-  {{ bigquery_view_options(config, model) }}
-  as {{ sql }};
-
-{% endmacro %}
-
-{% macro bigquery__create_schema(relation) -%}
-  {{ adapter.create_schema(relation) }}
-{% endmacro %}
-
-{% macro bigquery__drop_schema(relation) -%}
-  {{ adapter.drop_schema(relation) }}
-{% endmacro %}
-
-{% macro bigquery__drop_relation(relation) -%}
-  {% call statement('drop_relation') -%}
-    drop {{ relation.type }} if exists {{ relation }}
-  {%- endcall %}
-{% endmacro %}
-
-{% macro bigquery__get_columns_in_relation(relation) -%}
-  {{ return(adapter.get_columns_in_relation(relation)) }}
-{% endmacro %}
-
-
-{% macro bigquery__list_relations_without_caching(schema_relation) -%}
-  {{ return(adapter.list_relations_without_caching(schema_relation)) }}
-{%- endmacro %}
-
-
-{% macro bigquery__current_timestamp() -%}
-  CURRENT_TIMESTAMP()
-{%- endmacro %}
-
-
-{% macro bigquery__snapshot_string_as_time(timestamp) -%}
-    {%- set result = 'TIMESTAMP("' ~ timestamp ~ '")' -%}
-    {{ return(result) }}
-{%- endmacro %}
-
-
-{% macro bigquery__list_schemas(database) -%}
-  {{ return(adapter.list_schemas(database)) }}
-{% endmacro %}
-
-
-{% macro bigquery__check_schema_exists(information_schema, schema) %}
-  {{ return(adapter.check_schema_exists(information_schema.database, schema)) }}
-{% endmacro %}
-
-{#-- relation-level macro is not implemented. This is handled in the CTAs statement #}
-{% macro bigquery__persist_docs(relation, model, for_relation, for_columns) -%}
-  {% if for_columns and config.persist_column_docs() and model.columns %}
-    {% do alter_column_comment(relation, model.columns) %}
-  {% endif %}
-{% endmacro %}
-
-{% macro bigquery__alter_column_comment(relation, column_dict) -%}
-  {% do adapter.update_columns(relation, column_dict) %}
-{% endmacro %}
-
-{% macro bigquery__rename_relation(from_relation, to_relation) -%}
-  {% do adapter.rename_relation(from_relation, to_relation) %}
-{% endmacro %}
-
-{% macro bigquery__alter_relation_add_columns(relation, add_columns) %}
-  
-  {% set sql -%}
-     
-     alter {{ relation.type }} {{ relation }}
-        {% for column in add_columns %}
-          add column {{ column.name }} {{ column.data_type }}{{ ',' if not loop.last }}
-        {% endfor %}
-  
-  {%- endset -%}
-
-  {{ return(run_query(sql)) }}
-
-{% endmacro %}
-
-{% macro bigquery__alter_relation_drop_columns(relation, drop_columns) %}
-  
-  {% set sql -%}
-     
-     alter {{ relation.type }} {{ relation }}
-
-        {% for column in drop_columns %}
-          drop column {{ column.name }}{{ ',' if not loop.last }}
-        {% endfor %}
-  
-  {%- endset -%}
-  
-  {{ return(run_query(sql)) }}
-
-{% endmacro %}
-
-
-{% macro bigquery__alter_column_type(relation, column_name, new_column_type) -%}
-  {#
-    Changing a column's data type using a query requires you to scan the entire table.
-    The query charges can be significant if the table is very large.
-
-    https://cloud.google.com/bigquery/docs/manually-changing-schemas#changing_a_columns_data_type
-  #}
-  {% set relation_columns = get_columns_in_relation(relation) %}
-
-  {% set sql %}
-    select
-      {%- for col in relation_columns -%}
-        {% if col.column == column_name %}
-          CAST({{ col.quoted }} AS {{ new_column_type }}) AS {{ col.quoted }}
-        {%- else %}
-          {{ col.quoted }}
-        {%- endif %}
-        {%- if not loop.last %},{% endif -%}
-      {%- endfor %}
-    from {{ relation }}
-  {% endset %}
-
-  {% call statement('alter_column_type') %}
-    {{ create_table_as(False, relation, sql)}}
-  {%- endcall %}
-
-{% endmacro %}

plugins/bigquery/dbt/include/bigquery/macros/catalog.sql

@@ -1,209 +0,0 @@
-
-{% macro bigquery__get_catalog(information_schema, schemas) -%}
-
-  {%- if (schemas | length) == 0 -%}
-    {# Hopefully nothing cares about the columns we return when there are no rows #}
-    {%- set query  = "select 1 as id limit 0" -%}
-  {%- else -%}
-
-  {%- set query -%}
-    with tables as (
-        select
-            project_id as table_database,
-            dataset_id as table_schema,
-            table_id as original_table_name,
-
-            concat(project_id, '.', dataset_id, '.', table_id) as relation_id,
-
-            row_count,
-            size_bytes as size_bytes,
-            case
-                when type = 1 then 'table'
-                when type = 2 then 'view'
-                else 'external'
-            end as table_type,
-
-            REGEXP_CONTAINS(table_id, '^.+[0-9]{8}$') and coalesce(type, 0) = 1 as is_date_shard,
-            REGEXP_EXTRACT(table_id, '^(.+)[0-9]{8}$') as shard_base_name,
-            REGEXP_EXTRACT(table_id, '^.+([0-9]{8})$') as shard_name
-
-        from {{ information_schema.replace(information_schema_view='__TABLES__') }}
-        where (
-          {%- for schema in schemas -%}
-            upper(dataset_id) = upper('{{ schema }}'){%- if not loop.last %} or {% endif -%}
-          {%- endfor -%}
-        )
-    ),
-
-    extracted as (
-
-        select *,
-            case
-                when is_date_shard then shard_base_name
-                else original_table_name
-            end as table_name
-
-        from tables
-
-    ),
-
-    unsharded_tables as (
-
-        select
-            table_database,
-            table_schema,
-            table_name,
-            coalesce(table_type, 'external') as table_type,
-            is_date_shard,
-
-            struct(
-                min(shard_name) as shard_min,
-                max(shard_name) as shard_max,
-                count(*) as shard_count
-            ) as table_shards,
-
-            sum(size_bytes) as size_bytes,
-            sum(row_count) as row_count,
-
-            max(relation_id) as relation_id
-
-        from extracted
-        group by 1,2,3,4,5
-
-    ),
-
-    info_schema_columns as (
-
-        select
-            concat(table_catalog, '.', table_schema, '.', table_name) as relation_id,
-            table_catalog as table_database,
-            table_schema,
-            table_name,
-
-            -- use the "real" column name from the paths query below
-            column_name as base_column_name,
-            ordinal_position as column_index,
-
-            is_partitioning_column,
-            clustering_ordinal_position
-
-        from {{ information_schema.replace(information_schema_view='COLUMNS') }}
-        where ordinal_position is not null
-
-    ),
-
-    info_schema_column_paths as (
-
-        select
-            concat(table_catalog, '.', table_schema, '.', table_name) as relation_id,
-            field_path as column_name,
-            data_type as column_type,
-            column_name as base_column_name,
-            description as column_comment
-
-        from {{ information_schema.replace(information_schema_view='COLUMN_FIELD_PATHS') }}
-
-    ),
-
-    columns as (
-
-        select * except (base_column_name)
-        from info_schema_columns
-        join info_schema_column_paths using (relation_id, base_column_name)
-
-    ),
-
-    column_stats as (
-
-        select
-            table_database,
-            table_schema,
-            table_name,
-            max(relation_id) as relation_id,
-            max(case when is_partitioning_column = 'YES' then 1 else 0 end) = 1 as is_partitioned,
-            max(case when is_partitioning_column = 'YES' then column_name else null end) as partition_column,
-            max(case when clustering_ordinal_position is not null then 1 else 0 end) = 1 as is_clustered,
-            array_to_string(
-                array_agg(
-                    case
-                        when clustering_ordinal_position is not null then column_name
-                        else null
-                    end ignore nulls
-                    order by clustering_ordinal_position
-                ), ', '
-            ) as clustering_columns
-
-        from columns
-        group by 1,2,3
-
-    )
-
-    select
-        unsharded_tables.table_database,
-        unsharded_tables.table_schema,
-        case
-            when is_date_shard then concat(unsharded_tables.table_name, '*')
-            else unsharded_tables.table_name
-        end as table_name,
-        unsharded_tables.table_type,
-
-        -- coalesce name and type for External tables - these columns are not
-        -- present in the COLUMN_FIELD_PATHS resultset
-        coalesce(columns.column_name, '<unknown>') as column_name,
-        -- invent a row number to account for nested fields -- BQ does
-        -- not treat these nested properties as independent fields
-        row_number() over (
-            partition by relation_id
-            order by columns.column_index, columns.column_name
-        ) as column_index,
-        coalesce(columns.column_type, '<unknown>') as column_type,
-        columns.column_comment,
-
-        'Shard count' as `stats__date_shards__label`,
-        table_shards.shard_count as `stats__date_shards__value`,
-        'The number of date shards in this table' as `stats__date_shards__description`,
-        is_date_shard as `stats__date_shards__include`,
-
-        'Shard (min)' as `stats__date_shard_min__label`,
-        table_shards.shard_min as `stats__date_shard_min__value`,
-        'The first date shard in this table' as `stats__date_shard_min__description`,
-        is_date_shard as `stats__date_shard_min__include`,
-
-        'Shard (max)' as `stats__date_shard_max__label`,
-        table_shards.shard_max as `stats__date_shard_max__value`,
-        'The last date shard in this table' as `stats__date_shard_max__description`,
-        is_date_shard as `stats__date_shard_max__include`,
-
-        '# Rows' as `stats__num_rows__label`,
-        row_count as `stats__num_rows__value`,
-        'Approximate count of rows in this table' as `stats__num_rows__description`,
-        (unsharded_tables.table_type = 'table') as `stats__num_rows__include`,
-
-        'Approximate Size' as `stats__num_bytes__label`,
-        size_bytes as `stats__num_bytes__value`,
-        'Approximate size of table as reported by BigQuery' as `stats__num_bytes__description`,
-        (unsharded_tables.table_type = 'table') as `stats__num_bytes__include`,
-
-        'Partitioned By' as `stats__partitioning_type__label`,
-        partition_column as `stats__partitioning_type__value`,
-        'The partitioning column for this table' as `stats__partitioning_type__description`,
-        is_partitioned as `stats__partitioning_type__include`,
-
-        'Clustered By' as `stats__clustering_fields__label`,
-        clustering_columns as `stats__clustering_fields__value`,
-        'The clustering columns for this table' as `stats__clustering_fields__description`,
-        is_clustered as `stats__clustering_fields__include`
-
-    -- join using relation_id (an actual relation, not a shard prefix) to make
-    -- sure that column metadata is picked up through the join. This will only
-    -- return the column information for the "max" table in a date-sharded table set
-    from unsharded_tables
-    left join columns using (relation_id)
-    left join column_stats using (relation_id)
-  {%- endset -%}
-
-  {%- endif -%}
-
-  {{ return(run_query(query)) }}
-
-{%- endmacro %}

plugins/bigquery/dbt/include/bigquery/macros/etc.sql

@@ -1,15 +0,0 @@
-{% macro date_sharded_table(base_name) %}
-    {{ return(base_name ~ "[DBT__PARTITION_DATE]") }}
-{% endmacro %}
-
-{% macro grant_access_to(entity, entity_type, role, grant_target_dict) -%}
-  {% do adapter.grant_access_to(entity, entity_type, role, grant_target_dict) %}
-{% endmacro %}
-
-{%- macro get_partitions_metadata(table) -%}
-  {%- if execute -%}
-    {%- set res = adapter.get_partitions_metadata(table) -%}
-    {{- return(res) -}}
-  {%- endif -%}
-  {{- return(None) -}}
-{%- endmacro -%}

plugins/bigquery/dbt/include/bigquery/macros/materializations/copy.sql

@@ -1,32 +0,0 @@
-{% materialization copy, adapter='bigquery' -%}
-
-  {# Setup #}
-  {{ run_hooks(pre_hooks) }}
-
-  {% set destination = this.incorporate(type='table') %}
-
-  {# there can be several ref() or source() according to BQ copy API docs #}
-  {# cycle over ref() and source() to create source tables array #}
-  {% set source_array = [] %}
-  {% for ref_table in model.refs %}
-    {{ source_array.append(ref(*ref_table)) }}
-  {% endfor %}
-
-  {% for src_table in model.sources %}
-    {{ source_array.append(source(*src_table)) }}
-  {% endfor %}
-
-  {# Call adapter's copy_table function #}
-  {%- set result_str = adapter.copy_table(
-      source_array,
-      destination,
-      config.get('copy_materialization', default = 'table')) -%}
-
-  {{ store_result('main', response=result_str) }}
-
-  {# Clean up #}
-  {{ run_hooks(post_hooks) }}
-  {{ adapter.commit() }}
-
-  {{ return({'relations': [destination]}) }}
-{%- endmaterialization %}

plugins/bigquery/dbt/include/bigquery/macros/materializations/incremental.sql

@@ -1,189 +0,0 @@
-
-{% macro dbt_bigquery_validate_get_incremental_strategy(config) %}
-  {#-- Find and validate the incremental strategy #}
-  {%- set strategy = config.get("incremental_strategy", default="merge") -%}
-
-  {% set invalid_strategy_msg -%}
-    Invalid incremental strategy provided: {{ strategy }}
-    Expected one of: 'merge', 'insert_overwrite'
-  {%- endset %}
-  {% if strategy not in ['merge', 'insert_overwrite'] %}
-    {% do exceptions.raise_compiler_error(invalid_strategy_msg) %}
-  {% endif %}
-
-  {% do return(strategy) %}
-{% endmacro %}
-
-
-{% macro bq_insert_overwrite(
-    tmp_relation, target_relation, sql, unique_key, partition_by, partitions, dest_columns, tmp_relation_exists
-) %}
-
-  {% if partitions is not none and partitions != [] %} {# static #}
-
-      {% set predicate -%}
-          {{ partition_by.render(alias='DBT_INTERNAL_DEST') }} in (
-              {{ partitions | join (', ') }}
-          )
-      {%- endset %}
-
-      {%- set source_sql -%}
-        (
-          {{sql}}
-        )
-      {%- endset -%}
-
-      {{ get_insert_overwrite_merge_sql(target_relation, source_sql, dest_columns, [predicate], include_sql_header=true) }}
-
-  {% else %} {# dynamic #}
-
-      {% set predicate -%}
-          {{ partition_by.render(alias='DBT_INTERNAL_DEST') }} in unnest(dbt_partitions_for_replacement)
-      {%- endset %}
-
-      {%- set source_sql -%}
-      (
-        select * from {{ tmp_relation }}
-      )
-      {%- endset -%}
-
-      -- generated script to merge partitions into {{ target_relation }}
-      declare dbt_partitions_for_replacement array<{{ partition_by.data_type }}>;
-      declare _dbt_max_partition {{ partition_by.data_type }} default (
-          select max({{ partition_by.field }}) from {{ this }}
-          where {{ partition_by.field }} is not null
-      );
-
-      {# have we already created the temp table to check for schema changes? #}
-      {% if not tmp_relation_exists %}
-        -- 1. create a temp table
-        {{ create_table_as(True, tmp_relation, sql) }}
-      {% else %}
-        -- 1. temp table already exists, we used it to check for schema changes
-      {% endif %}
-
-      -- 2. define partitions to update
-      set (dbt_partitions_for_replacement) = (
-          select as struct
-              array_agg(distinct {{ partition_by.render() }})
-          from {{ tmp_relation }}
-      );
-
-      {#
-        TODO: include_sql_header is a hack; consider a better approach that includes
-              the sql_header at the materialization-level instead
-      #}
-      -- 3. run the merge statement
-      {{ get_insert_overwrite_merge_sql(target_relation, source_sql, dest_columns, [predicate], include_sql_header=false) }};
-
-      -- 4. clean up the temp table
-      drop table if exists {{ tmp_relation }}
-
-  {% endif %}
-
-{% endmacro %}
-
-
-{% macro bq_generate_incremental_build_sql(
-    strategy, tmp_relation, target_relation, sql, unique_key, partition_by, partitions, dest_columns, tmp_relation_exists
-) %}
-  {#-- if partitioned, use BQ scripting to get the range of partition values to be updated --#}
-  {% if strategy == 'insert_overwrite' %}
-
-    {% set missing_partition_msg -%}
-      The 'insert_overwrite' strategy requires the `partition_by` config.
-    {%- endset %}
-    {% if partition_by is none %}
-      {% do exceptions.raise_compiler_error(missing_partition_msg) %}
-    {% endif %}
-
-    {% set build_sql = bq_insert_overwrite(
-        tmp_relation, target_relation, sql, unique_key, partition_by, partitions, dest_columns, on_schema_change
-    ) %}
-
-  {% else %} {# strategy == 'merge' #}
-    {%- set source_sql -%}
-      {%- if tmp_relation_exists -%}
-        (
-          select * from {{ tmp_relation }}
-        )
-      {%- else -%} {#-- wrap sql in parens to make it a subquery --#}
-        (
-          {{sql}}
-        )
-      {%- endif -%}
-    {%- endset -%}
-
-    {% set build_sql = get_merge_sql(target_relation, source_sql, unique_key, dest_columns) %}
-
-  {% endif %}
-
-  {{ return(build_sql) }}
-
-{% endmacro %}
-
-{% materialization incremental, adapter='bigquery' -%}
-
-  {%- set unique_key = config.get('unique_key') -%}
-  {%- set full_refresh_mode = (should_full_refresh()) -%}
-
-  {%- set target_relation = this %}
-  {%- set existing_relation = load_relation(this) %}
-  {%- set tmp_relation = make_temp_relation(this) %}
-
-  {#-- Validate early so we don't run SQL if the strategy is invalid --#}
-  {% set strategy = dbt_bigquery_validate_get_incremental_strategy(config) -%}
-
-  {%- set raw_partition_by = config.get('partition_by', none) -%}
-  {%- set partition_by = adapter.parse_partition_by(raw_partition_by) -%}
-  {%- set partitions = config.get('partitions', none) -%}
-  {%- set cluster_by = config.get('cluster_by', none) -%}
-
-  {% set on_schema_change = incremental_validate_on_schema_change(config.get('on_schema_change'), default='ignore') %}
-
-  {{ run_hooks(pre_hooks) }}
-
-  {% if existing_relation is none %}
-      {% set build_sql = create_table_as(False, target_relation, sql) %}
-  
-  {% elif existing_relation.is_view %}
-      {#-- There's no way to atomically replace a view with a table on BQ --#}
-      {{ adapter.drop_relation(existing_relation) }}
-      {% set build_sql = create_table_as(False, target_relation, sql) %}
-  
-  {% elif full_refresh_mode %}
-      {#-- If the partition/cluster config has changed, then we must drop and recreate --#}
-      {% if not adapter.is_replaceable(existing_relation, partition_by, cluster_by) %}
-          {% do log("Hard refreshing " ~ existing_relation ~ " because it is not replaceable") %}
-          {{ adapter.drop_relation(existing_relation) }}
-      {% endif %}
-      {% set build_sql = create_table_as(False, target_relation, sql) %}
-  
-  {% else %}
-    {% set tmp_relation_exists = false %}
-    {% if on_schema_change != 'ignore' %} {# Check first, since otherwise we may not build a temp table #}
-      {% do run_query(create_table_as(True, tmp_relation, sql)) %}
-      {% set tmp_relation_exists = true %}
-      {% do process_schema_changes(on_schema_change, tmp_relation, existing_relation) %}
-    {% endif %}
-    
-    {% set dest_columns = adapter.get_columns_in_relation(existing_relation) %}
-    {% set build_sql = bq_generate_incremental_build_sql(
-        strategy, tmp_relation, target_relation, sql, unique_key, partition_by, partitions, dest_columns, tmp_relation_exists
-    ) %}
-
-  {% endif %}
-
-  {%- call statement('main') -%}
-    {{ build_sql }}
-  {% endcall %}
-
-  {{ run_hooks(post_hooks) }}
-
-  {% set target_relation = this.incorporate(type='table') %}
-
-  {% do persist_docs(target_relation, model) %}
-
-  {{ return({'relations': [target_relation]}) }}
-
-{%- endmaterialization %}

plugins/bigquery/dbt/include/bigquery/macros/materializations/seed.sql

@@ -1,19 +0,0 @@
-
-{% macro bigquery__create_csv_table(model, agate_table) %}
-    -- no-op
-{% endmacro %}
-
-{% macro bigquery__reset_csv_table(model, full_refresh, old_relation, agate_table) %}
-    {{ adapter.drop_relation(old_relation) }}
-{% endmacro %}
-
-{% macro bigquery__load_csv_rows(model, agate_table) %}
-
-  {%- set column_override = model['config'].get('column_types', {}) -%}
-  {{ adapter.load_dataframe(model['database'], model['schema'], model['alias'],
-  							agate_table, column_override) }}
-  {% if config.persist_relation_docs() and 'description' in model %}
-
-  	{{ adapter.update_table_description(model['database'], model['schema'], model['alias'], model['description']) }}
-  {% endif %}
-{% endmacro %}

plugins/bigquery/dbt/include/bigquery/macros/materializations/snapshot.sql

@@ -1,15 +0,0 @@
-{% macro bigquery__snapshot_hash_arguments(args) -%}
-  to_hex(md5(concat({%- for arg in args -%}
-    coalesce(cast({{ arg }} as string), ''){% if not loop.last %}, '|',{% endif -%}
-  {%- endfor -%}
-  )))
-{%- endmacro %}
-
-{% macro bigquery__create_columns(relation, columns) %}
-  {{ adapter.alter_table_add_columns(relation, columns) }}
-{% endmacro %}
-
-{% macro bigquery__post_snapshot(staging_relation) %}
-  -- Clean up the snapshot temp table
-  {% do drop_relation(staging_relation) %}
-{% endmacro %}

plugins/bigquery/dbt/include/bigquery/macros/materializations/table.sql

@@ -1,84 +0,0 @@
-{% macro make_date_partitioned_table(model, relation, dates, should_create, verbose=False) %}
-
-  {% if should_create %}
-      {{ adapter.make_date_partitioned_table(relation) }}
-  {% endif %}
-
-  {% for date in dates %}
-    {% set date = (date | string) %}
-    {% if verbose %}
-        {% set table_start_time = modules.datetime.datetime.now().strftime("%H:%M:%S") %}
-        {{ log(table_start_time ~ ' | -> Running for day ' ~ date, info=True) }}
-    {% endif %}
-
-    {% set fixed_sql = model['compiled_sql'] | replace('[DBT__PARTITION_DATE]', date) %}
-    {% set _ = adapter.execute_model(model, 'table', fixed_sql, decorator=date) %}
-  {% endfor %}
-
-  {% set num_days = dates | length %}
-  {% if num_days == 1 %}
-      {% set result_str = 'CREATED 1 PARTITION' %}
-  {% else %}
-      {% set result_str = 'CREATED ' ~ num_days ~ ' PARTITIONS' %}
-  {% endif %}
-
-  {{ store_result('main', response=result_str) }}
-
-{% endmacro %}
-
-{% materialization table, adapter='bigquery' -%}
-
-  {%- set identifier = model['alias'] -%}
-  {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}
-  {%- set exists_not_as_table = (old_relation is not none and not old_relation.is_table) -%}
-  {%- set target_relation = api.Relation.create(database=database, schema=schema, identifier=identifier, type='table') -%}
-  {%- set verbose = config.get('verbose', False) -%}
-
-  {# partitions: iterate over each partition, running a separate query in a for-loop #}
-  {%- set partitions = config.get('partitions') -%}
-
-  {% if partitions %}
-      {% if partitions is number or partitions is string %}
-        {% set partitions = [(partitions | string)] %}
-      {% endif %}
-
-      {% if partitions is not iterable %}
-        {{ exceptions.raise_compiler_error("Provided `partitions` configuration is not a list. Got: " ~ partitions, model) }}
-      {% endif %}
-  {% endif %}
-
-  {{ run_hooks(pre_hooks) }}
-
-  {#
-      Since dbt uses WRITE_TRUNCATE mode for tables, we only need to drop this thing
-      if it is not a table. If it _is_ already a table, then we can overwrite it without downtime
-  #}
-  {%- if exists_not_as_table -%}
-      {{ adapter.drop_relation(old_relation) }}
-  {%- endif -%}
-
-  -- build model
-  {% if partitions %}
-    {# Create the dp-table if 1. it does not exist or 2. it existed, but we just dropped it #}
-    {%- set should_create = (old_relation is none or exists_not_as_table) -%}
-    {{ make_date_partitioned_table(model, target_relation, partitions, should_create, verbose) }}
-  {% else %}
-    {%- set raw_partition_by = config.get('partition_by', none) -%}
-    {%- set partition_by = adapter.parse_partition_by(raw_partition_by) -%}
-    {%- set cluster_by = config.get('cluster_by', none) -%}
-    {% if not adapter.is_replaceable(old_relation, partition_by, cluster_by) %}
-      {% do log("Hard refreshing " ~ old_relation ~ " because it is not replaceable") %}
-      {% do adapter.drop_relation(old_relation) %}
-    {% endif %}
-    {% call statement('main') -%}
-      {{ create_table_as(False, target_relation, sql) }}
-    {% endcall -%}
-  {% endif %}
-
-  {{ run_hooks(post_hooks) }}
-
-  {% do persist_docs(target_relation, model) %}
-
-  {{ return({'relations': [target_relation]}) }}
-
-{% endmaterialization %}

plugins/bigquery/dbt/include/bigquery/macros/materializations/view.sql

@@ -1,25 +0,0 @@
-
-{% macro bigquery__handle_existing_table(full_refresh, old_relation) %}
-    {%- if full_refresh -%}
-      {{ adapter.drop_relation(old_relation) }}
-    {%- else -%}
-      {{ exceptions.relation_wrong_type(old_relation, 'view') }}
-    {%- endif -%}
-{% endmacro %}
-
-
-{% materialization view, adapter='bigquery' -%}
-    {% set to_return = create_or_replace_view() %}
-
-    {% set target_relation = this.incorporate(type='view') %}
-    {% do persist_docs(target_relation, model) %}
-
-    {% if config.get('grant_access_to') %}
-      {% for grant_target_dict in config.get('grant_access_to') %}
-        {% do adapter.grant_access_to(this, 'view', None, grant_target_dict) %}
-      {% endfor %}
-    {% endif %}
-
-    {% do return(to_return) %}
-
-{%- endmaterialization %}

plugins/bigquery/dbt/include/bigquery/sample_profiles.yml

@@ -1,26 +0,0 @@
-default:
-  outputs:
-
-    dev:
-      type: bigquery
-      method: oauth
-      project: [GCP project id]
-      dataset: [the name of your dbt dataset] # You can also use "schema" here
-      threads: [1 or more]
-      timeout_seconds: 300
-      location: US # Optional, one of US or EU
-      priority: interactive
-      retries: 1
-
-    prod:
-      type: bigquery
-      method: service-account
-      project: [GCP project id]
-      dataset: [the name of your dbt dataset]
-      threads: [1 or more]
-      keyfile: [/path/to/bigquery/keyfile.json]
-      timeout_seconds: 300
-      priority: interactive
-      retries: 1
-
-  target: dev

plugins/bigquery/setup.py

@@ -1,73 +0,0 @@
-#!/usr/bin/env python
-import os
-import sys
-
-if sys.version_info < (3, 6):
-    print('Error: dbt does not support this version of Python.')
-    print('Please upgrade to Python 3.6 or higher.')
-    sys.exit(1)
-
-
-from setuptools import setup
-try:
-    from setuptools import find_namespace_packages
-except ImportError:
-    # the user has a downlevel version of setuptools.
-    print('Error: dbt requires setuptools v40.1.0 or higher.')
-    print('Please upgrade setuptools with "pip install --upgrade setuptools" '
-          'and try again')
-    sys.exit(1)
-
-
-package_name = "dbt-bigquery"
-package_version = "0.21.0rc1"
-description = """The bigquery adapter plugin for dbt (data build tool)"""
-
-this_directory = os.path.abspath(os.path.dirname(__file__))
-with open(os.path.join(this_directory, 'README.md')) as f:
-    long_description = f.read()
-
-setup(
-    name=package_name,
-    version=package_version,
-    description=description,
-    long_description=long_description,
-    long_description_content_type='text/markdown',
-    author="dbt Labs",
-    author_email="info@dbtlabs.com",
-    url="https://github.com/dbt-labs/dbt",
-    packages=find_namespace_packages(include=['dbt', 'dbt.*']),
-    package_data={
-        'dbt': [
-            'include/bigquery/dbt_project.yml',
-            'include/bigquery/sample_profiles.yml',
-            'include/bigquery/macros/*.sql',
-            'include/bigquery/macros/**/*.sql',
-        ]
-    },
-    install_requires=[
-        'dbt-core=={}'.format(package_version),
-        'protobuf>=3.13.0,<4',
-        'google-cloud-core>=1.3.0,<2',
-        'google-cloud-bigquery>=1.25.0,<3',
-        'google-api-core>=1.16.0,<2',
-        'googleapis-common-protos>=1.6.0,<2',
-        'six>=1.14.0',
-    ],
-    zip_safe=False,
-    classifiers=[
-        'Development Status :: 5 - Production/Stable',
-
-        'License :: OSI Approved :: Apache Software License',
-
-        'Operating System :: Microsoft :: Windows',
-        'Operating System :: MacOS :: MacOS X',
-        'Operating System :: POSIX :: Linux',
-
-        'Programming Language :: Python :: 3.6',
-        'Programming Language :: Python :: 3.7',
-        'Programming Language :: Python :: 3.8',
-        'Programming Language :: Python :: 3.9',
-    ],
-    python_requires=">=3.6.2",
-)

plugins/postgres/README.md

@@ -1,32 +0,0 @@
-<p align="center">
-  <img src="https://raw.githubusercontent.com/dbt-labs/dbt/6c6649f9129d5d108aa3b0526f634cd8f3a9d1ed/etc/dbt-logo-full.svg" alt="dbt logo" width="500"/>
-</p>
-
-**[dbt](https://www.getdbt.com/)** (data build tool) enables data analysts and engineers to transform their data using the same practices that software engineers use to build applications.
-
-dbt is the T in ELT. Organize, cleanse, denormalize, filter, rename, and pre-aggregate the raw data in your warehouse so that it's ready for analysis.
-
-## dbt-postgres
-
-The `dbt-postgres` package contains all of the code required to make dbt operate on a Postgres database. For
-more information on using dbt with Postgres, consult [the docs](https://docs.getdbt.com/docs/profile-postgres).
-
-
-## Find out more
-
--   Check out the [Introduction to dbt](https://docs.getdbt.com/docs/introduction/).
--   Read the [dbt Viewpoint](https://docs.getdbt.com/docs/about/viewpoint/).
-
-## Join thousands of analysts in the dbt community
-
--   Join the [chat](http://community.getdbt.com/) on Slack.
--   Find community posts on [dbt Discourse](https://discourse.getdbt.com).
-
-## Reporting bugs and contributing code
-
--   Want to report a bug or request a feature? Let us know on [Slack](http://community.getdbt.com/), or open [an issue](https://github.com/dbt-labs/dbt/issues/new).
--   Want to help us build dbt? Check out the [Contributing Getting Started Guide](https://github.com/dbt-labs/dbt/blob/HEAD/CONTRIBUTING.md)
-
-## Code of Conduct
-
-Everyone interacting in the dbt project's codebases, issue trackers, chat rooms, and mailing lists is expected to follow the [dbt Code of Conduct](https://community.getdbt.com/code-of-conduct).

plugins/postgres/dbt/adapters/postgres/__init__.py

@@ -1,14 +0,0 @@
-# these are mostly just exports, #noqa them so flake8 will be happy
-from dbt.adapters.postgres.connections import PostgresConnectionManager  # noqa
-from dbt.adapters.postgres.connections import PostgresCredentials
-from dbt.adapters.postgres.relation import PostgresColumn  # noqa
-from dbt.adapters.postgres.relation import PostgresRelation  # noqa: F401
-from dbt.adapters.postgres.impl import PostgresAdapter
-
-from dbt.adapters.base import AdapterPlugin
-from dbt.include import postgres
-
-Plugin = AdapterPlugin(
-    adapter=PostgresAdapter,
-    credentials=PostgresCredentials,
-    include_path=postgres.PACKAGE_PATH)

plugins/postgres/dbt/adapters/postgres/__version__.py

@@ -1 +0,0 @@
-version = '0.21.0rc1'

plugins/postgres/dbt/adapters/postgres/connections.py

@@ -1,186 +0,0 @@
-from contextlib import contextmanager
-
-import psycopg2
-
-import dbt.exceptions
-from dbt.adapters.base import Credentials
-from dbt.adapters.sql import SQLConnectionManager
-from dbt.contracts.connection import AdapterResponse
-from dbt.logger import GLOBAL_LOGGER as logger
-
-from dbt.helper_types import Port
-from dataclasses import dataclass
-from typing import Optional
-
-
-@dataclass
-class PostgresCredentials(Credentials):
-    host: str
-    user: str
-    port: Port
-    password: str  # on postgres the password is mandatory
-    connect_timeout: int = 10
-    role: Optional[str] = None
-    search_path: Optional[str] = None
-    keepalives_idle: int = 0  # 0 means to use the default value
-    sslmode: Optional[str] = None
-    sslcert: Optional[str] = None
-    sslkey: Optional[str] = None
-    sslrootcert: Optional[str] = None
-    application_name: Optional[str] = 'dbt'
-
-    _ALIASES = {
-        'dbname': 'database',
-        'pass': 'password'
-    }
-
-    @property
-    def type(self):
-        return 'postgres'
-
-    @property
-    def unique_field(self):
-        return self.host
-
-    def _connection_keys(self):
-        return ('host', 'port', 'user', 'database', 'schema', 'search_path',
-                'keepalives_idle', 'sslmode')
-
-
-class PostgresConnectionManager(SQLConnectionManager):
-    TYPE = 'postgres'
-
-    @contextmanager
-    def exception_handler(self, sql):
-        try:
-            yield
-
-        except psycopg2.DatabaseError as e:
-            logger.debug('Postgres error: {}'.format(str(e)))
-
-            try:
-                self.rollback_if_open()
-            except psycopg2.Error:
-                logger.debug("Failed to release connection!")
-                pass
-
-            raise dbt.exceptions.DatabaseException(str(e).strip()) from e
-
-        except Exception as e:
-            logger.debug("Error running SQL: {}", sql)
-            logger.debug("Rolling back transaction.")
-            self.rollback_if_open()
-            if isinstance(e, dbt.exceptions.RuntimeException):
-                # during a sql query, an internal to dbt exception was raised.
-                # this sounds a lot like a signal handler and probably has
-                # useful information, so raise it without modification.
-                raise
-
-            raise dbt.exceptions.RuntimeException(e) from e
-
-    @classmethod
-    def open(cls, connection):
-        if connection.state == 'open':
-            logger.debug('Connection is already open, skipping open.')
-            return connection
-
-        credentials = cls.get_credentials(connection.credentials)
-        kwargs = {}
-        # we don't want to pass 0 along to connect() as postgres will try to
-        # call an invalid setsockopt() call (contrary to the docs).
-        if credentials.keepalives_idle:
-            kwargs['keepalives_idle'] = credentials.keepalives_idle
-
-        # psycopg2 doesn't support search_path officially,
-        # see https://github.com/psycopg/psycopg2/issues/465
-        search_path = credentials.search_path
-        if search_path is not None and search_path != '':
-            # see https://postgresql.org/docs/9.5/libpq-connect.html
-            kwargs['options'] = '-c search_path={}'.format(
-                search_path.replace(' ', '\\ '))
-
-        if credentials.sslmode:
-            kwargs['sslmode'] = credentials.sslmode
-
-        if credentials.sslcert is not None:
-            kwargs["sslcert"] = credentials.sslcert
-
-        if credentials.sslkey is not None:
-            kwargs["sslkey"] = credentials.sslkey
-
-        if credentials.sslrootcert is not None:
-            kwargs["sslrootcert"] = credentials.sslrootcert
-
-        if credentials.application_name:
-            kwargs['application_name'] = credentials.application_name
-
-        try:
-            handle = psycopg2.connect(
-                dbname=credentials.database,
-                user=credentials.user,
-                host=credentials.host,
-                password=credentials.password,
-                port=credentials.port,
-                connect_timeout=credentials.connect_timeout,
-                **kwargs)
-
-            if credentials.role:
-                handle.cursor().execute('set role {}'.format(credentials.role))
-
-            connection.handle = handle
-            connection.state = 'open'
-        except psycopg2.Error as e:
-            logger.debug("Got an error when attempting to open a postgres "
-                         "connection: '{}'"
-                         .format(e))
-
-            connection.handle = None
-            connection.state = 'fail'
-
-            raise dbt.exceptions.FailedToConnectException(str(e))
-
-        return connection
-
-    def cancel(self, connection):
-        connection_name = connection.name
-        try:
-            pid = connection.handle.get_backend_pid()
-        except psycopg2.InterfaceError as exc:
-            # if the connection is already closed, not much to cancel!
-            if 'already closed' in str(exc):
-                logger.debug(
-                    f'Connection {connection_name} was already closed'
-                )
-                return
-            # probably bad, re-raise it
-            raise
-
-        sql = "select pg_terminate_backend({})".format(pid)
-
-        logger.debug("Cancelling query '{}' ({})".format(connection_name, pid))
-
-        _, cursor = self.add_query(sql)
-        res = cursor.fetchone()
-
-        logger.debug("Cancel query '{}': {}".format(connection_name, res))
-
-    @classmethod
-    def get_credentials(cls, credentials):
-        return credentials
-
-    @classmethod
-    def get_response(cls, cursor) -> AdapterResponse:
-        message = str(cursor.statusmessage)
-        rows = cursor.rowcount
-        status_message_parts = message.split() if message is not None else []
-        status_messsage_strings = [
-            part
-            for part in status_message_parts
-            if not part.isdigit()
-        ]
-        code = ' '.join(status_messsage_strings)
-        return AdapterResponse(
-            _message=message,
-            code=code,
-            rows_affected=rows
-        )

plugins/postgres/dbt/adapters/postgres/impl.py

@@ -1,142 +0,0 @@
-from datetime import datetime
-from dataclasses import dataclass
-from typing import Optional, Set, List, Any
-from dbt.adapters.base.meta import available
-from dbt.adapters.base.impl import AdapterConfig
-from dbt.adapters.sql import SQLAdapter
-from dbt.adapters.postgres import PostgresConnectionManager
-from dbt.adapters.postgres import PostgresColumn
-from dbt.adapters.postgres import PostgresRelation
-from dbt.dataclass_schema import dbtClassMixin, ValidationError
-import dbt.exceptions
-import dbt.utils
-
-
-# note that this isn't an adapter macro, so just a single underscore
-GET_RELATIONS_MACRO_NAME = 'postgres_get_relations'
-
-
-@dataclass
-class PostgresIndexConfig(dbtClassMixin):
-    columns: List[str]
-    unique: bool = False
-    type: Optional[str] = None
-
-    def render(self, relation):
-        # We append the current timestamp to the index name because otherwise
-        # the index will only be created on every other run. See
-        # https://github.com/dbt-labs/dbt/issues/1945#issuecomment-576714925
-        # for an explanation.
-        now = datetime.utcnow().isoformat()
-        inputs = (self.columns +
-                  [relation.render(), str(self.unique), str(self.type), now])
-        string = '_'.join(inputs)
-        return dbt.utils.md5(string)
-
-    @classmethod
-    def parse(cls, raw_index) -> Optional['PostgresIndexConfig']:
-        if raw_index is None:
-            return None
-        try:
-            cls.validate(raw_index)
-            return cls.from_dict(raw_index)
-        except ValidationError as exc:
-            msg = dbt.exceptions.validator_error_message(exc)
-            dbt.exceptions.raise_compiler_error(
-                f'Could not parse index config: {msg}'
-            )
-        except TypeError:
-            dbt.exceptions.raise_compiler_error(
-                f'Invalid index config:\n'
-                f'  Got: {raw_index}\n'
-                f'  Expected a dictionary with at minimum a "columns" key'
-            )
-
-
-@dataclass
-class PostgresConfig(AdapterConfig):
-    unlogged: Optional[bool] = None
-    indexes: Optional[List[PostgresIndexConfig]] = None
-
-
-class PostgresAdapter(SQLAdapter):
-    Relation = PostgresRelation
-    ConnectionManager = PostgresConnectionManager
-    Column = PostgresColumn
-
-    AdapterSpecificConfigs = PostgresConfig
-
-    @classmethod
-    def date_function(cls):
-        return 'now()'
-
-    @available
-    def verify_database(self, database):
-        if database.startswith('"'):
-            database = database.strip('"')
-        expected = self.config.credentials.database
-        if database.lower() != expected.lower():
-            raise dbt.exceptions.NotImplementedException(
-                'Cross-db references not allowed in {} ({} vs {})'
-                .format(self.type(), database, expected)
-            )
-        # return an empty string on success so macros can call this
-        return ''
-
-    @available
-    def parse_index(self, raw_index: Any) -> Optional[PostgresIndexConfig]:
-        return PostgresIndexConfig.parse(raw_index)
-
-    def _link_cached_database_relations(self, schemas: Set[str]):
-        """
-        :param schemas: The set of schemas that should have links added.
-        """
-        database = self.config.credentials.database
-        table = self.execute_macro(GET_RELATIONS_MACRO_NAME)
-
-        for (dep_schema, dep_name, refed_schema, refed_name) in table:
-            dependent = self.Relation.create(
-                database=database,
-                schema=dep_schema,
-                identifier=dep_name
-            )
-            referenced = self.Relation.create(
-                database=database,
-                schema=refed_schema,
-                identifier=refed_name
-            )
-
-            # don't record in cache if this relation isn't in a relevant
-            # schema
-            if refed_schema.lower() in schemas:
-                self.cache.add_link(referenced, dependent)
-
-    def _get_catalog_schemas(self, manifest):
-        # postgres only allow one database (the main one)
-        schemas = super()._get_catalog_schemas(manifest)
-        try:
-            return schemas.flatten()
-        except dbt.exceptions.RuntimeException as exc:
-            dbt.exceptions.raise_compiler_error(
-                'Cross-db references not allowed in adapter {}: Got {}'.format(
-                    self.type(), exc.msg
-                )
-            )
-
-    def _link_cached_relations(self, manifest):
-        schemas: Set[str] = set()
-        relations_schemas = self._get_cache_schemas(manifest)
-        for relation in relations_schemas:
-            self.verify_database(relation.database)
-            schemas.add(relation.schema.lower())
-
-        self._link_cached_database_relations(schemas)
-
-    def _relations_cache_for_schemas(self, manifest):
-        super()._relations_cache_for_schemas(manifest)
-        self._link_cached_relations(manifest)
-
-    def timestamp_add_sql(
-        self, add_to: str, number: int = 1, interval: str = 'hour'
-    ) -> str:
-        return f"{add_to} + interval '{number} {interval}'"

plugins/postgres/dbt/adapters/postgres/relation.py

@@ -1,29 +0,0 @@
-from dbt.adapters.base import Column
-from dataclasses import dataclass
-from dbt.adapters.base.relation import BaseRelation
-from dbt.exceptions import RuntimeException
-
-
-@dataclass(frozen=True, eq=False, repr=False)
-class PostgresRelation(BaseRelation):
-    def __post_init__(self):
-        # Check for length of Postgres table/view names.
-        # Check self.type to exclude test relation identifiers
-        if (self.identifier is not None and self.type is not None and
-                len(self.identifier) > self.relation_max_name_length()):
-            raise RuntimeException(
-                f"Relation name '{self.identifier}' "
-                f"is longer than {self.relation_max_name_length()} characters"
-            )
-
-    def relation_max_name_length(self):
-        return 63
-
-
-class PostgresColumn(Column):
-    @property
-    def data_type(self):
-        # on postgres, do not convert 'text' to 'varchar()'
-        if self.dtype.lower() == 'text':
-            return self.dtype
-        return super().data_type

plugins/postgres/dbt/include/postgres/__init__.py

@@ -1,2 +0,0 @@
-import os
-PACKAGE_PATH = os.path.dirname(__file__)

plugins/postgres/dbt/include/postgres/dbt_project.yml

@@ -1,5 +0,0 @@
-config-version: 2
-name: dbt_postgres
-version: 1.0
-
-macro-paths: ["macros"]

plugins/postgres/dbt/include/postgres/macros/adapters.sql

@@ -1,188 +0,0 @@
-{% macro postgres__create_table_as(temporary, relation, sql) -%}
-  {%- set unlogged = config.get('unlogged', default=false) -%}
-  {%- set sql_header = config.get('sql_header', none) -%}
-
-  {{ sql_header if sql_header is not none }}
-
-  create {% if temporary -%}
-    temporary
-  {%- elif unlogged -%}
-    unlogged
-  {%- endif %} table {{ relation }}
-  as (
-    {{ sql }}
-  );
-{%- endmacro %}
-
-{% macro postgres__get_create_index_sql(relation, index_dict) -%}
-  {%- set index_config = adapter.parse_index(index_dict) -%}
-  {%- set comma_separated_columns = ", ".join(index_config.columns) -%}
-  {%- set index_name = index_config.render(relation) -%}
-
-  create {% if index_config.unique -%}
-    unique
-  {%- endif %} index if not exists
-  "{{ index_name }}"
-  on {{ relation }} {% if index_config.type -%}
-    using {{ index_config.type }}
-  {%- endif %}
-  ({{ comma_separated_columns }});
-{%- endmacro %}
-
-{% macro postgres__create_schema(relation) -%}
-  {% if relation.database -%}
-    {{ adapter.verify_database(relation.database) }}
-  {%- endif -%}
-  {%- call statement('create_schema') -%}
-    create schema if not exists {{ relation.without_identifier().include(database=False) }}
-  {%- endcall -%}
-{% endmacro %}
-
-{% macro postgres__drop_schema(relation) -%}
-  {% if relation.database -%}
-    {{ adapter.verify_database(relation.database) }}
-  {%- endif -%}
-  {%- call statement('drop_schema') -%}
-    drop schema if exists {{ relation.without_identifier().include(database=False) }} cascade
-  {%- endcall -%}
-{% endmacro %}
-
-{% macro postgres__get_columns_in_relation(relation) -%}
-  {% call statement('get_columns_in_relation', fetch_result=True) %}
-      select
-          column_name,
-          data_type,
-          character_maximum_length,
-          numeric_precision,
-          numeric_scale
-
-      from {{ relation.information_schema('columns') }}
-      where table_name = '{{ relation.identifier }}'
-        {% if relation.schema %}
-        and table_schema = '{{ relation.schema }}'
-        {% endif %}
-      order by ordinal_position
-
-  {% endcall %}
-  {% set table = load_result('get_columns_in_relation').table %}
-  {{ return(sql_convert_columns_in_relation(table)) }}
-{% endmacro %}
-
-
-{% macro postgres__list_relations_without_caching(schema_relation) %}
-  {% call statement('list_relations_without_caching', fetch_result=True) -%}
-    select
-      '{{ schema_relation.database }}' as database,
-      tablename as name,
-      schemaname as schema,
-      'table' as type
-    from pg_tables
-    where schemaname ilike '{{ schema_relation.schema }}'
-    union all
-    select
-      '{{ schema_relation.database }}' as database,
-      viewname as name,
-      schemaname as schema,
-      'view' as type
-    from pg_views
-    where schemaname ilike '{{ schema_relation.schema }}'
-  {% endcall %}
-  {{ return(load_result('list_relations_without_caching').table) }}
-{% endmacro %}
-
-{% macro postgres__information_schema_name(database) -%}
-  {% if database_name -%}
-    {{ adapter.verify_database(database_name) }}
-  {%- endif -%}
-  information_schema
-{%- endmacro %}
-
-{% macro postgres__list_schemas(database) %}
-  {% if database -%}
-    {{ adapter.verify_database(database) }}
-  {%- endif -%}
-  {% call statement('list_schemas', fetch_result=True, auto_begin=False) %}
-    select distinct nspname from pg_namespace
-  {% endcall %}
-  {{ return(load_result('list_schemas').table) }}
-{% endmacro %}
-
-{% macro postgres__check_schema_exists(information_schema, schema) -%}
-  {% if information_schema.database -%}
-    {{ adapter.verify_database(information_schema.database) }}
-  {%- endif -%}
-  {% call statement('check_schema_exists', fetch_result=True, auto_begin=False) %}
-    select count(*) from pg_namespace where nspname = '{{ schema }}'
-  {% endcall %}
-  {{ return(load_result('check_schema_exists').table) }}
-{% endmacro %}
-
-
-{% macro postgres__current_timestamp() -%}
-  now()
-{%- endmacro %}
-
-{% macro postgres__snapshot_string_as_time(timestamp) -%}
-    {%- set result = "'" ~ timestamp ~ "'::timestamp without time zone" -%}
-    {{ return(result) }}
-{%- endmacro %}
-
-
-{% macro postgres__snapshot_get_time() -%}
-  {{ current_timestamp() }}::timestamp without time zone
-{%- endmacro %}
-
-{#
-  Postgres tables have a maximum length off 63 characters, anything longer is silently truncated.
-  Temp relations add a lot of extra characters to the end of table namers to ensure uniqueness.
-  To prevent this going over the character limit, the base_relation name is truncated to ensure
-  that name + suffix + uniquestring is < 63 characters.  
-#}
-{% macro postgres__make_temp_relation(base_relation, suffix) %}
-    {% set dt = modules.datetime.datetime.now() %}
-    {% set dtstring = dt.strftime("%H%M%S%f") %}
-    {% set suffix_length = suffix|length + dtstring|length %}
-    {% set relation_max_name_length = 63 %}
-    {% if suffix_length > relation_max_name_length %}
-        {% do exceptions.raise_compiler_error('Temp relation suffix is too long (' ~ suffix|length ~ ' characters). Maximum length is ' ~ (relation_max_name_length - dtstring|length) ~ ' characters.') %}
-    {% endif %}
-    {% set tmp_identifier = base_relation.identifier[:relation_max_name_length - suffix_length] ~ suffix ~ dtstring %}
-    {% do return(base_relation.incorporate(
-                                  path={
-                                    "identifier": tmp_identifier,
-                                    "schema": none,
-                                    "database": none
-                                  })) -%}
-{% endmacro %}
-
-
-{#
-  By using dollar-quoting like this, users can embed anything they want into their comments
-  (including nested dollar-quoting), as long as they do not use this exact dollar-quoting
-  label. It would be nice to just pick a new one but eventually you do have to give up.
-#}
-{% macro postgres_escape_comment(comment) -%}
-  {% if comment is not string %}
-    {% do exceptions.raise_compiler_error('cannot escape a non-string: ' ~ comment) %}
-  {% endif %}
-  {%- set magic = '$dbt_comment_literal_block$' -%}
-  {%- if magic in comment -%}
-    {%- do exceptions.raise_compiler_error('The string ' ~ magic ~ ' is not allowed in comments.') -%}
-  {%- endif -%}
-  {{ magic }}{{ comment }}{{ magic }}
-{%- endmacro %}
-
-
-{% macro postgres__alter_relation_comment(relation, comment) %}
-  {% set escaped_comment = postgres_escape_comment(comment) %}
-  comment on {{ relation.type }} {{ relation }} is {{ escaped_comment }};
-{% endmacro %}
-
-
-{% macro postgres__alter_column_comment(relation, column_dict) %}
-  {% for column_name in column_dict %}
-    {% set comment = column_dict[column_name]['description'] %}
-    {% set escaped_comment = postgres_escape_comment(comment) %}
-    comment on column {{ relation }}.{{ adapter.quote(column_name) if column_dict[column_name]['quote'] else column_name }} is {{ escaped_comment }};
-  {% endfor %}
-{% endmacro %}

plugins/postgres/dbt/include/postgres/macros/catalog.sql

@@ -1,53 +0,0 @@
-
-{% macro postgres__get_catalog(information_schema, schemas) -%}
-
-  {%- call statement('catalog', fetch_result=True) -%}
-    {#
-      If the user has multiple databases set and the first one is wrong, this will fail.
-      But we won't fail in the case where there are multiple quoting-difference-only dbs, which is better.
-    #}
-    {% set database = information_schema.database %}
-    {{ adapter.verify_database(database) }}
-
-    select
-        '{{ database }}' as table_database,
-        sch.nspname as table_schema,
-        tbl.relname as table_name,
-        case tbl.relkind
-            when 'v' then 'VIEW'
-            else 'BASE TABLE'
-        end as table_type,
-        tbl_desc.description as table_comment,
-        col.attname as column_name,
-        col.attnum as column_index,
-        pg_catalog.format_type(col.atttypid, col.atttypmod) as column_type,
-        col_desc.description as column_comment,
-        pg_get_userbyid(tbl.relowner) as table_owner
-
-    from pg_catalog.pg_namespace sch
-    join pg_catalog.pg_class tbl on tbl.relnamespace = sch.oid
-    join pg_catalog.pg_attribute col on col.attrelid = tbl.oid
-    left outer join pg_catalog.pg_description tbl_desc on (tbl_desc.objoid = tbl.oid and tbl_desc.objsubid = 0)
-    left outer join pg_catalog.pg_description col_desc on (col_desc.objoid = tbl.oid and col_desc.objsubid = col.attnum)
-
-    where (
-        {%- for schema in schemas -%}
-          upper(sch.nspname) = upper('{{ schema }}'){%- if not loop.last %} or {% endif -%}
-        {%- endfor -%}
-      )
-      and not pg_is_other_temp_schema(sch.oid) -- not a temporary schema belonging to another session
-      and tbl.relpersistence = 'p' -- [p]ermanent table. Other values are [u]nlogged table, [t]emporary table
-      and tbl.relkind in ('r', 'v', 'f', 'p') -- o[r]dinary table, [v]iew, [f]oreign table, [p]artitioned table. Other values are [i]ndex, [S]equence, [c]omposite type, [t]OAST table, [m]aterialized view
-      and col.attnum > 0 -- negative numbers are used for system columns such as oid
-      and not col.attisdropped -- column as not been dropped
-
-    order by
-        sch.nspname,
-        tbl.relname,
-        col.attnum
-
-  {%- endcall -%}
-
-  {{ return(load_result('catalog').table) }}
-
-{%- endmacro %}

plugins/postgres/dbt/include/postgres/macros/materializations/snapshot_merge.sql

@@ -1,18 +0,0 @@
-
-{% macro postgres__snapshot_merge_sql(target, source, insert_cols) -%}
-    {%- set insert_cols_csv = insert_cols | join(', ') -%}
-
-    update {{ target }}
-    set dbt_valid_to = DBT_INTERNAL_SOURCE.dbt_valid_to
-    from {{ source }} as DBT_INTERNAL_SOURCE
-    where DBT_INTERNAL_SOURCE.dbt_scd_id::text = {{ target }}.dbt_scd_id::text
-      and DBT_INTERNAL_SOURCE.dbt_change_type::text in ('update'::text, 'delete'::text)
-      and {{ target }}.dbt_valid_to is null;
-
-    insert into {{ target }} ({{ insert_cols_csv }})
-    select {% for column in insert_cols -%}
-        DBT_INTERNAL_SOURCE.{{ column }} {%- if not loop.last %}, {%- endif %}
-    {%- endfor %}
-    from {{ source }} as DBT_INTERNAL_SOURCE
-    where DBT_INTERNAL_SOURCE.dbt_change_type::text = 'insert'::text;
-{% endmacro %}

plugins/postgres/dbt/include/postgres/macros/relations.sql

@@ -1,76 +0,0 @@
-{% macro postgres_get_relations () -%}
-
-  {#
-      -- in pg_depend, objid is the dependent, refobjid is the referenced object
-      --  > a pg_depend entry indicates that the referenced object cannot be
-      --  > dropped without also dropping the dependent object.
-  #}
-
-  {%- call statement('relations', fetch_result=True) -%}
-    with relation as (
-        select
-            pg_rewrite.ev_class as class,
-            pg_rewrite.oid as id
-        from pg_rewrite
-    ),
-    class as (
-        select
-            oid as id,
-            relname as name,
-            relnamespace as schema,
-            relkind as kind
-        from pg_class
-    ),
-    dependency as (
-        select
-            pg_depend.objid as id,
-            pg_depend.refobjid as ref
-        from pg_depend
-    ),
-    schema as (
-        select
-            pg_namespace.oid as id,
-            pg_namespace.nspname as name
-        from pg_namespace
-        where nspname != 'information_schema' and nspname not like 'pg\_%'
-    ),
-    referenced as (
-        select
-            relation.id AS id,
-            referenced_class.name ,
-            referenced_class.schema ,
-            referenced_class.kind
-        from relation
-        join class as referenced_class on relation.class=referenced_class.id
-        where referenced_class.kind in ('r', 'v')
-    ),
-    relationships as (
-        select
-            referenced.name as referenced_name,
-            referenced.schema as referenced_schema_id,
-            dependent_class.name as dependent_name,
-            dependent_class.schema as dependent_schema_id,
-            referenced.kind as kind
-        from referenced
-        join dependency on referenced.id=dependency.id
-        join class as dependent_class on dependency.ref=dependent_class.id
-        where
-            (referenced.name != dependent_class.name or
-             referenced.schema != dependent_class.schema)
-    )
-
-    select
-        referenced_schema.name as referenced_schema,
-        relationships.referenced_name as referenced_name,
-        dependent_schema.name as dependent_schema,
-        relationships.dependent_name as dependent_name
-    from relationships
-    join schema as dependent_schema on relationships.dependent_schema_id=dependent_schema.id
-    join schema as referenced_schema on relationships.referenced_schema_id=referenced_schema.id
-    group by referenced_schema, referenced_name, dependent_schema, dependent_name
-    order by referenced_schema, referenced_name, dependent_schema, dependent_name;
-
-  {%- endcall -%}
-
-  {{ return(load_result('relations').table) }}
-{% endmacro %}

plugins/postgres/dbt/include/postgres/sample_profiles.yml

@@ -1,24 +0,0 @@
-default:
-  outputs:
-
-    dev:
-      type: postgres
-      threads: [1 or more]
-      host: [host]
-      port: [port]
-      user: [dev_username]
-      pass: [dev_password]
-      dbname: [dbname]
-      schema: [dev_schema]
-
-    prod:
-      type: postgres
-      threads: [1 or more]
-      host: [host]
-      port: [port]
-      user: [prod_username]
-      pass: [prod_password]
-      dbname: [dbname]
-      schema: [prod_schema]
-
-  target: dev

plugins/postgres/setup.py

@@ -1,91 +0,0 @@
-#!/usr/bin/env python
-import os
-import sys
-
-if sys.version_info < (3, 6):
-    print('Error: dbt does not support this version of Python.')
-    print('Please upgrade to Python 3.6 or higher.')
-    sys.exit(1)
-
-
-from setuptools import setup
-try:
-    from setuptools import find_namespace_packages
-except ImportError:
-    # the user has a downlevel version of setuptools.
-    print('Error: dbt requires setuptools v40.1.0 or higher.')
-    print('Please upgrade setuptools with "pip install --upgrade setuptools" '
-          'and try again')
-    sys.exit(1)
-
-
-PSYCOPG2_MESSAGE = '''
-No package name override was set.
-Using 'psycopg2-binary' package to satisfy 'psycopg2'
-
-If you experience segmentation faults, silent crashes, or installation errors,
-consider retrying with the 'DBT_PSYCOPG2_NAME' environment variable set to
-'psycopg2'. It may require a compiler toolchain and development libraries!
-'''.strip()
-
-
-def _dbt_psycopg2_name():
-    # if the user chose something, use that
-    package_name = os.getenv('DBT_PSYCOPG2_NAME', '')
-    if package_name:
-        return package_name
-
-    # default to psycopg2-binary for all OSes/versions
-    print(PSYCOPG2_MESSAGE)
-    return 'psycopg2-binary'
-
-
-package_name = "dbt-postgres"
-package_version = "0.21.0rc1"
-description = """The postgres adpter plugin for dbt (data build tool)"""
-
-this_directory = os.path.abspath(os.path.dirname(__file__))
-with open(os.path.join(this_directory, 'README.md')) as f:
-    long_description = f.read()
-
-DBT_PSYCOPG2_NAME = _dbt_psycopg2_name()
-
-setup(
-    name=package_name,
-    version=package_version,
-    description=description,
-    long_description=description,
-    long_description_content_type='text/markdown',
-    author="dbt Labs",
-    author_email="info@dbtlabs.com",
-    url="https://github.com/dbt-labs/dbt",
-    packages=find_namespace_packages(include=['dbt', 'dbt.*']),
-    package_data={
-        'dbt': [
-            'include/postgres/dbt_project.yml',
-            'include/postgres/sample_profiles.yml',
-            'include/postgres/macros/*.sql',
-            'include/postgres/macros/**/*.sql',
-        ]
-    },
-    install_requires=[
-        'dbt-core=={}'.format(package_version),
-        '{}~=2.8'.format(DBT_PSYCOPG2_NAME),
-    ],
-    zip_safe=False,
-    classifiers=[
-        'Development Status :: 5 - Production/Stable',
-
-        'License :: OSI Approved :: Apache Software License',
-
-        'Operating System :: Microsoft :: Windows',
-        'Operating System :: MacOS :: MacOS X',
-        'Operating System :: POSIX :: Linux',
-
-        'Programming Language :: Python :: 3.6',
-        'Programming Language :: Python :: 3.7',
-        'Programming Language :: Python :: 3.8',
-        'Programming Language :: Python :: 3.9',
-    ],
-    python_requires=">=3.6.2",
-)

plugins/redshift/README.md

@@ -1,32 +0,0 @@
-<p align="center">
-  <img src="https://raw.githubusercontent.com/dbt-labs/dbt/6c6649f9129d5d108aa3b0526f634cd8f3a9d1ed/etc/dbt-logo-full.svg" alt="dbt logo" width="500"/>
-</p>
-
-**[dbt](https://www.getdbt.com/)** (data build tool) enables data analysts and engineers to transform their data using the same practices that software engineers use to build applications.
-
-dbt is the T in ELT. Organize, cleanse, denormalize, filter, rename, and pre-aggregate the raw data in your warehouse so that it's ready for analysis.
-
-## dbt-redshift
-
-The `dbt-redshift` package contains all of the code required to make dbt operate on a Redshift database. For
-more information on using dbt with Redshift, consult [the docs](https://docs.getdbt.com/docs/profile-redshift).
-
-
-## Find out more
-
--   Check out the [Introduction to dbt](https://docs.getdbt.com/docs/introduction/).
--   Read the [dbt Viewpoint](https://docs.getdbt.com/docs/about/viewpoint/).
-
-## Join thousands of analysts in the dbt community
-
--   Join the [chat](http://community.getdbt.com/) on Slack.
--   Find community posts on [dbt Discourse](https://discourse.getdbt.com).
-
-## Reporting bugs and contributing code
-
--   Want to report a bug or request a feature? Let us know on [Slack](http://community.getdbt.com/), or open [an issue](https://github.com/dbt-labs/dbt/issues/new).
--   Want to help us build dbt? Check out the [Contributing Getting Started Guide](https://github.com/dbt-labs/dbt/blob/HEAD/CONTRIBUTING.md)
-
-## Code of Conduct
-
-Everyone interacting in the dbt project's codebases, issue trackers, chat rooms, and mailing lists is expected to follow the [dbt Code of Conduct](https://community.getdbt.com/code-of-conduct).

plugins/redshift/dbt/adapters/redshift/__init__.py

@@ -1,15 +0,0 @@
-from dbt.adapters.redshift.connections import RedshiftConnectionManager  # noqa
-from dbt.adapters.redshift.connections import RedshiftCredentials
-from dbt.adapters.redshift.relation import RedshiftColumn  # noqa
-from dbt.adapters.redshift.relation import RedshiftRelation  # noqa: F401
-from dbt.adapters.redshift.impl import RedshiftAdapter
-
-
-from dbt.adapters.base import AdapterPlugin
-from dbt.include import redshift
-
-Plugin = AdapterPlugin(
-    adapter=RedshiftAdapter,
-    credentials=RedshiftCredentials,
-    include_path=redshift.PACKAGE_PATH,
-    dependencies=['postgres'])

plugins/redshift/dbt/adapters/redshift/__version__.py

@@ -1 +0,0 @@
-version = '0.21.0rc1'

plugins/redshift/dbt/adapters/redshift/connections.py

@@ -1,174 +0,0 @@
-from multiprocessing import Lock
-from contextlib import contextmanager
-from typing import NewType
-
-from dbt.adapters.postgres import PostgresConnectionManager
-from dbt.adapters.postgres import PostgresCredentials
-from dbt.logger import GLOBAL_LOGGER as logger  # noqa
-import dbt.exceptions
-import dbt.flags
-
-import boto3
-
-from dbt.dataclass_schema import FieldEncoder, dbtClassMixin, StrEnum
-
-from dataclasses import dataclass, field
-from typing import Optional, List
-
-drop_lock: Lock = dbt.flags.MP_CONTEXT.Lock()
-
-
-IAMDuration = NewType('IAMDuration', int)
-
-
-class IAMDurationEncoder(FieldEncoder):
-    @property
-    def json_schema(self):
-        return {'type': 'integer', 'minimum': 0, 'maximum': 65535}
-
-
-dbtClassMixin.register_field_encoders({IAMDuration: IAMDurationEncoder()})
-
-
-class RedshiftConnectionMethod(StrEnum):
-    DATABASE = 'database'
-    IAM = 'iam'
-
-
-@dataclass
-class RedshiftCredentials(PostgresCredentials):
-    method: RedshiftConnectionMethod = RedshiftConnectionMethod.DATABASE
-    password: Optional[str] = None
-    cluster_id: Optional[str] = field(
-        default=None,
-        metadata={'description': 'If using IAM auth, the name of the cluster'},
-    )
-    iam_profile: Optional[str] = None
-    iam_duration_seconds: int = 900
-    search_path: Optional[str] = None
-    keepalives_idle: int = 240
-    autocreate: bool = False
-    db_groups: List[str] = field(default_factory=list)
-    ra3_node: Optional[bool] = False
-
-    @property
-    def type(self):
-        return 'redshift'
-
-    def _connection_keys(self):
-        keys = super()._connection_keys()
-        return keys + (
-            'method',
-            'cluster_id',
-            'iam_profile',
-            'iam_duration_seconds'
-        )
-
-
-class RedshiftConnectionManager(PostgresConnectionManager):
-    TYPE = 'redshift'
-
-    @contextmanager
-    def fresh_transaction(self, name=None):
-        """On entrance to this context manager, hold an exclusive lock and
-        create a fresh transaction for redshift, then commit and begin a new
-        one before releasing the lock on exit.
-
-        See drop_relation in RedshiftAdapter for more information.
-
-        :param Optional[str] name: The name of the connection to use, or None
-            to use the default.
-        """
-        with drop_lock:
-            connection = self.get_thread_connection()
-
-            if connection.transaction_open:
-                self.commit()
-
-            self.begin()
-            yield
-
-            self.commit()
-            self.begin()
-
-    @classmethod
-    def fetch_cluster_credentials(cls, db_user, db_name, cluster_id,
-                                  iam_profile, duration_s, autocreate,
-                                  db_groups):
-        """Fetches temporary login credentials from AWS. The specified user
-        must already exist in the database, or else an error will occur"""
-
-        if iam_profile is None:
-            session = boto3.Session()
-            boto_client = session.client("redshift")
-        else:
-            logger.debug("Connecting to Redshift using 'IAM'" +
-                         f"with profile {iam_profile}")
-            boto_session = boto3.Session(
-                profile_name=iam_profile
-            )
-            boto_client = boto_session.client('redshift')
-
-        try:
-            return boto_client.get_cluster_credentials(
-                DbUser=db_user,
-                DbName=db_name,
-                ClusterIdentifier=cluster_id,
-                DurationSeconds=duration_s,
-                AutoCreate=autocreate,
-                DbGroups=db_groups,)
-
-        except boto_client.exceptions.ClientError as e:
-            raise dbt.exceptions.FailedToConnectException(
-                "Unable to get temporary Redshift cluster credentials: {}"
-                .format(e))
-
-    @classmethod
-    def get_tmp_iam_cluster_credentials(cls, credentials):
-        cluster_id = credentials.cluster_id
-
-        # default via:
-        # boto3.readthedocs.io/en/latest/reference/services/redshift.html
-        iam_duration_s = credentials.iam_duration_seconds
-
-        if not cluster_id:
-            raise dbt.exceptions.FailedToConnectException(
-                "'cluster_id' must be provided in profile if IAM "
-                "authentication method selected")
-
-        cluster_creds = cls.fetch_cluster_credentials(
-            credentials.user,
-            credentials.database,
-            credentials.cluster_id,
-            credentials.iam_profile,
-            iam_duration_s,
-            credentials.autocreate,
-            credentials.db_groups,
-        )
-
-        # replace username and password with temporary redshift credentials
-        return credentials.replace(user=cluster_creds.get('DbUser'),
-                                   password=cluster_creds.get('DbPassword'))
-
-    @classmethod
-    def get_credentials(cls, credentials):
-        method = credentials.method
-
-        # Support missing 'method' for backwards compatibility
-        if method == 'database' or method is None:
-            logger.debug("Connecting to Redshift using 'database' credentials")
-            # this requirement is really annoying to encode into json schema,
-            # so validate it here
-            if credentials.password is None:
-                raise dbt.exceptions.FailedToConnectException(
-                    "'password' field is required for 'database' credentials"
-                )
-            return credentials
-
-        elif method == 'iam':
-            logger.debug("Connecting to Redshift using 'IAM' credentials")
-            return cls.get_tmp_iam_cluster_credentials(credentials)
-
-        else:
-            raise dbt.exceptions.FailedToConnectException(
-                "Invalid 'method' in profile: '{}'".format(method))

plugins/redshift/dbt/adapters/redshift/impl.py

@@ -1,88 +0,0 @@
-from dataclasses import dataclass
-from typing import Optional
-from dbt.adapters.base.impl import AdapterConfig
-from dbt.adapters.sql import SQLAdapter
-from dbt.adapters.base.meta import available
-from dbt.adapters.postgres import PostgresAdapter
-from dbt.adapters.redshift import RedshiftConnectionManager
-from dbt.adapters.redshift import RedshiftColumn
-from dbt.adapters.redshift import RedshiftRelation
-from dbt.logger import GLOBAL_LOGGER as logger  # noqa
-import dbt.exceptions
-
-
-@dataclass
-class RedshiftConfig(AdapterConfig):
-    sort_type: Optional[str] = None
-    dist: Optional[str] = None
-    sort: Optional[str] = None
-    bind: Optional[bool] = None
-
-
-class RedshiftAdapter(PostgresAdapter, SQLAdapter):
-    Relation = RedshiftRelation
-    ConnectionManager = RedshiftConnectionManager
-    Column = RedshiftColumn
-
-    AdapterSpecificConfigs = RedshiftConfig
-
-    @classmethod
-    def date_function(cls):
-        return 'getdate()'
-
-    def drop_relation(self, relation):
-        """
-        In Redshift, DROP TABLE ... CASCADE should not be used
-        inside a transaction. Redshift doesn't prevent the CASCADE
-        part from conflicting with concurrent transactions. If we do
-        attempt to drop two tables with CASCADE at once, we'll often
-        get the dreaded:
-
-            table was dropped by a concurrent transaction
-
-        So, we need to lock around calls to the underlying
-        drop_relation() function.
-
-        https://docs.aws.amazon.com/redshift/latest/dg/r_DROP_TABLE.html
-        """
-        with self.connections.fresh_transaction():
-            return super().drop_relation(relation)
-
-    @classmethod
-    def convert_text_type(cls, agate_table, col_idx):
-        column = agate_table.columns[col_idx]
-        # `lens` must be a list, so this can't be a generator expression,
-        # because max() raises ane exception if its argument has no members.
-        lens = [len(d.encode("utf-8")) for d in column.values_without_nulls()]
-        max_len = max(lens) if lens else 64
-        return "varchar({})".format(max_len)
-
-    @classmethod
-    def convert_time_type(cls, agate_table, col_idx):
-        return "varchar(24)"
-
-    @available
-    def verify_database(self, database):
-        if database.startswith('"'):
-            database = database.strip('"')
-        expected = self.config.credentials.database
-        ra3_node = self.config.credentials.ra3_node
-
-        if database.lower() != expected.lower() and not ra3_node:
-            raise dbt.exceptions.NotImplementedException(
-                'Cross-db references allowed only in RA3.* node. ({} vs {})'
-                .format(database, expected)
-            )
-        # return an empty string on success so macros can call this
-        return ''
-
-    def _get_catalog_schemas(self, manifest):
-        # redshift(besides ra3) only allow one database (the main one)
-        schemas = super(SQLAdapter, self)._get_catalog_schemas(manifest)
-        try:
-            return schemas.flatten(allow_multiple_databases=self.config.credentials.ra3_node)
-        except dbt.exceptions.RuntimeException as exc:
-            dbt.exceptions.raise_compiler_error(
-                'Cross-db references allowed only in {} RA3.* node. Got {}'
-                .format(self.type(), exc.msg)
-            )

plugins/redshift/dbt/adapters/redshift/relation.py

@@ -1,15 +0,0 @@
-from dbt.adapters.base import Column
-from dataclasses import dataclass
-from dbt.adapters.postgres.relation import PostgresRelation
-
-
-@dataclass(frozen=True, eq=False, repr=False)
-class RedshiftRelation(PostgresRelation):
-    # Override the method in the Postgres Relation
-    # because Redshift allows longer names
-    def relation_max_name_length(self):
-        return 127
-
-
-class RedshiftColumn(Column):
-    pass  # redshift does not inherit from postgres here

plugins/redshift/dbt/include/redshift/__init__.py

@@ -1,3 +0,0 @@
-import os
-
-PACKAGE_PATH = os.path.dirname(__file__)

plugins/redshift/dbt/include/redshift/dbt_project.yml

@@ -1,5 +0,0 @@
-config-version: 2
-name: dbt_redshift
-version: 1.0
-
-macro-paths: ["macros"]

plugins/redshift/dbt/include/redshift/macros/adapters.sql

@@ -1,283 +0,0 @@
-
-{% macro dist(dist) %}
-  {%- if dist is not none -%}
-      {%- set dist = dist.strip().lower() -%}
-
-      {%- if dist in ['all', 'even'] -%}
-        diststyle {{ dist }}
-      {%- elif dist == "auto" -%}
-      {%- else -%}
-        diststyle key distkey ({{ dist }})
-      {%- endif -%}
-
-  {%- endif -%}
-{%- endmacro -%}
-
-
-{% macro sort(sort_type, sort) %}
-  {%- if sort is not none %}
-      {{ sort_type | default('compound', boolean=true) }} sortkey(
-      {%- if sort is string -%}
-        {%- set sort = [sort] -%}
-      {%- endif -%}
-      {%- for item in sort -%}
-        {{ item }}
-        {%- if not loop.last -%},{%- endif -%}
-      {%- endfor -%}
-      )
-  {%- endif %}
-{%- endmacro -%}
-
-
-{% macro redshift__create_table_as(temporary, relation, sql) -%}
-
-  {%- set _dist = config.get('dist') -%}
-  {%- set _sort_type = config.get(
-          'sort_type',
-          validator=validation.any['compound', 'interleaved']) -%}
-  {%- set _sort = config.get(
-          'sort',
-          validator=validation.any[list, basestring]) -%}
-  {%- set sql_header = config.get('sql_header', none) -%}
-
-  {{ sql_header if sql_header is not none }}
-
-  create {% if temporary -%}temporary{%- endif %} table
-    {{ relation.include(database=(not temporary), schema=(not temporary)) }}
-    {{ dist(_dist) }}
-    {{ sort(_sort_type, _sort) }}
-  as (
-    {{ sql }}
-  );
-{%- endmacro %}
-
-
-{% macro redshift__create_view_as(relation, sql) -%}
-  {%- set binding = config.get('bind', default=True) -%}
-
-  {% set bind_qualifier = '' if binding else 'with no schema binding' %}
-  {%- set sql_header = config.get('sql_header', none) -%}
-
-  {{ sql_header if sql_header is not none }}
-
-  create view {{ relation }} as (
-    {{ sql }}
-  ) {{ bind_qualifier }};
-{% endmacro %}
-
-
-{% macro redshift__create_schema(relation) -%}
-  {{ postgres__create_schema(relation) }}
-{% endmacro %}
-
-
-{% macro redshift__drop_schema(relation) -%}
-  {{ postgres__drop_schema(relation) }}
-{% endmacro %}
-
-
-{% macro redshift__get_columns_in_relation(relation) -%}
-  {% call statement('get_columns_in_relation', fetch_result=True) %}
-      with bound_views as (
-        select
-          ordinal_position,
-          table_schema,
-          column_name,
-          data_type,
-          character_maximum_length,
-          numeric_precision,
-          numeric_scale
-
-        from information_schema."columns"
-        where table_name = '{{ relation.identifier }}'
-    ),
-
-    unbound_views as (
-      select
-        ordinal_position,
-        view_schema,
-        col_name,
-        case
-          when col_type ilike 'character varying%' then
-            'character varying'
-          when col_type ilike 'numeric%' then 'numeric'
-          else col_type
-        end as col_type,
-        case
-          when col_type like 'character%'
-          then nullif(REGEXP_SUBSTR(col_type, '[0-9]+'), '')::int
-          else null
-        end as character_maximum_length,
-        case
-          when col_type like 'numeric%'
-          then nullif(
-            SPLIT_PART(REGEXP_SUBSTR(col_type, '[0-9,]+'), ',', 1),
-            '')::int
-          else null
-        end as numeric_precision,
-        case
-          when col_type like 'numeric%'
-          then nullif(
-            SPLIT_PART(REGEXP_SUBSTR(col_type, '[0-9,]+'), ',', 2),
-            '')::int
-          else null
-        end as numeric_scale
-
-      from pg_get_late_binding_view_cols()
-      cols(view_schema name, view_name name, col_name name,
-           col_type varchar, ordinal_position int)
-      where view_name = '{{ relation.identifier }}'
-    ),
-
-    external_views as (
-      select
-        columnnum,
-        schemaname,
-        columnname,
-        case
-          when external_type ilike 'character varying%' or external_type ilike 'varchar%'
-          then 'character varying'
-          when external_type ilike 'numeric%' then 'numeric'
-          else external_type
-        end as external_type,
-        case
-          when external_type like 'character%' or external_type like 'varchar%'
-          then nullif(
-            REGEXP_SUBSTR(external_type, '[0-9]+'),
-            '')::int
-          else null
-        end as character_maximum_length,
-        case
-          when external_type like 'numeric%'
-          then nullif(
-            SPLIT_PART(REGEXP_SUBSTR(external_type, '[0-9,]+'), ',', 1),
-            '')::int
-          else null
-        end as numeric_precision,
-        case
-          when external_type like 'numeric%'
-          then nullif(
-            SPLIT_PART(REGEXP_SUBSTR(external_type, '[0-9,]+'), ',', 2),
-            '')::int
-          else null
-        end as numeric_scale
-      from
-        pg_catalog.svv_external_columns
-      where
-        schemaname = '{{ relation.schema }}'
-        and tablename = '{{ relation.identifier }}'
-
-    ),
-
-    unioned as (
-      select * from bound_views
-      union all
-      select * from unbound_views
-      union all
-      select * from external_views
-    )
-
-    select
-      column_name,
-      data_type,
-      character_maximum_length,
-      numeric_precision,
-      numeric_scale
-
-    from unioned
-    {% if relation.schema %}
-    where table_schema = '{{ relation.schema }}'
-    {% endif %}
-    order by ordinal_position
-  {% endcall %}
-  {% set table = load_result('get_columns_in_relation').table %}
-  {{ return(sql_convert_columns_in_relation(table)) }}
-{% endmacro %}
-
-
-{% macro redshift__list_relations_without_caching(schema_relation) %}
-  {{ return(postgres__list_relations_without_caching(schema_relation)) }}
-{% endmacro %}
-
-
-{% macro redshift__information_schema_name(database) -%}
-  {{ return(postgres__information_schema_name(database)) }}
-{%- endmacro %}
-
-
-{% macro redshift__list_schemas(database) -%}
-  {{ return(postgres__list_schemas(database)) }}
-{%- endmacro %}
-
-
-{% macro redshift__check_schema_exists(information_schema, schema) -%}
-  {{ return(postgres__check_schema_exists(information_schema, schema)) }}
-{%- endmacro %}
-
-{% macro redshift__current_timestamp() -%}
-  getdate()
-{%- endmacro %}
-
-{% macro redshift__snapshot_get_time() -%}
-  {{ current_timestamp() }}::timestamp
-{%- endmacro %}
-
-
-{% macro redshift__snapshot_string_as_time(timestamp) -%}
-    {%- set result = "'" ~ timestamp ~ "'::timestamp" -%}
-    {{ return(result) }}
-{%- endmacro %}
-
-{% macro redshift__make_temp_relation(base_relation, suffix) %}
-    {% do return(postgres__make_temp_relation(base_relation, suffix)) %}
-{% endmacro %}
-
-
-{% macro redshift__persist_docs(relation, model, for_relation, for_columns) -%}
-  {% if for_relation and config.persist_relation_docs() and model.description %}
-    {% do run_query(alter_relation_comment(relation, model.description)) %}
-  {% endif %}
-
-  {# Override: do not set column comments for LBVs #}
-  {% set is_lbv = config.get('materialized') == 'view' and config.get('bind') == false %}
-  {% if for_columns and config.persist_column_docs() and model.columns and not is_lbv %}
-    {% do run_query(alter_column_comment(relation, model.columns)) %}
-  {% endif %}
-{% endmacro %}
-
-
-{% macro redshift__alter_relation_comment(relation, comment) %}
-  {% do return(postgres__alter_relation_comment(relation, comment)) %}
-{% endmacro %}
-
-
-{% macro redshift__alter_column_comment(relation, column_dict) %}
-  {% do return(postgres__alter_column_comment(relation, column_dict)) %}
-{% endmacro %}
-
-
-{% macro redshift__alter_relation_add_remove_columns(relation, add_columns, remove_columns) %}
-  
-  {% if add_columns %}
-  
-    {% for column in add_columns %}
-      {% set sql -%}
-          alter {{ relation.type }} {{ relation }} add column {{ column.name }} {{ column.data_type }}
-      {% endset %}
-      {% do run_query(sql) %}
-    {% endfor %}
-
-  {% endif %}
-  
-  {% if remove_columns %}
-  
-    {% for column in remove_columns %}
-      {% set sql -%}
-          alter {{ relation.type }} {{ relation }} drop column {{ column.name }}
-      {% endset %}
-      {% do run_query(sql) %}
-    {% endfor %}
-    
-  {% endif %}
-
-{% endmacro %}

plugins/redshift/dbt/include/redshift/macros/catalog.sql

@@ -1,242 +0,0 @@
-
-{% macro redshift__get_base_catalog(information_schema, schemas) -%}
-  {%- call statement('base_catalog', fetch_result=True) -%}
-    {% set database = information_schema.database %}
-    {{ adapter.verify_database(database) }}
-
-    with late_binding as (
-      select
-        '{{ database }}'::varchar as table_database,
-        table_schema,
-        table_name,
-        'LATE BINDING VIEW'::varchar as table_type,
-        null::text as table_comment,
-
-        column_name,
-        column_index,
-        column_type,
-        null::text as column_comment
-      from pg_get_late_binding_view_cols()
-        cols(table_schema name, table_name name, column_name name,
-             column_type varchar,
-             column_index int)
-        order by "column_index"
-    ),
-
-    early_binding as (
-        select
-            '{{ database }}'::varchar as table_database,
-            sch.nspname as table_schema,
-            tbl.relname as table_name,
-            case tbl.relkind
-                when 'v' then 'VIEW'
-                else 'BASE TABLE'
-            end as table_type,
-            tbl_desc.description as table_comment,
-            col.attname as column_name,
-            col.attnum as column_index,
-            pg_catalog.format_type(col.atttypid, col.atttypmod) as column_type,
-            col_desc.description as column_comment
-
-        from pg_catalog.pg_namespace sch
-        join pg_catalog.pg_class tbl on tbl.relnamespace = sch.oid
-        join pg_catalog.pg_attribute col on col.attrelid = tbl.oid
-        left outer join pg_catalog.pg_description tbl_desc on (tbl_desc.objoid = tbl.oid and tbl_desc.objsubid = 0)
-        left outer join pg_catalog.pg_description col_desc on (col_desc.objoid = tbl.oid and col_desc.objsubid = col.attnum)
-        where (
-            {%- for schema in schemas -%}
-              upper(sch.nspname) = upper('{{ schema }}'){%- if not loop.last %} or {% endif -%}
-            {%- endfor -%}
-          )
-            and tbl.relkind in ('r', 'v', 'f', 'p')
-            and col.attnum > 0
-            and not col.attisdropped
-    ),
-
-    table_owners as (
-
-        select
-            '{{ database }}'::varchar as table_database,
-            schemaname as table_schema,
-            tablename as table_name,
-            tableowner as table_owner
-
-        from pg_tables
-
-        union all
-
-        select
-            '{{ database }}'::varchar as table_database,
-            schemaname as table_schema,
-            viewname as table_name,
-            viewowner as table_owner
-
-        from pg_views
-
-    ),
-
-    unioned as (
-
-        select *
-        from early_binding
-
-        union all
-
-        select *
-        from late_binding
-
-    )
-
-    select *,
-        table_database || '.' || table_schema || '.' || table_name as table_id
-
-    from unioned
-    join table_owners using (table_database, table_schema, table_name)
-
-    where (
-        {%- for schema in schemas -%}
-          upper(table_schema) = upper('{{ schema }}'){%- if not loop.last %} or {% endif -%}
-        {%- endfor -%}
-      )
-
-    order by "column_index"
-  {%- endcall -%}
-
-  {{ return(load_result('base_catalog').table) }}
-{%- endmacro %}
-
-{% macro redshift__get_extended_catalog(schemas) %}
-  {%- call statement('extended_catalog', fetch_result=True) -%}
-
-    select
-        "database" || '.' || "schema" || '.' || "table" as table_id,
-
-        'Encoded'::text as "stats:encoded:label",
-        encoded as "stats:encoded:value",
-        'Indicates whether any column in the table has compression encoding defined.'::text as "stats:encoded:description",
-        true as "stats:encoded:include",
-
-        'Dist Style' as "stats:diststyle:label",
-        diststyle as "stats:diststyle:value",
-        'Distribution style or distribution key column, if key distribution is defined.'::text as "stats:diststyle:description",
-        true as "stats:diststyle:include",
-
-        'Sort Key 1' as "stats:sortkey1:label",
-        -- handle 0xFF byte in response for interleaved sort styles
-        case
-            when sortkey1 like 'INTERLEAVED%' then 'INTERLEAVED'::text
-            else sortkey1
-        end as "stats:sortkey1:value",
-        'First column in the sort key.'::text as "stats:sortkey1:description",
-        (sortkey1 is not null) as "stats:sortkey1:include",
-
-        'Max Varchar' as "stats:max_varchar:label",
-        max_varchar as "stats:max_varchar:value",
-        'Size of the largest column that uses a VARCHAR data type.'::text as "stats:max_varchar:description",
-        true as "stats:max_varchar:include",
-
-        -- exclude this, as the data is strangely returned with null-byte characters
-        'Sort Key 1 Encoding' as "stats:sortkey1_enc:label",
-        sortkey1_enc as "stats:sortkey1_enc:value",
-        'Compression encoding of the first column in the sort key.' as "stats:sortkey1_enc:description",
-        false as "stats:sortkey1_enc:include",
-
-        '# Sort Keys' as "stats:sortkey_num:label",
-        sortkey_num as "stats:sortkey_num:value",
-        'Number of columns defined as sort keys.' as "stats:sortkey_num:description",
-        (sortkey_num > 0) as "stats:sortkey_num:include",
-
-        'Approximate Size' as "stats:size:label",
-        size * 1000000 as "stats:size:value",
-        'Approximate size of the table, calculated from a count of 1MB blocks'::text as "stats:size:description",
-        true as "stats:size:include",
-
-        'Disk Utilization' as "stats:pct_used:label",
-        pct_used / 100.0 as "stats:pct_used:value",
-        'Percent of available space that is used by the table.'::text as "stats:pct_used:description",
-        true as "stats:pct_used:include",
-
-        'Unsorted %' as "stats:unsorted:label",
-        unsorted / 100.0 as "stats:unsorted:value",
-        'Percent of unsorted rows in the table.'::text as "stats:unsorted:description",
-        (unsorted is not null) as "stats:unsorted:include",
-
-        'Stats Off' as "stats:stats_off:label",
-        stats_off as "stats:stats_off:value",
-        'Number that indicates how stale the table statistics are; 0 is current, 100 is out of date.'::text as "stats:stats_off:description",
-        true as "stats:stats_off:include",
-
-        'Approximate Row Count' as "stats:rows:label",
-        tbl_rows as "stats:rows:value",
-        'Approximate number of rows in the table. This value includes rows marked for deletion, but not yet vacuumed.'::text as "stats:rows:description",
-        true as "stats:rows:include",
-
-        'Sort Key Skew' as "stats:skew_sortkey1:label",
-        skew_sortkey1 as "stats:skew_sortkey1:value",
-        'Ratio of the size of the largest non-sort key column to the size of the first column of the sort key.'::text as "stats:skew_sortkey1:description",
-        (skew_sortkey1 is not null) as "stats:skew_sortkey1:include",
-
-        'Skew Rows' as "stats:skew_rows:label",
-        skew_rows as "stats:skew_rows:value",
-        'Ratio of the number of rows in the slice with the most rows to the number of rows in the slice with the fewest rows.'::text as "stats:skew_rows:description",
-        (skew_rows is not null) as "stats:skew_rows:include"
-
-    from svv_table_info
-    where (
-        {%- for schema in schemas -%}
-          upper(schema) = upper('{{ schema }}'){%- if not loop.last %} or {% endif -%}
-        {%- endfor -%}
-    )
-
-  {%- endcall -%}
-
-  {{ return(load_result('extended_catalog').table) }}
-
-{% endmacro %}
-
-{% macro redshift__can_select_from(table_name) %}
-
-  {%- call statement('has_table_privilege', fetch_result=True) -%}
-
-    select has_table_privilege(current_user, '{{ table_name }}', 'SELECT') as can_select
-
-  {%- endcall -%}
-
-  {% set can_select = load_result('has_table_privilege').table[0]['can_select'] %}
-  {{ return(can_select) }}
-
-{% endmacro %}
-
-{% macro redshift__no_svv_table_info_warning() %}
-
-    {% set msg %}
-
-    Warning: The database user "{{ target.user }}" has insufficient permissions to
-    query the "svv_table_info" table. Please grant SELECT permissions on this table
-    to the "{{ target.user }}" user to fetch extended table details from Redshift.
-
-    {% endset %}
-
-    {{ log(msg, info=True) }}
-
-{% endmacro %}
-
-
-{% macro redshift__get_catalog(information_schema, schemas) %}
-
-    {#-- Compute a left-outer join in memory. Some Redshift queries are
-      -- leader-only, and cannot be joined to other compute-based queries #}
-
-    {% set catalog = redshift__get_base_catalog(information_schema, schemas) %}
-
-    {% set select_extended =  redshift__can_select_from('svv_table_info') %}
-    {% if select_extended %}
-        {% set extended_catalog = redshift__get_extended_catalog(schemas) %}
-        {% set catalog = catalog.join(extended_catalog, 'table_id') %}
-    {% else %}
-        {{ redshift__no_svv_table_info_warning() }}
-    {% endif %}
-
-    {{ return(catalog.exclude(['table_id'])) }}
-
-{% endmacro %}

plugins/redshift/dbt/include/redshift/macros/materializations/snapshot_merge.sql

@@ -1,4 +0,0 @@
-
-{% macro redshift__snapshot_merge_sql(target, source, insert_cols) -%}
-    {{ postgres__snapshot_merge_sql(target, source, insert_cols) }}
-{% endmacro %}

plugins/redshift/dbt/include/redshift/macros/relations.sql

@@ -1,3 +0,0 @@
-{% macro redshift__get_relations () -%}
-  {{ return(dbt.postgres__get_relations()) }}
-{% endmacro %}

plugins/redshift/dbt/include/redshift/sample_profiles.yml

@@ -1,25 +0,0 @@
-default:
-  outputs:
-
-    dev:
-      type: redshift
-      threads: [1 or more]
-      host: [host]
-      port: [port]
-      user: [dev_username]
-      pass: [dev_password]
-      dbname: [dbname]
-      schema: [dev_schema]
-
-    prod:
-      type: redshift
-      method: iam
-      cluster_id: [cluster_id]
-      threads: [1 or more]
-      host: [host]
-      port: [port]
-      user: [prod_user]
-      dbname: [dbname]
-      schema: [prod_schema]
-
-  target: dev

plugins/redshift/setup.py

@@ -1,70 +0,0 @@
-#!/usr/bin/env python
-import os
-import sys
-
-if sys.version_info < (3, 6):
-    print('Error: dbt does not support this version of Python.')
-    print('Please upgrade to Python 3.6 or higher.')
-    sys.exit(1)
-
-
-from setuptools import setup
-try:
-    from setuptools import find_namespace_packages
-except ImportError:
-    # the user has a downlevel version of setuptools.
-    print('Error: dbt requires setuptools v40.1.0 or higher.')
-    print('Please upgrade setuptools with "pip install --upgrade setuptools" '
-          'and try again')
-    sys.exit(1)
-
-
-package_name = "dbt-redshift"
-package_version = "0.21.0rc1"
-description = """The redshift adapter plugin for dbt (data build tool)"""
-
-this_directory = os.path.abspath(os.path.dirname(__file__))
-with open(os.path.join(this_directory, 'README.md')) as f:
-    long_description = f.read()
-
-setup(
-    name=package_name,
-    version=package_version,
-    description=description,
-    long_description=description,
-    long_description_content_type='text/markdown',
-    author="dbt Labs",
-    author_email="info@dbtlabs.com",
-    url="https://github.com/dbt-labs/dbt",
-    packages=find_namespace_packages(include=['dbt', 'dbt.*']),
-    package_data={
-        'dbt': [
-            'include/redshift/dbt_project.yml',
-            'include/redshift/sample_profiles.yml',
-            'include/redshift/macros/*.sql',
-            'include/redshift/macros/**/*.sql',
-        ]
-    },
-    install_requires=[
-        'dbt-core=={}'.format(package_version),
-        'dbt-postgres=={}'.format(package_version),
-        # the following are all to match snowflake-connector-python
-        'boto3>=1.4.4,<2.0.0',
-    ],
-    zip_safe=False,
-    classifiers=[
-        'Development Status :: 5 - Production/Stable',
-
-        'License :: OSI Approved :: Apache Software License',
-
-        'Operating System :: Microsoft :: Windows',
-        'Operating System :: MacOS :: MacOS X',
-        'Operating System :: POSIX :: Linux',
-
-        'Programming Language :: Python :: 3.6',
-        'Programming Language :: Python :: 3.7',
-        'Programming Language :: Python :: 3.8',
-        'Programming Language :: Python :: 3.9',
-    ],
-    python_requires=">=3.6.2",
-)

plugins/snowflake/README.md

@@ -1,32 +0,0 @@
-<p align="center">
-  <img src="https://raw.githubusercontent.com/dbt-labs/dbt/6c6649f9129d5d108aa3b0526f634cd8f3a9d1ed/etc/dbt-logo-full.svg" alt="dbt logo" width="500"/>
-</p>
-
-**[dbt](https://www.getdbt.com/)** (data build tool) enables data analysts and engineers to transform their data using the same practices that software engineers use to build applications.
-
-dbt is the T in ELT. Organize, cleanse, denormalize, filter, rename, and pre-aggregate the raw data in your warehouse so that it's ready for analysis.
-
-## dbt-snowflake
-
-The `dbt-snowflake` package contains all of the code required to make dbt operate on a Snowflake database. For
-more information on using dbt with Snowflake, consult [the docs](https://docs.getdbt.com/docs/profile-snowflake).
-
-
-## Find out more
-
--   Check out the [Introduction to dbt](https://docs.getdbt.com/docs/introduction/).
--   Read the [dbt Viewpoint](https://docs.getdbt.com/docs/about/viewpoint/).
-
-## Join thousands of analysts in the dbt community
-
--   Join the [chat](http://community.getdbt.com/) on Slack.
--   Find community posts on [dbt Discourse](https://discourse.getdbt.com).
-
-## Reporting bugs and contributing code
-
--   Want to report a bug or request a feature? Let us know on [Slack](http://community.getdbt.com/), or open [an issue](https://github.com/dbt-labs/dbt/issues/new).
--   Want to help us build dbt? Check out the [Contributing Getting Started Guide](https://github.com/dbt-labs/dbt/blob/HEAD/CONTRIBUTING.md)
-
-## Code of Conduct
-
-Everyone interacting in the dbt project's codebases, issue trackers, chat rooms, and mailing lists is expected to follow the [dbt Code of Conduct](https://community.getdbt.com/code-of-conduct).

plugins/snowflake/dbt/adapters/snowflake/__init__.py

@@ -1,13 +0,0 @@
-from dbt.adapters.snowflake.column import SnowflakeColumn  # noqa
-from dbt.adapters.snowflake.connections import SnowflakeConnectionManager  # noqa
-from dbt.adapters.snowflake.connections import SnowflakeCredentials
-from dbt.adapters.snowflake.relation import SnowflakeRelation  # noqa
-from dbt.adapters.snowflake.impl import SnowflakeAdapter
-
-from dbt.adapters.base import AdapterPlugin
-from dbt.include import snowflake
-
-Plugin = AdapterPlugin(
-    adapter=SnowflakeAdapter,
-    credentials=SnowflakeCredentials,
-    include_path=snowflake.PACKAGE_PATH)

plugins/snowflake/dbt/adapters/snowflake/__version__.py

@@ -1 +0,0 @@
-version = '0.21.0rc1'

plugins/snowflake/dbt/adapters/snowflake/column.py

@@ -1,31 +0,0 @@
-from dataclasses import dataclass
-
-from dbt.adapters.base.column import Column
-from dbt.exceptions import RuntimeException
-
-
-@dataclass
-class SnowflakeColumn(Column):
-    def is_integer(self) -> bool:
-        # everything that smells like an int is actually a NUMBER(38, 0)
-        return False
-
-    def is_numeric(self) -> bool:
-        return self.dtype.lower() in [
-            'int', 'integer', 'bigint', 'smallint', 'tinyint', 'byteint',
-            'numeric', 'decimal', 'number'
-        ]
-
-    def is_float(self):
-        return self.dtype.lower() in [
-            'float', 'float4', 'float8', 'double', 'double precision', 'real',
-        ]
-
-    def string_size(self) -> int:
-        if not self.is_string():
-            raise RuntimeException("Called string_size() on non-string field!")
-
-        if self.dtype == 'text' or self.char_size is None:
-            return 16777216
-        else:
-            return int(self.char_size)

plugins/snowflake/dbt/adapters/snowflake/connections.py

@@ -1,375 +0,0 @@
-import base64
-import datetime
-import pytz
-import re
-from contextlib import contextmanager
-from dataclasses import dataclass
-from io import StringIO
-from time import sleep
-from typing import Optional
-
-from cryptography.hazmat.backends import default_backend
-from cryptography.hazmat.primitives import serialization
-import requests
-import snowflake.connector
-import snowflake.connector.errors
-
-from dbt.exceptions import (
-    InternalException, RuntimeException, FailedToConnectException,
-    DatabaseException, warn_or_error
-)
-from dbt.adapters.base import Credentials
-from dbt.contracts.connection import AdapterResponse
-from dbt.adapters.sql import SQLConnectionManager
-from dbt.logger import GLOBAL_LOGGER as logger
-
-
-_TOKEN_REQUEST_URL = 'https://{}.snowflakecomputing.com/oauth/token-request'
-
-
-@dataclass
-class SnowflakeCredentials(Credentials):
-    account: str
-    user: str
-    warehouse: Optional[str] = None
-    role: Optional[str] = None
-    password: Optional[str] = None
-    authenticator: Optional[str] = None
-    private_key_path: Optional[str] = None
-    private_key_passphrase: Optional[str] = None
-    token: Optional[str] = None
-    oauth_client_id: Optional[str] = None
-    oauth_client_secret: Optional[str] = None
-    query_tag: Optional[str] = None
-    client_session_keep_alive: bool = False
-
-    def __post_init__(self):
-        if (
-            self.authenticator != 'oauth' and
-            (self.oauth_client_secret or self.oauth_client_id or self.token)
-        ):
-            # the user probably forgot to set 'authenticator' like I keep doing
-            warn_or_error(
-                'Authenticator is not set to oauth, but an oauth-only '
-                'parameter is set! Did you mean to set authenticator: oauth?'
-            )
-
-    @property
-    def type(self):
-        return 'snowflake'
-
-    @property
-    def unique_field(self):
-        return self.account
-
-    def _connection_keys(self):
-        return (
-            'account', 'user', 'database', 'schema', 'warehouse', 'role',
-            'client_session_keep_alive'
-        )
-
-    def auth_args(self):
-        # Pull all of the optional authentication args for the connector,
-        # let connector handle the actual arg validation
-        result = {}
-        if self.password:
-            result['password'] = self.password
-        if self.authenticator:
-            result['authenticator'] = self.authenticator
-            if self.authenticator == 'oauth':
-                token = self.token
-                # if we have a client ID/client secret, the token is a refresh
-                # token, not an access token
-                if self.oauth_client_id and self.oauth_client_secret:
-                    token = self._get_access_token()
-                elif self.oauth_client_id:
-                    warn_or_error(
-                        'Invalid profile: got an oauth_client_id, but not an '
-                        'oauth_client_secret!'
-                    )
-                elif self.oauth_client_secret:
-                    warn_or_error(
-                        'Invalid profile: got an oauth_client_secret, but not '
-                        'an oauth_client_id!'
-                    )
-
-                result['token'] = token
-            # enable the token cache
-            result['client_store_temporary_credential'] = True
-        result['private_key'] = self._get_private_key()
-        return result
-
-    def _get_access_token(self) -> str:
-        if self.authenticator != 'oauth':
-            raise InternalException('Can only get access tokens for oauth')
-        missing = any(
-            x is None for x in
-            (self.oauth_client_id, self.oauth_client_secret, self.token)
-        )
-        if missing:
-            raise InternalException(
-                'need a client ID a client secret, and a refresh token to get '
-                'an access token'
-            )
-
-        # should the full url be a config item?
-        token_url = _TOKEN_REQUEST_URL.format(self.account)
-        # I think this is only used to redirect on success, which we ignore
-        # (it does not have to match the integration's settings in snowflake)
-        redirect_uri = 'http://localhost:9999'
-        data = {
-            'grant_type': 'refresh_token',
-            'refresh_token': self.token,
-            'redirect_uri': redirect_uri
-        }
-
-        auth = base64.b64encode(
-            f'{self.oauth_client_id}:{self.oauth_client_secret}'
-            .encode('ascii')
-        ).decode('ascii')
-        headers = {
-            'Authorization': f'Basic {auth}',
-            'Content-type': 'application/x-www-form-urlencoded;charset=utf-8'
-        }
-
-        result_json = None
-        max_iter = 20
-        # Attempt to obtain JSON for 1 second before throwing an error
-        for i in range(max_iter):
-            result = requests.post(token_url, headers=headers, data=data)
-            try:
-                result_json = result.json()
-                break
-            except ValueError as e:
-                message = result.text
-                logger.debug(f"Got a non-json response ({result.status_code}): \
-                              {e}, message: {message}")
-                sleep(0.05)
-
-        if result_json is None:
-            raise DatabaseException(f"""Did not receive valid json with access_token.
-                                        Showing json response: {result_json}""")
-
-        return result_json['access_token']
-
-    def _get_private_key(self):
-        """Get Snowflake private key by path or None."""
-        if not self.private_key_path:
-            return None
-
-        if self.private_key_passphrase:
-            encoded_passphrase = self.private_key_passphrase.encode()
-        else:
-            encoded_passphrase = None
-
-        with open(self.private_key_path, 'rb') as key:
-            p_key = serialization.load_pem_private_key(
-                key.read(),
-                password=encoded_passphrase,
-                backend=default_backend())
-
-        return p_key.private_bytes(
-            encoding=serialization.Encoding.DER,
-            format=serialization.PrivateFormat.PKCS8,
-            encryption_algorithm=serialization.NoEncryption())
-
-
-class SnowflakeConnectionManager(SQLConnectionManager):
-    TYPE = 'snowflake'
-
-    @contextmanager
-    def exception_handler(self, sql):
-        try:
-            yield
-        except snowflake.connector.errors.ProgrammingError as e:
-            msg = str(e)
-
-            logger.debug('Snowflake query id: {}'.format(e.sfqid))
-            logger.debug('Snowflake error: {}'.format(msg))
-
-            if 'Empty SQL statement' in msg:
-                logger.debug("got empty sql statement, moving on")
-            elif 'This session does not have a current database' in msg:
-                raise FailedToConnectException(
-                    ('{}\n\nThis error sometimes occurs when invalid '
-                     'credentials are provided, or when your default role '
-                     'does not have access to use the specified database. '
-                     'Please double check your profile and try again.')
-                    .format(msg))
-            else:
-                raise DatabaseException(msg)
-        except Exception as e:
-            if isinstance(e, snowflake.connector.errors.Error):
-                logger.debug('Snowflake query id: {}'.format(e.sfqid))
-
-            logger.debug("Error running SQL: {}", sql)
-            logger.debug("Rolling back transaction.")
-            self.rollback_if_open()
-            if isinstance(e, RuntimeException):
-                # during a sql query, an internal to dbt exception was raised.
-                # this sounds a lot like a signal handler and probably has
-                # useful information, so raise it without modification.
-                raise
-            raise RuntimeException(str(e)) from e
-
-    @classmethod
-    def open(cls, connection):
-        if connection.state == 'open':
-            logger.debug('Connection is already open, skipping open.')
-            return connection
-
-        try:
-            creds = connection.credentials
-
-            handle = snowflake.connector.connect(
-                account=creds.account,
-                user=creds.user,
-                database=creds.database,
-                schema=creds.schema,
-                warehouse=creds.warehouse,
-                role=creds.role,
-                autocommit=True,
-                client_session_keep_alive=creds.client_session_keep_alive,
-                application='dbt',
-                **creds.auth_args()
-            )
-
-            if creds.query_tag:
-                handle.cursor().execute(
-                    ("alter session set query_tag = '{}'")
-                    .format(creds.query_tag))
-
-            connection.handle = handle
-            connection.state = 'open'
-        except snowflake.connector.errors.Error as e:
-            logger.debug("Got an error when attempting to open a snowflake "
-                         "connection: '{}'"
-                         .format(e))
-
-            connection.handle = None
-            connection.state = 'fail'
-
-            raise FailedToConnectException(str(e))
-
-    def cancel(self, connection):
-        handle = connection.handle
-        sid = handle.session_id
-
-        connection_name = connection.name
-
-        sql = 'select system$abort_session({})'.format(sid)
-
-        logger.debug("Cancelling query '{}' ({})".format(connection_name, sid))
-
-        _, cursor = self.add_query(sql)
-        res = cursor.fetchone()
-
-        logger.debug("Cancel query '{}': {}".format(connection_name, res))
-
-    @classmethod
-    def get_response(cls, cursor) -> AdapterResponse:
-        code = cursor.sqlstate
-
-        if code is None:
-            code = 'SUCCESS'
-
-        return AdapterResponse(
-            _message="{} {}".format(code, cursor.rowcount),
-            rows_affected=cursor.rowcount,
-            code=code
-        )
-
-    # disable transactional logic by default on Snowflake
-    # except for DML statements where explicitly defined
-    def add_begin_query(self, *args, **kwargs):
-        pass
-
-    def add_commit_query(self, *args, **kwargs):
-        pass
-
-    def begin(self):
-        pass
-
-    def commit(self):
-        pass
-
-    def clear_transaction(self):
-        pass
-
-    @classmethod
-    def _split_queries(cls, sql):
-        "Splits sql statements at semicolons into discrete queries"
-
-        sql_s = str(sql)
-        sql_buf = StringIO(sql_s)
-        split_query = snowflake.connector.util_text.split_statements(sql_buf)
-        return [part[0] for part in split_query]
-
-    @classmethod
-    def process_results(cls, column_names, rows):
-        # Override for Snowflake. The datetime objects returned by
-        # snowflake-connector-python are not pickleable, so we need
-        # to replace them with sane timezones
-        fixed = []
-        for row in rows:
-            fixed_row = []
-            for col in row:
-                if isinstance(col, datetime.datetime) and col.tzinfo:
-                    offset = col.utcoffset()
-                    offset_seconds = offset.total_seconds()
-                    new_timezone = pytz.FixedOffset(offset_seconds // 60)
-                    col = col.astimezone(tz=new_timezone)
-                fixed_row.append(col)
-
-            fixed.append(fixed_row)
-
-        return super().process_results(column_names, fixed)
-
-    def add_query(self, sql, auto_begin=True,
-                  bindings=None, abridge_sql_log=False):
-
-        connection = None
-        cursor = None
-
-        if bindings:
-            # The snowflake connector is more strict than, eg., psycopg2 -
-            # which allows any iterable thing to be passed as a binding.
-            bindings = tuple(bindings)
-
-        queries = self._split_queries(sql)
-
-        for individual_query in queries:
-            # hack -- after the last ';', remove comments and don't run
-            # empty queries. this avoids using exceptions as flow control,
-            # and also allows us to return the status of the last cursor
-            without_comments = re.sub(
-                re.compile(
-                    r'(\".*?\"|\'.*?\')|(/\*.*?\*/|--[^\r\n]*$)', re.MULTILINE
-                ),
-                '', individual_query).strip()
-
-            if without_comments == "":
-                continue
-
-            connection, cursor = super().add_query(
-                individual_query, auto_begin,
-                bindings=bindings,
-                abridge_sql_log=abridge_sql_log
-            )
-
-        if cursor is None:
-            conn = self.get_thread_connection()
-            if conn is None or conn.name is None:
-                conn_name = '<None>'
-            else:
-                conn_name = conn.name
-
-            raise RuntimeException(
-                "Tried to run an empty query on model '{}'. If you are "
-                "conditionally running\nsql, eg. in a model hook, make "
-                "sure your `else` clause contains valid sql!\n\n"
-                "Provided SQL:\n{}"
-                .format(conn_name, sql)
-            )
-
-        return connection, cursor

plugins/snowflake/dbt/adapters/snowflake/impl.py

@@ -1,190 +0,0 @@
-from dataclasses import dataclass
-from typing import Mapping, Any, Optional, List, Union
-
-import agate
-
-from dbt.adapters.base.impl import AdapterConfig
-from dbt.adapters.sql import SQLAdapter
-from dbt.adapters.sql.impl import (
-    LIST_SCHEMAS_MACRO_NAME,
-    LIST_RELATIONS_MACRO_NAME,
-)
-from dbt.adapters.snowflake import SnowflakeConnectionManager
-from dbt.adapters.snowflake import SnowflakeRelation
-from dbt.adapters.snowflake import SnowflakeColumn
-from dbt.contracts.graph.manifest import Manifest
-from dbt.exceptions import (
-    raise_compiler_error, RuntimeException, DatabaseException
-)
-from dbt.utils import filter_null_values
-
-
-@dataclass
-class SnowflakeConfig(AdapterConfig):
-    transient: Optional[bool] = None
-    cluster_by: Optional[Union[str, List[str]]] = None
-    automatic_clustering: Optional[bool] = None
-    secure: Optional[bool] = None
-    copy_grants: Optional[bool] = None
-    snowflake_warehouse: Optional[str] = None
-    query_tag: Optional[str] = None
-    merge_update_columns: Optional[str] = None
-
-
-class SnowflakeAdapter(SQLAdapter):
-    Relation = SnowflakeRelation
-    Column = SnowflakeColumn
-    ConnectionManager = SnowflakeConnectionManager
-
-    AdapterSpecificConfigs = SnowflakeConfig
-
-    @classmethod
-    def date_function(cls):
-        return "CURRENT_TIMESTAMP()"
-
-    @classmethod
-    def _catalog_filter_table(
-        cls, table: agate.Table, manifest: Manifest
-    ) -> agate.Table:
-        # On snowflake, users can set QUOTED_IDENTIFIERS_IGNORE_CASE, so force
-        # the column names to their lowercased forms.
-        lowered = table.rename(
-            column_names=[c.lower() for c in table.column_names]
-        )
-        return super()._catalog_filter_table(lowered, manifest)
-
-    def _make_match_kwargs(self, database, schema, identifier):
-        quoting = self.config.quoting
-        if identifier is not None and quoting["identifier"] is False:
-            identifier = identifier.upper()
-
-        if schema is not None and quoting["schema"] is False:
-            schema = schema.upper()
-
-        if database is not None and quoting["database"] is False:
-            database = database.upper()
-
-        return filter_null_values(
-            {"identifier": identifier, "schema": schema, "database": database}
-        )
-
-    def _get_warehouse(self) -> str:
-        _, table = self.execute(
-            'select current_warehouse() as warehouse',
-            fetch=True
-        )
-        if len(table) == 0 or len(table[0]) == 0:
-            # can this happen?
-            raise RuntimeException(
-                'Could not get current warehouse: no results'
-            )
-        return str(table[0][0])
-
-    def _use_warehouse(self, warehouse: str):
-        """Use the given warehouse. Quotes are never applied."""
-        self.execute('use warehouse {}'.format(warehouse))
-
-    def pre_model_hook(self, config: Mapping[str, Any]) -> Optional[str]:
-        default_warehouse = self.config.credentials.warehouse
-        warehouse = config.get('snowflake_warehouse', default_warehouse)
-        if warehouse == default_warehouse or warehouse is None:
-            return None
-        previous = self._get_warehouse()
-        self._use_warehouse(warehouse)
-        return previous
-
-    def post_model_hook(
-        self, config: Mapping[str, Any], context: Optional[str]
-    ) -> None:
-        if context is not None:
-            self._use_warehouse(context)
-
-    def list_schemas(self, database: str) -> List[str]:
-        try:
-            results = self.execute_macro(
-                LIST_SCHEMAS_MACRO_NAME,
-                kwargs={'database': database}
-            )
-        except DatabaseException as exc:
-            msg = (
-                f'Database error while listing schemas in database '
-                f'"{database}"\n{exc}'
-            )
-            raise RuntimeException(msg)
-        # this uses 'show terse schemas in database', and the column name we
-        # want is 'name'
-
-        return [row['name'] for row in results]
-
-    def get_columns_in_relation(self, relation):
-        try:
-            return super().get_columns_in_relation(relation)
-        except DatabaseException as exc:
-            if 'does not exist or not authorized' in str(exc):
-                return []
-            else:
-                raise
-
-    def list_relations_without_caching(
-            self, schema_relation: SnowflakeRelation
-    ) -> List[SnowflakeRelation]:
-        kwargs = {'schema_relation': schema_relation}
-        try:
-            results = self.execute_macro(
-                LIST_RELATIONS_MACRO_NAME,
-                kwargs=kwargs
-            )
-        except DatabaseException as exc:
-            # if the schema doesn't exist, we just want to return.
-            # Alternatively, we could query the list of schemas before we start
-            # and skip listing the missing ones, which sounds expensive.
-            if 'Object does not exist' in str(exc):
-                return []
-            raise
-
-        relations = []
-        quote_policy = {
-            'database': True,
-            'schema': True,
-            'identifier': True
-        }
-
-        columns = ['database_name', 'schema_name', 'name', 'kind']
-        for _database, _schema, _identifier, _type in results.select(columns):
-            try:
-                _type = self.Relation.get_relation_type(_type.lower())
-            except ValueError:
-                _type = self.Relation.External
-            relations.append(self.Relation.create(
-                database=_database,
-                schema=_schema,
-                identifier=_identifier,
-                quote_policy=quote_policy,
-                type=_type
-            ))
-
-        return relations
-
-    def quote_seed_column(
-        self, column: str, quote_config: Optional[bool]
-    ) -> str:
-        quote_columns: bool = False
-        if isinstance(quote_config, bool):
-            quote_columns = quote_config
-        elif quote_config is None:
-            pass
-        else:
-            raise_compiler_error(
-                f'The seed configuration value of "quote_columns" has an '
-                f'invalid type {type(quote_config)}'
-            )
-
-        if quote_columns:
-            return self.quote(column)
-        else:
-            return column
-
-    def timestamp_add_sql(
-        self, add_to: str, number: int = 1, interval: str = 'hour'
-    ) -> str:
-        return f'DATEADD({interval}, {number}, {add_to})'

plugins/snowflake/dbt/adapters/snowflake/relation.py

@@ -1,14 +0,0 @@
-from dataclasses import dataclass
-from dbt.adapters.base.relation import BaseRelation, Policy
-
-
-@dataclass
-class SnowflakeQuotePolicy(Policy):
-    database: bool = False
-    schema: bool = False
-    identifier: bool = False
-
-
-@dataclass(frozen=True, eq=False, repr=False)
-class SnowflakeRelation(BaseRelation):
-    quote_policy: SnowflakeQuotePolicy = SnowflakeQuotePolicy()

plugins/snowflake/dbt/include/snowflake/__init__.py

@@ -1,2 +0,0 @@
-import os
-PACKAGE_PATH = os.path.dirname(__file__)

plugins/snowflake/dbt/include/snowflake/dbt_project.yml

@@ -1,5 +0,0 @@
-config-version: 2
-name: dbt_snowflake
-version: 1.0
-
-macro-paths: ["macros"]

plugins/snowflake/dbt/include/snowflake/macros/adapters.sql

@@ -1,251 +0,0 @@
-{% macro snowflake__create_table_as(temporary, relation, sql) -%}
-  {%- set transient = config.get('transient', default=true) -%}
-  {%- set cluster_by_keys = config.get('cluster_by', default=none) -%}
-  {%- set enable_automatic_clustering = config.get('automatic_clustering', default=false) -%}
-  {%- set copy_grants = config.get('copy_grants', default=false) -%}
-
-  {%- if cluster_by_keys is not none and cluster_by_keys is string -%}
-    {%- set cluster_by_keys = [cluster_by_keys] -%}
-  {%- endif -%}
-  {%- if cluster_by_keys is not none -%}
-    {%- set cluster_by_string = cluster_by_keys|join(", ")-%}
-  {% else %}
-    {%- set cluster_by_string = none -%}
-  {%- endif -%}
-  {%- set sql_header = config.get('sql_header', none) -%}
-
-  {{ sql_header if sql_header is not none }}
-
-      create or replace {% if temporary -%}
-        temporary
-      {%- elif transient -%}
-        transient
-      {%- endif %} table {{ relation }} {% if copy_grants and not temporary -%} copy grants {%- endif %} as
-      (
-        {%- if cluster_by_string is not none -%}
-          select * from(
-            {{ sql }}
-            ) order by ({{ cluster_by_string }})
-        {%- else -%}
-          {{ sql }}
-        {%- endif %}
-      );
-    {% if cluster_by_string is not none and not temporary -%}
-      alter table {{relation}} cluster by ({{cluster_by_string}});
-    {%- endif -%}
-    {% if enable_automatic_clustering and cluster_by_string is not none and not temporary  -%}
-      alter table {{relation}} resume recluster;
-    {%- endif -%}
-{% endmacro %}
-
-{% macro snowflake__create_view_as(relation, sql) -%}
-  {%- set secure = config.get('secure', default=false) -%}
-  {%- set copy_grants = config.get('copy_grants', default=false) -%}
-  {%- set sql_header = config.get('sql_header', none) -%}
-
-  {{ sql_header if sql_header is not none }}
-  create or replace {% if secure -%}
-    secure
-  {%- endif %} view {{ relation }} {% if copy_grants -%} copy grants {%- endif %} as (
-    {{ sql }}
-  );
-{% endmacro %}
-
-{% macro snowflake__get_columns_in_relation(relation) -%}
-  {%- set sql -%}
-    describe table {{ relation }}
-  {%- endset -%}
-  {%- set result = run_query(sql) -%}
-
-  {% set maximum = 10000 %}
-  {% if (result | length) >= maximum %}
-    {% set msg %}
-      Too many columns in relation {{ relation }}! dbt can only get
-      information about relations with fewer than {{ maximum }} columns.
-    {% endset %}
-    {% do exceptions.raise_compiler_error(msg) %}
-  {% endif %}
-
-  {% set columns = [] %}
-  {% for row in result %}
-    {% do columns.append(api.Column.from_description(row['name'], row['type'])) %}
-  {% endfor %}
-  {% do return(columns) %}
-{% endmacro %}
-
-{% macro snowflake__list_schemas(database) -%}
-  {# 10k limit from here: https://docs.snowflake.net/manuals/sql-reference/sql/show-schemas.html#usage-notes #}
-  {% set maximum = 10000 %}
-  {% set sql -%}
-    show terse schemas in database {{ database }}
-    limit {{ maximum }}
-  {%- endset %}
-  {% set result = run_query(sql) %}
-  {% if (result | length) >= maximum %}
-    {% set msg %}
-      Too many schemas in database {{ database }}! dbt can only get
-      information about databases with fewer than {{ maximum }} schemas.
-    {% endset %}
-    {% do exceptions.raise_compiler_error(msg) %}
-  {% endif %}
-  {{ return(result) }}
-{% endmacro %}
-
-
-{% macro snowflake__list_relations_without_caching(schema_relation) %}
-  {%- set sql -%}
-    show terse objects in {{ schema_relation }}
-  {%- endset -%}
-
-  {%- set result = run_query(sql) -%}
-  {% set maximum = 10000 %}
-  {% if (result | length) >= maximum %}
-    {% set msg %}
-      Too many schemas in schema  {{ schema_relation }}! dbt can only get
-      information about schemas with fewer than {{ maximum }} objects.
-    {% endset %}
-    {% do exceptions.raise_compiler_error(msg) %}
-  {% endif %}
-  {%- do return(result) -%}
-{% endmacro %}
-
-
-{% macro snowflake__check_schema_exists(information_schema, schema) -%}
-  {% call statement('check_schema_exists', fetch_result=True) -%}
-        select count(*)
-        from {{ information_schema }}.schemata
-        where upper(schema_name) = upper('{{ schema }}')
-            and upper(catalog_name) = upper('{{ information_schema.database }}')
-  {%- endcall %}
-  {{ return(load_result('check_schema_exists').table) }}
-{%- endmacro %}
-
-{% macro snowflake__current_timestamp() -%}
-  convert_timezone('UTC', current_timestamp())
-{%- endmacro %}
-
-
-{% macro snowflake__snapshot_string_as_time(timestamp) -%}
-    {%- set result = "to_timestamp_ntz('" ~ timestamp ~ "')" -%}
-    {{ return(result) }}
-{%- endmacro %}
-
-
-{% macro snowflake__snapshot_get_time() -%}
-  to_timestamp_ntz({{ current_timestamp() }})
-{%- endmacro %}
-
-
-{% macro snowflake__rename_relation(from_relation, to_relation) -%}
-  {% call statement('rename_relation') -%}
-    alter table {{ from_relation }} rename to {{ to_relation }}
-  {%- endcall %}
-{% endmacro %}
-
-
-{% macro snowflake__alter_column_type(relation, column_name, new_column_type) -%}
-  {% call statement('alter_column_type') %}
-    alter table {{ relation }} alter {{ adapter.quote(column_name) }} set data type {{ new_column_type }};
-  {% endcall %}
-{% endmacro %}
-
-{% macro snowflake__alter_relation_comment(relation, relation_comment) -%}
-  comment on {{ relation.type }} {{ relation }} IS $${{ relation_comment | replace('$', '[$]') }}$$;
-{% endmacro %}
-
-
-{% macro snowflake__alter_column_comment(relation, column_dict) -%}
-    {% set existing_columns = adapter.get_columns_in_relation(relation) | map(attribute="name") | list %}
-    alter {{ relation.type }} {{ relation }} alter
-    {% for column_name in column_dict if (column_name in existing_columns) or (column_name|upper in existing_columns) %}
-        {{ adapter.quote(column_name) if column_dict[column_name]['quote'] else column_name }} COMMENT $${{ column_dict[column_name]['description'] | replace('$', '[$]') }}$$ {{ ',' if not loop.last else ';' }}
-    {% endfor %}
-{% endmacro %}
-
-
-{% macro get_current_query_tag() -%}
-  {{ return(run_query("show parameters like 'query_tag' in session").rows[0]['value']) }}
-{% endmacro %}
-
-
-{% macro set_query_tag() -%}
-  {% set new_query_tag = config.get('query_tag') %}
-  {% if new_query_tag %}
-    {% set original_query_tag = get_current_query_tag() %}
-    {{ log("Setting query_tag to '" ~ new_query_tag ~ "'. Will reset to '" ~ original_query_tag ~ "' after materialization.") }}
-    {% do run_query("alter session set query_tag = '{}'".format(new_query_tag)) %}
-    {{ return(original_query_tag)}}
-  {% endif %}
-  {{ return(none)}}
-{% endmacro %}
-
-{% macro unset_query_tag(original_query_tag) -%}
-  {% set new_query_tag = config.get('query_tag') %}
-  {% if new_query_tag %}
-    {% if original_query_tag %}
-      {{ log("Resetting query_tag to '" ~ original_query_tag ~ "'.") }}
-      {% do run_query("alter session set query_tag = '{}'".format(original_query_tag)) %}
-    {% else %}
-      {{ log("No original query_tag, unsetting parameter.") }}
-      {% do run_query("alter session unset query_tag") %}
-    {% endif %}
-  {% endif %}
-{% endmacro %} 
-
-
-{% macro snowflake__alter_relation_add_remove_columns(relation, add_columns, remove_columns) %}
-  
-  {% if add_columns %}
-    
-    {% set sql -%}
-       alter {{ relation.type }} {{ relation }} add column
-          {% for column in add_columns %}
-            {{ column.name }} {{ column.data_type }}{{ ',' if not loop.last }}
-          {% endfor %}
-    {%- endset -%}
-
-    {% do run_query(sql) %}
-
-  {% endif %}
-
-  {% if remove_columns %}
-  
-    {% set sql -%}
-        alter {{ relation.type }} {{ relation }} drop column
-            {% for column in remove_columns %}
-                {{ column.name }}{{ ',' if not loop.last }}
-            {% endfor %}
-    {%- endset -%}
-    
-    {% do run_query(sql) %}
-    
-  {% endif %}
-
-{% endmacro %}
-
-
-{% macro snowflake_dml_explicit_transaction(dml) %}
-  {#
-    Use this macro to wrap all INSERT, MERGE, UPDATE, DELETE, and TRUNCATE 
-    statements before passing them into run_query(), or calling in the 'main' statement
-    of a materialization
-  #}
-  {% set dml_transaction -%}
-    begin;
-    {{ dml }};
-    commit;
-  {%- endset %}
-  
-  {% do return(dml_transaction) %}
-
-{% endmacro %}
-
-
-{% macro snowflake__truncate_relation(relation) -%}
-  {% set truncate_dml %}
-    truncate table {{ relation }}
-  {% endset %}
-  {% call statement('truncate_relation') -%}
-    {{ snowflake_dml_explicit_transaction(truncate_dml) }}
-  {%- endcall %}
-{% endmacro %}

plugins/snowflake/dbt/include/snowflake/macros/catalog.sql

@@ -1,67 +0,0 @@
-{% macro snowflake__get_catalog(information_schema, schemas) -%}
-  {% set query %}
-      with tables as (
-
-          select
-              table_catalog as "table_database",
-              table_schema as "table_schema",
-              table_name as "table_name",
-              table_type as "table_type",
-              comment as "table_comment",
-
-              -- note: this is the _role_ that owns the table
-              table_owner as "table_owner",
-
-              'Clustering Key' as "stats:clustering_key:label",
-              clustering_key as "stats:clustering_key:value",
-              'The key used to cluster this table' as "stats:clustering_key:description",
-              (clustering_key is not null) as "stats:clustering_key:include",
-
-              'Row Count' as "stats:row_count:label",
-              row_count as "stats:row_count:value",
-              'An approximate count of rows in this table' as "stats:row_count:description",
-              (row_count is not null) as "stats:row_count:include",
-
-              'Approximate Size' as "stats:bytes:label",
-              bytes as "stats:bytes:value",
-              'Approximate size of the table as reported by Snowflake' as "stats:bytes:description",
-              (bytes is not null) as "stats:bytes:include",
-
-              'Last Modified' as "stats:last_modified:label",
-              to_varchar(convert_timezone('UTC', last_altered), 'yyyy-mm-dd HH24:MI'||'UTC') as "stats:last_modified:value",
-              'The timestamp for last update/change' as "stats:last_modified:description",
-              (last_altered is not null and table_type='BASE TABLE') as "stats:last_modified:include"
-
-          from {{ information_schema }}.tables
-
-      ),
-
-      columns as (
-
-          select
-              table_catalog as "table_database",
-              table_schema as "table_schema",
-              table_name as "table_name",
-
-              column_name as "column_name",
-              ordinal_position as "column_index",
-              data_type as "column_type",
-              comment as "column_comment"
-
-          from {{ information_schema }}.columns
-      )
-
-      select *
-      from tables
-      join columns using ("table_database", "table_schema", "table_name")
-      where (
-        {%- for schema in schemas -%}
-          upper("table_schema") = upper('{{ schema }}'){%- if not loop.last %} or {% endif -%}
-        {%- endfor -%}
-      )
-      order by "column_index"
-    {%- endset -%}
-
-  {{ return(run_query(query)) }}
-
-{%- endmacro %}

plugins/snowflake/dbt/include/snowflake/macros/materializations/incremental.sql

@@ -1,80 +0,0 @@
-
-{% macro dbt_snowflake_validate_get_incremental_strategy(config) %}
-  {#-- Find and validate the incremental strategy #}
-  {%- set strategy = config.get("incremental_strategy", default="merge") -%}
-
-  {% set invalid_strategy_msg -%}
-    Invalid incremental strategy provided: {{ strategy }}
-    Expected one of: 'merge', 'delete+insert'
-  {%- endset %}
-  {% if strategy not in ['merge', 'delete+insert'] %}
-    {% do exceptions.raise_compiler_error(invalid_strategy_msg) %}
-  {% endif %}
-
-  {% do return(strategy) %}
-{% endmacro %}
-
-{% macro dbt_snowflake_get_incremental_sql(strategy, tmp_relation, target_relation, unique_key, dest_columns) %}
-  {% if strategy == 'merge' %}
-    {% do return(get_merge_sql(target_relation, tmp_relation, unique_key, dest_columns)) %}
-  {% elif strategy == 'delete+insert' %}
-    {% do return(get_delete_insert_merge_sql(target_relation, tmp_relation, unique_key, dest_columns)) %}
-  {% else %}
-    {% do exceptions.raise_compiler_error('invalid strategy: ' ~ strategy) %}
-  {% endif %}
-{% endmacro %}
-
-{% materialization incremental, adapter='snowflake' -%}
-   
-  {% set original_query_tag = set_query_tag() %}
-
-  {%- set unique_key = config.get('unique_key') -%}
-  {%- set full_refresh_mode = (should_full_refresh()) -%}
-
-  {% set target_relation = this %}
-  {% set existing_relation = load_relation(this) %}
-  {% set tmp_relation = make_temp_relation(this) %}
-
-  {#-- Validate early so we don't run SQL if the strategy is invalid --#}
-  {% set strategy = dbt_snowflake_validate_get_incremental_strategy(config) -%}
-  {% set on_schema_change = incremental_validate_on_schema_change(config.get('on_schema_change'), default='ignore') %}
-
-  {{ run_hooks(pre_hooks) }}
-
-  {% if existing_relation is none %}
-    {% set build_sql = create_table_as(False, target_relation, sql) %}
-  
-  {% elif existing_relation.is_view %}
-    {#-- Can't overwrite a view with a table - we must drop --#}
-    {{ log("Dropping relation " ~ target_relation ~ " because it is a view and this model is a table.") }}
-    {% do adapter.drop_relation(existing_relation) %}
-    {% set build_sql = create_table_as(False, target_relation, sql) %}
-  
-  {% elif full_refresh_mode %}
-    {% set build_sql = create_table_as(False, target_relation, sql) %}
-  
-  {% else %}
-    {% do run_query(create_table_as(True, tmp_relation, sql)) %}
-    {% do adapter.expand_target_column_types(
-           from_relation=tmp_relation,
-           to_relation=target_relation) %}
-    {% do process_schema_changes(on_schema_change, tmp_relation, existing_relation) %}
-    {% set dest_columns = adapter.get_columns_in_relation(existing_relation) %}
-    {% set build_sql = dbt_snowflake_get_incremental_sql(strategy, tmp_relation, target_relation, unique_key, dest_columns) %}
-  
-  {% endif %}
-
-  {%- call statement('main') -%}
-    {{ build_sql }}
-  {%- endcall -%}
-
-  {{ run_hooks(post_hooks) }}
-
-  {% set target_relation = target_relation.incorporate(type='table') %}
-  {% do persist_docs(target_relation, model) %}
-
-  {% do unset_query_tag(original_query_tag) %}
-
-  {{ return({'relations': [target_relation]}) }}
-
-{%- endmaterialization %}

plugins/snowflake/dbt/include/snowflake/macros/materializations/merge.sql

@@ -1,44 +0,0 @@
-{% macro snowflake__get_merge_sql(target, source_sql, unique_key, dest_columns, predicates) -%}
-
-    {#
-       Workaround for Snowflake not being happy with a merge on a constant-false predicate.
-       When no unique_key is provided, this macro will do a regular insert. If a unique_key
-       is provided, then this macro will do a proper merge instead.
-    #}
-
-    {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute='name')) -%}
-    {%- set sql_header = config.get('sql_header', none) -%}
-
-    {%- set dml -%}
-    {%- if unique_key is none -%}
-
-        {{ sql_header if sql_header is not none }}
-
-        insert into {{ target }} ({{ dest_cols_csv }})
-        (
-            select {{ dest_cols_csv }}
-            from {{ source_sql }}
-        )
-
-    {%- else -%}
-
-        {{ default__get_merge_sql(target, source_sql, unique_key, dest_columns, predicates) }}
-
-    {%- endif -%}
-    {%- endset -%}
-    
-    {% do return(snowflake_dml_explicit_transaction(dml)) %}
-
-{% endmacro %}
-
-
-{% macro snowflake__get_delete_insert_merge_sql(target, source, unique_key, dest_columns) %}
-    {% set dml = default__get_delete_insert_merge_sql(target, source, unique_key, dest_columns) %}
-    {% do return(snowflake_dml_explicit_transaction(dml)) %}
-{% endmacro %}
-
-
-{% macro snowflake__snapshot_merge_sql(target, source, insert_cols) %}
-    {% set dml = default__snapshot_merge_sql(target, source, insert_cols) %}
-    {% do return(snowflake_dml_explicit_transaction(dml)) %}
-{% endmacro %}

plugins/snowflake/dbt/include/snowflake/macros/materializations/seed.sql

@@ -1,37 +0,0 @@
-{% macro snowflake__load_csv_rows(model, agate_table) %}
-    {% set batch_size = get_batch_size() %}
-    {% set cols_sql = get_seed_column_quoted_csv(model, agate_table.column_names) %}
-    {% set bindings = [] %}
-
-    {% set statements = [] %}
-
-    {% for chunk in agate_table.rows | batch(batch_size) %}
-        {% set bindings = [] %}
-
-        {% for row in chunk %}
-            {% do bindings.extend(row) %}
-        {% endfor %}
-
-        {% set sql %}
-            insert into {{ this.render() }} ({{ cols_sql }}) values
-            {% for row in chunk -%}
-                ({%- for column in agate_table.column_names -%}
-                    %s
-                    {%- if not loop.last%},{%- endif %}
-                {%- endfor -%})
-                {%- if not loop.last%},{%- endif %}
-            {%- endfor %}
-        {% endset %}
-
-        {% do adapter.add_query('BEGIN', auto_begin=False) %}
-        {% do adapter.add_query(sql, bindings=bindings, abridge_sql_log=True) %}
-        {% do adapter.add_query('COMMIT', auto_begin=False) %}
-
-        {% if loop.index0 == 0 %}
-            {% do statements.append(sql) %}
-        {% endif %}
-    {% endfor %}
-
-    {# Return SQL so we can render it out into the compiled files #}
-    {{ return(statements[0]) }}
-{% endmacro %}

plugins/snowflake/dbt/include/snowflake/macros/materializations/table.sql

@@ -1,34 +0,0 @@
-{% materialization table, adapter='snowflake' %}
-
-  {% set original_query_tag = set_query_tag() %}
-
-  {%- set identifier = model['alias'] -%}
-
-  {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}
-  {%- set target_relation = api.Relation.create(identifier=identifier,
-                                                schema=schema,
-                                                database=database, type='table') -%}
-
-  {{ run_hooks(pre_hooks) }}
-
-  {#-- Drop the relation if it was a view to "convert" it in a table. This may lead to
-    -- downtime, but it should be a relatively infrequent occurrence  #}
-  {% if old_relation is not none and not old_relation.is_table %}
-    {{ log("Dropping relation " ~ old_relation ~ " because it is of type " ~ old_relation.type) }}
-    {{ drop_relation_if_exists(old_relation) }}
-  {% endif %}
-
-  --build model
-  {% call statement('main') -%}
-    {{ create_table_as(false, target_relation, sql) }}
-  {%- endcall %}
-
-  {{ run_hooks(post_hooks) }}
-
-  {% do persist_docs(target_relation, model) %}
-
-  {% do unset_query_tag(original_query_tag) %}
-
-  {{ return({'relations': [target_relation]}) }}
-
-{% endmaterialization %}

plugins/snowflake/dbt/include/snowflake/macros/materializations/view.sql

@@ -1,13 +0,0 @@
-{% materialization view, adapter='snowflake' -%}
-
-    {% set original_query_tag = set_query_tag() %}
-    {% set to_return = create_or_replace_view() %}
-
-    {% set target_relation = this.incorporate(type='view') %}
-    {% do persist_docs(target_relation, model, for_columns=false) %}
-
-    {% do return(to_return) %}
-
-    {% do unset_query_tag(original_query_tag) %}
-
-{%- endmaterialization %}

plugins/snowflake/dbt/include/snowflake/sample_profiles.yml

@@ -1,29 +0,0 @@
-default:
-  outputs:
-
-    dev: # User-Password config
-      type: snowflake
-      account: [account id + region (if applicable)]
-      user: [username]
-      password: [password]
-      role: [user role]
-      database: [database name]
-      warehouse: [warehouse name]
-      schema: [dbt schema]
-      threads: [1 or more]
-      client_session_keep_alive: False
-
-    prod: # Keypair config
-      type: snowflake
-      account: [account id + region (if applicable)]
-      user: [username]
-      role: [user role]
-      private_key_path: [path/to/private.key]
-      private_key_passphrase: [passphrase for the private key, if key is encrypted]
-      database: [database name]
-      warehouse: [warehouse name]
-      schema: [dbt schema]
-      threads: [1 or more]
-      client_session_keep_alive: False
-  
-  target: dev

plugins/snowflake/setup.py

@@ -1,70 +0,0 @@
-#!/usr/bin/env python
-import os
-import sys
-
-if sys.version_info < (3, 6):
-    print('Error: dbt does not support this version of Python.')
-    print('Please upgrade to Python 3.6 or higher.')
-    sys.exit(1)
-
-
-from setuptools import setup
-try:
-    from setuptools import find_namespace_packages
-except ImportError:
-    # the user has a downlevel version of setuptools.
-    print('Error: dbt requires setuptools v40.1.0 or higher.')
-    print('Please upgrade setuptools with "pip install --upgrade setuptools" '
-          'and try again')
-    sys.exit(1)
-
-
-package_name = "dbt-snowflake"
-package_version = "0.21.0rc1"
-description = """The snowflake adapter plugin for dbt (data build tool)"""
-
-this_directory = os.path.abspath(os.path.dirname(__file__))
-with open(os.path.join(this_directory, 'README.md')) as f:
-    long_description = f.read()
-
-setup(
-    name=package_name,
-    version=package_version,
-    description=description,
-    long_description=description,
-    long_description_content_type='text/markdown',
-    author="dbt Labs",
-    author_email="info@dbtlabs.com",
-    url="https://github.com/dbt-labs/dbt",
-    packages=find_namespace_packages(include=['dbt', 'dbt.*']),
-    package_data={
-        'dbt': [
-            'include/snowflake/dbt_project.yml',
-            'include/snowflake/sample_profiles.yml',
-            'include/snowflake/macros/*.sql',
-            'include/snowflake/macros/**/*.sql',
-        ]
-    },
-    install_requires=[
-        'dbt-core=={}'.format(package_version),
-        'snowflake-connector-python[secure-local-storage]>=2.4.1,<2.6.0',
-        'requests<3.0.0',
-        'cryptography>=3.2,<4',
-    ],
-    zip_safe=False,
-    classifiers=[
-        'Development Status :: 5 - Production/Stable',
-
-        'License :: OSI Approved :: Apache Software License',
-
-        'Operating System :: Microsoft :: Windows',
-        'Operating System :: MacOS :: MacOS X',
-        'Operating System :: POSIX :: Linux',
-
-        'Programming Language :: Python :: 3.6',
-        'Programming Language :: Python :: 3.7',
-        'Programming Language :: Python :: 3.8',
-        'Programming Language :: Python :: 3.9',
-    ],
-    python_requires=">=3.6.2",
-)

requirements.txt

@@ -1,5 +1,3 @@
 ./core
 ./plugins/postgres
-./plugins/redshift
-./plugins/snowflake
 ./plugins/bigquery

scripts/build-dist.sh

@@ -14,7 +14,7 @@ rm -rf "$DBT_PATH"/dist
 rm -rf "$DBT_PATH"/build
 mkdir -p "$DBT_PATH"/dist
 
-for SUBPATH in core plugins/postgres plugins/redshift plugins/bigquery plugins/snowflake
+for SUBPATH in core plugins/postgres plugins/bigquery
 do
     rm -rf "$DBT_PATH"/"$SUBPATH"/dist
     rm -rf "$DBT_PATH"/"$SUBPATH"/build

setup.py

@@ -44,8 +44,6 @@ setup(
     install_requires=[
         'dbt-core=={}'.format(package_version),
         'dbt-postgres=={}'.format(package_version),
-        'dbt-redshift=={}'.format(package_version),
-        'dbt-snowflake=={}'.format(package_version),
         'dbt-bigquery=={}'.format(package_version),
     ],
     zip_safe=False,

test.env.sample

@@ -14,9 +14,3 @@ SNOWFLAKE_TEST_OAUTH_CLIENT_SECRET=
 
 BIGQUERY_TEST_SERVICE_ACCOUNT_JSON=
 BIGQUERY_TEST_ALT_DATABASE=
-
-REDSHIFT_TEST_HOST=
-REDSHIFT_TEST_USER=
-REDSHIFT_TEST_PASS=
-REDSHIFT_TEST_PORT=
-REDSHIFT_TEST_DBNAME=

test/integration/001_simple_copy_test/models-snowflake/incremental_overwrite.sql

@@ -1,9 +1,6 @@
 
 {{ config(materialized='incremental', unique_key='id') }}
 
--- this will fail on snowflake with "merge" due
--- to the nondeterministic join on id
-
 select 1 as id
 union all
 select 1 as id

test/integration/001_simple_copy_test/test_simple_copy.py

@@ -105,89 +105,6 @@ class TestSimpleCopy(BaseTestSimpleCopy):
 
         self.assertManyTablesEqual(["seed", "view_model", "materialized"])
 
-    @use_profile("snowflake")
-    def test__snowflake__simple_copy(self):
-        self.use_default_project({
-            "data-paths": [self.dir("seed-initial")],
-            "seeds": {
-                'quote_columns': False,
-            }
-        })
-        results = self.run_dbt(["seed"])
-        self.assertEqual(len(results),  1)
-        results = self.run_dbt()
-        self.assertEqual(len(results),  7)
-
-        self.assertManyTablesEqual(["SEED", "VIEW_MODEL", "INCREMENTAL", "MATERIALIZED", "GET_AND_REF"])
-
-        self.use_default_project({"data-paths": [self.dir("seed-update")]})
-        results = self.run_dbt(["seed"])
-        self.assertEqual(len(results),  1)
-        results = self.run_dbt()
-        self.assertEqual(len(results),  7)
-
-        self.assertManyTablesEqual(["SEED", "VIEW_MODEL", "INCREMENTAL", "MATERIALIZED", "GET_AND_REF"])
-
-        self.use_default_project({
-            "test-paths": [self.dir("tests")],
-            "data-paths": [self.dir("seed-update")],
-        })
-        self.run_dbt(['test'])
-
-    @use_profile("snowflake")
-    def test__snowflake__simple_copy__quoting_off(self):
-        self.use_default_project({
-            "quoting": {"identifier": False},
-            "data-paths": [self.dir("snowflake-seed-initial")],
-        })
-
-        results = self.run_dbt(["seed"])
-        self.assertEqual(len(results),  1)
-        results = self.run_dbt()
-        self.assertEqual(len(results),  7)
-
-        self.assertManyTablesEqual(["SEED", "VIEW_MODEL", "INCREMENTAL", "MATERIALIZED", "GET_AND_REF"])
-
-        self.use_default_project({
-            "data-paths": [self.dir("snowflake-seed-update")],
-            "quoting": {"identifier": False},
-        })
-        results = self.run_dbt(["seed"])
-        self.assertEqual(len(results),  1)
-        results = self.run_dbt()
-        self.assertEqual(len(results),  7)
-
-        self.assertManyTablesEqual(["SEED", "VIEW_MODEL", "INCREMENTAL", "MATERIALIZED", "GET_AND_REF"])
-
-        self.use_default_project({
-            "test-paths": [self.dir("tests")],
-            "data-paths": [self.dir("snowflake-seed-update")],
-            "quoting": {"identifier": False},
-        })
-        self.run_dbt(['test'])
-
-    @use_profile("snowflake")
-    def test__snowflake__seed__quoting_switch(self):
-        self.use_default_project({
-            "quoting": {"identifier": False},
-            "data-paths": [self.dir("snowflake-seed-initial")],
-        })
-
-        results = self.run_dbt(["seed"])
-        self.assertEqual(len(results),  1)
-
-        self.use_default_project({
-            "data-paths": [self.dir("snowflake-seed-update")],
-            "quoting": {"identifier": True},
-        })
-        results = self.run_dbt(["seed"], expect_pass=False)
-
-        self.use_default_project({
-            "test-paths": [self.dir("tests")],
-            "data-paths": [self.dir("snowflake-seed-initial")],
-        })
-        self.run_dbt(['test'])
-
     @use_profile("bigquery")
     def test__bigquery__simple_copy(self):
         results = self.run_dbt(["seed"])
@@ -213,155 +130,6 @@ class TestSimpleCopy(BaseTestSimpleCopy):
         self.assertTablesEqual("seed", "get_and_ref")
 
 
-class TestSimpleCopyQuotingIdentifierOn(BaseTestSimpleCopy):
-    @property
-    def project_config(self):
-        return self.seed_quote_cfg_with({
-            'quoting': {
-                'identifier': True,
-            },
-        })
-
-    @use_profile("snowflake")
-    def test__snowflake__simple_copy__quoting_on(self):
-        self.use_default_project({
-            "data-paths": [self.dir("snowflake-seed-initial")],
-        })
-
-        results = self.run_dbt(["seed"])
-        self.assertEqual(len(results),  1)
-        results = self.run_dbt()
-        self.assertEqual(len(results),  7)
-
-        self.assertManyTablesEqual(["seed", "view_model", "incremental", "materialized", "get_and_ref"])
-
-        self.use_default_project({
-            "data-paths": [self.dir("snowflake-seed-update")],
-        })
-        results = self.run_dbt(["seed"])
-        self.assertEqual(len(results),  1)
-        results = self.run_dbt()
-        self.assertEqual(len(results),  7)
-
-        self.assertManyTablesEqual(["seed", "view_model", "incremental", "materialized", "get_and_ref"])
-
-        # can't run the test as this one's identifiers will be the wrong case
-
-
-class BaseLowercasedSchemaTest(BaseTestSimpleCopy):
-    def unique_schema(self):
-        # bypass the forced uppercasing that unique_schema() does on snowflake
-        return super().unique_schema().lower()
-
-
-class TestSnowflakeSimpleLowercasedSchemaCopy(BaseLowercasedSchemaTest):
-    @use_profile('snowflake')
-    def test__snowflake__simple_copy(self):
-        self.use_default_project({"data-paths": [self.dir("snowflake-seed-initial")]})
-
-        results = self.run_dbt(["seed"])
-        self.assertEqual(len(results),  1)
-        results = self.run_dbt()
-        self.assertEqual(len(results),  7)
-
-        self.assertManyTablesEqual(["SEED", "VIEW_MODEL", "INCREMENTAL", "MATERIALIZED", "GET_AND_REF"])
-
-        self.use_default_project({"data-paths": [self.dir("snowflake-seed-update")]})
-
-        results = self.run_dbt(["seed"])
-        self.assertEqual(len(results),  1)
-        results = self.run_dbt()
-        self.assertEqual(len(results),  7)
-
-        self.assertManyTablesEqual(["SEED", "VIEW_MODEL", "INCREMENTAL", "MATERIALIZED", "GET_AND_REF"])
-
-        self.use_default_project({
-            "test-paths": [self.dir("tests")],
-            "data-paths": [self.dir("snowflake-seed-update")],
-        })
-        self.run_dbt(['test'])
-
-
-class TestSnowflakeSimpleLowercasedSchemaQuoted(BaseLowercasedSchemaTest):
-    @property
-    def project_config(self):
-        return self.seed_quote_cfg_with({
-            'quoting': {'identifier': False, 'schema': True}
-        })
-
-    @use_profile("snowflake")
-    def test__snowflake__seed__quoting_switch_schema_lower(self):
-        self.use_default_project({
-            "data-paths": [self.dir("snowflake-seed-initial")],
-        })
-
-        results = self.run_dbt(["seed"])
-        self.assertEqual(len(results),  1)
-        # this is intentional - should not error!
-        results = self.run_dbt(["seed"])
-        self.assertEqual(len(results),  1)
-
-        self.use_default_project({
-            "data-paths": [self.dir("snowflake-seed-update")],
-            "quoting": {"identifier": False, "schema": False},
-        })
-        results = self.run_dbt(["seed"], expect_pass=False)
-
-
-class TestSnowflakeSimpleUppercasedSchemaQuoted(BaseTestSimpleCopy):
-    @property
-    def project_config(self):
-        return self.seed_quote_cfg_with({
-            'quoting': {'identifier': False, 'schema': True}
-        })
-
-    @use_profile("snowflake")
-    def test__snowflake__seed__quoting_switch_schema_upper(self):
-        self.use_default_project({
-            "data-paths": [self.dir("snowflake-seed-initial")],
-        })
-
-        results = self.run_dbt(["seed"])
-        self.assertEqual(len(results),  1)
-        # this is intentional - should not error!
-        results = self.run_dbt(["seed"])
-        self.assertEqual(len(results),  1)
-
-        self.use_default_project({
-            "data-paths": [self.dir("snowflake-seed-update")],
-            "quoting": {"identifier": False, "schema": False},
-        })
-        results = self.run_dbt(["seed"])
-
-
-class TestSnowflakeIncrementalOverwrite(BaseTestSimpleCopy):
-    @property
-    def models(self):
-        return self.dir("models-snowflake")
-
-    @use_profile("snowflake")
-    def test__snowflake__incremental_overwrite(self):
-        self.use_default_project({
-            "data-paths": [self.dir("snowflake-seed-initial")],
-        })
-        results = self.run_dbt(["run"])
-        self.assertEqual(len(results),  1)
-
-        results = self.run_dbt(["run"], expect_pass=False)
-        self.assertEqual(len(results),  1)
-
-        # Setting the incremental_strategy should make this succeed
-        self.use_default_project({
-            "models": {
-                "incremental_strategy": "delete+insert"
-            },
-            "data-paths": [self.dir("snowflake-seed-update")],
-        })
-
-        results = self.run_dbt(["run"])
-        self.assertEqual(len(results),  1)
-
-
 class TestShouting(BaseTestSimpleCopy):
     @property
     def models(self):
@@ -498,21 +266,3 @@ class TestIncrementalMergeColumns(BaseTestSimpleCopy):
         })
         self.seed_and_run()
         self.assertTablesEqual("incremental_update_cols", "expected_result")
-
-    @use_profile("snowflake")
-    def test__snowflake__incremental_merge_columns(self):
-        self.use_default_project({
-            "data-paths": ["seeds-merge-cols-initial"],
-            "seeds": {
-                "quote_columns": False
-            }
-        })
-        self.seed_and_run()
-        self.use_default_project({
-            "data-paths": ["seeds-merge-cols-update"],
-            "seeds": {
-                "quote_columns": False
-            }
-        })
-        self.seed_and_run()
-        self.assertTablesEqual("incremental_update_cols", "expected_result")

test/integration/002_varchar_widening_test/test_varchar_widening.py

@@ -27,19 +27,3 @@ class TestVarcharWidening(DBTIntegrationTest):
 
         self.assertTablesEqual("seed","incremental")
         self.assertTablesEqual("seed","materialized")
-
-    @use_profile('snowflake')
-    def test__snowflake__varchar_widening(self):
-        self.run_sql_file("seed.sql")
-
-        results = self.run_dbt()
-        self.assertEqual(len(results),  2)
-
-        self.assertManyTablesEqual(["SEED", "INCREMENTAL", "MATERIALIZED"])
-
-        self.run_sql_file("update.sql")
-
-        results = self.run_dbt()
-        self.assertEqual(len(results),  2)
-
-        self.assertManyTablesEqual(["SEED", "INCREMENTAL", "MATERIALIZED"])

test/integration/003_simple_reference_test/test_simple_reference.py

@@ -66,28 +66,6 @@ class TestSimpleReference(DBTIntegrationTest):
         self.assertTablesEqual("summary_expected","ephemeral_summary")
         self.assertTablesEqual("summary_expected","view_using_ref")
 
-    @use_profile('snowflake')
-    def test__snowflake__simple_reference(self):
-
-        results = self.run_dbt()
-        self.assertEqual(len(results),  8)
-
-        # Copies should match
-        self.assertManyTablesEqual(
-            ["SEED", "INCREMENTAL_COPY", "MATERIALIZED_COPY", "VIEW_COPY"],
-            ["SUMMARY_EXPECTED", "INCREMENTAL_SUMMARY", "MATERIALIZED_SUMMARY", "VIEW_SUMMARY", "EPHEMERAL_SUMMARY"]
-        )
-
-        self.run_sql_file("update.sql")
-
-        results = self.run_dbt()
-        self.assertEqual(len(results),  8)
-
-        self.assertManyTablesEqual(
-            ["SEED", "INCREMENTAL_COPY", "MATERIALIZED_COPY", "VIEW_COPY"],
-            ["SUMMARY_EXPECTED", "INCREMENTAL_SUMMARY", "MATERIALIZED_SUMMARY", "VIEW_SUMMARY", "EPHEMERAL_SUMMARY"]
-        )
-
     @use_profile('postgres')
     def test__postgres__simple_reference_with_models(self):
 
@@ -140,57 +118,6 @@ class TestSimpleReference(DBTIntegrationTest):
         self.assertTrue('ephemeral_summary' in created_models)
         self.assertEqual(created_models['ephemeral_summary'], 'table')
 
-    @use_profile('snowflake')
-    def test__snowflake__simple_reference_with_models(self):
-
-        # Run materialized_copy & ephemeral_copy
-        # ephemeral_copy should not actually be materialized b/c it is ephemeral
-        results = self.run_dbt(
-            ['run', '--models', 'materialized_copy', 'ephemeral_copy']
-        )
-        self.assertEqual(len(results),  1)
-
-        # Copies should match
-        self.assertTablesEqual("SEED", "MATERIALIZED_COPY")
-
-        created_models = self.get_models_in_schema()
-        self.assertTrue('MATERIALIZED_COPY' in created_models)
-
-    @use_profile('snowflake')
-    def test__snowflake__simple_reference_with_models_and_children(self):
-
-        # Run materialized_copy, ephemeral_copy, and their dependents
-        # ephemeral_copy should not actually be materialized b/c it is ephemeral
-        # the dependent ephemeral_summary, however, should be materialized as a table
-        results = self.run_dbt(
-            ['run', '--models', 'materialized_copy+', 'ephemeral_copy+']
-        )
-        self.assertEqual(len(results),  3)
-
-        # Copies should match
-        self.assertManyTablesEqual(
-            ["SEED", "MATERIALIZED_COPY"],
-            ["SUMMARY_EXPECTED", "MATERIALIZED_SUMMARY", "EPHEMERAL_SUMMARY"]
-        )
-
-        created_models = self.get_models_in_schema()
-
-        self.assertFalse('INCREMENTAL_COPY' in created_models)
-        self.assertFalse('INCREMENTAL_SUMMARY' in created_models)
-        self.assertFalse('VIEW_COPY' in created_models)
-        self.assertFalse('VIEW_SUMMARY' in created_models)
-
-        # make sure this wasn't errantly materialized
-        self.assertFalse('EPHEMERAL_COPY' in created_models)
-
-        self.assertTrue('MATERIALIZED_COPY' in created_models)
-        self.assertTrue('MATERIALIZED_SUMMARY' in created_models)
-        self.assertEqual(created_models['MATERIALIZED_COPY'], 'table')
-        self.assertEqual(created_models['MATERIALIZED_SUMMARY'], 'table')
-
-        self.assertTrue('EPHEMERAL_SUMMARY' in created_models)
-        self.assertEqual(created_models['EPHEMERAL_SUMMARY'], 'table')
-
 
 class TestErrorReference(DBTIntegrationTest):
     @property

test/integration/004_simple_snapshot_test/test_simple_snapshot.py

@@ -28,10 +28,7 @@ class BaseSimpleSnapshotTest(DBTIntegrationTest):
         self.assertEqual(len(results),  self.NUM_SNAPSHOT_MODELS)
 
     def assert_case_tables_equal(self, actual, expected):
-        if self.adapter_type == 'snowflake':
-            actual = actual.upper()
-            expected = expected.upper()
-
+        # this does something different on snowflake, but here it's just assertTablesEqual
         self.assertTablesEqual(actual, expected)
 
     def assert_expected(self):
@@ -69,34 +66,6 @@ class TestSimpleSnapshotFiles(BaseSimpleSnapshotTest):
 
         self.assert_expected()
 
-    @use_profile('snowflake')
-    def test__snowflake__simple_snapshot(self):
-        self.dbt_run_seed_snapshot()
-
-        self.assert_expected()
-
-        self.run_sql_file("invalidate_snowflake.sql")
-        self.run_sql_file("update.sql")
-
-        results = self.run_snapshot()
-        self.assertEqual(len(results),  self.NUM_SNAPSHOT_MODELS)
-
-        self.assert_expected()
-
-    @use_profile('redshift')
-    def test__redshift__simple_snapshot(self):
-        self.dbt_run_seed_snapshot()
-
-        self.assert_expected()
-
-        self.run_sql_file("invalidate_postgres.sql")
-        self.run_sql_file("update.sql")
-
-        results = self.run_snapshot()
-        self.assertEqual(len(results),  self.NUM_SNAPSHOT_MODELS)
-
-        self.assert_expected()
-
 
 class TestSimpleColumnSnapshotFiles(DBTIntegrationTest):
 
@@ -159,14 +128,6 @@ class TestSimpleColumnSnapshotFiles(DBTIntegrationTest):
     def test_postgres_renamed_source(self):
         self._run_snapshot_test()
 
-    @use_profile('snowflake')
-    def test_snowflake_renamed_source(self):
-        self._run_snapshot_test()
-
-    @use_profile('redshift')
-    def test_redshift_renamed_source(self):
-        self._run_snapshot_test()
-
     @use_profile('bigquery')
     def test_bigquery_renamed_source(self):
         self._run_snapshot_test()
@@ -419,9 +380,6 @@ class TestCrossDBSnapshotFiles(DBTIntegrationTest):
 
     @property
     def project_config(self):
-        if self.adapter_type == 'snowflake':
-            paths = ['test-snapshots-pg']
-        else:
         paths = ['test-snapshots-bq']
         return {
             'config-version': 2,
@@ -432,23 +390,6 @@ class TestCrossDBSnapshotFiles(DBTIntegrationTest):
     def run_snapshot(self):
         return self.run_dbt(['snapshot', '--vars', '{{"target_database": {}}}'.format(self.alternative_database)])
 
-    @use_profile('snowflake')
-    def test__snowflake__cross_snapshot(self):
-        self.run_sql_file("seed.sql")
-
-        results = self.run_snapshot()
-        self.assertEqual(len(results),  1)
-
-        self.assertTablesEqual("SNAPSHOT_EXPECTED", "SNAPSHOT_ACTUAL", table_b_db=self.alternative_database)
-
-        self.run_sql_file("invalidate_snowflake.sql")
-        self.run_sql_file("update.sql")
-
-        results = self.run_snapshot()
-        self.assertEqual(len(results),  1)
-
-        self.assertTablesEqual("SNAPSHOT_EXPECTED", "SNAPSHOT_ACTUAL", table_b_db=self.alternative_database)
-
     @use_profile('bigquery')
     def test__bigquery__cross_snapshot(self):
         self.run_sql_file("seed_bq.sql")
@@ -842,22 +783,9 @@ class TestSnapshotHardDelete(DBTIntegrationTest):
         self.run_sql_file('seed_bq.sql')
         self._test_snapshot_hard_delete()
 
-    @use_profile('snowflake')
-    def test__snowflake__snapshot_hard_delete(self):
-        self.run_sql_file('seed.sql')
-        self._test_snapshot_hard_delete()
-
-    @use_profile('redshift')
-    def test__redshift__snapshot_hard_delete(self):
-        self.run_sql_file('seed.sql')
-        self._test_snapshot_hard_delete()
-
     def _test_snapshot_hard_delete(self):
         self._snapshot()
 
-        if self.adapter_type == 'snowflake':
-            self.assertTablesEqual("SNAPSHOT_EXPECTED", "SNAPSHOT_ACTUAL")
-        else:
         self.assertTablesEqual("snapshot_expected", "snapshot_actual")
 
         self._invalidated_snapshot_datetime = None

test/integration/004_simple_snapshot_test/test_snapshot_check_cols.py

@@ -1,5 +1,4 @@
 from test.integration.base import DBTIntegrationTest, use_profile
-import dbt.exceptions
 
 
 class TestSimpleSnapshotFiles(DBTIntegrationTest):
@@ -35,11 +34,6 @@ class TestSimpleSnapshotFiles(DBTIntegrationTest):
     def assert_expected(self):
         self.run_dbt(['test', '--data', '--vars', 'version: 3'])
 
-    @use_profile('snowflake')
-    def test__snowflake__simple_snapshot(self):
-        self.test_snapshot_check_cols_cycle()
-        self.assert_expected()
-
     @use_profile('postgres')
     def test__postgres__simple_snapshot(self):
         self.test_snapshot_check_cols_cycle()
@@ -49,8 +43,3 @@ class TestSimpleSnapshotFiles(DBTIntegrationTest):
     def test__bigquery__simple_snapshot(self):
         self.test_snapshot_check_cols_cycle()
         self.assert_expected()
-
-    @use_profile('redshift')
-    def test__redshift__simple_snapshot(self):
-        self.test_snapshot_check_cols_cycle()
-        self.assert_expected()

test/integration/005_simple_seed_test/test_seed_type_override.py

@@ -60,66 +60,6 @@ class TestSimpleSeedColumnOverridePostgres(TestSimpleSeedColumnOverride):
         self.assertEqual(len(results),  10)
 
 
-class TestSimpleSeedColumnOverrideRedshift(TestSimpleSeedColumnOverride):
-    @property
-    def models(self):
-        return "models-rs"
-
-    @property
-    def profile_config(self):
-        return self.redshift_profile()
-
-    def seed_enabled_types(self):
-        return {
-            "id": "text",
-            "birthday": "date",
-        }
-
-    def seed_tricky_types(self):
-        return {
-            'id_str': 'text',
-            'looks_like_a_bool': 'text',
-            'looks_like_a_date': 'text',
-        }
-
-    @use_profile('redshift')
-    def test_redshift_simple_seed_with_column_override_redshift(self):
-        results = self.run_dbt(["seed", "--show"])
-        self.assertEqual(len(results),  2)
-        results = self.run_dbt(["test"])
-        self.assertEqual(len(results),  10)
-
-
-class TestSimpleSeedColumnOverrideSnowflake(TestSimpleSeedColumnOverride):
-    @property
-    def models(self):
-        return "models-snowflake"
-
-    def seed_enabled_types(self):
-        return {
-            "id": "FLOAT",
-            "birthday": "TEXT",
-        }
-
-    def seed_tricky_types(self):
-        return {
-            'id_str': 'TEXT',
-            'looks_like_a_bool': 'TEXT',
-            'looks_like_a_date': 'TEXT',
-        }
-
-    @property
-    def profile_config(self):
-        return self.snowflake_profile()
-
-    @use_profile('snowflake')
-    def test_snowflake_simple_seed_with_column_override_snowflake(self):
-        results = self.run_dbt(["seed", "--show"])
-        self.assertEqual(len(results),  2)
-        results = self.run_dbt(["test"])
-        self.assertEqual(len(results),  10)
-
-
 class TestSimpleSeedColumnOverrideBQ(TestSimpleSeedColumnOverride):
     @property
     def models(self):

test/integration/005_simple_seed_test/test_simple_seed.py

@@ -347,7 +347,3 @@ class TestSimpleBigSeedBatched(DBTIntegrationTest):
     def test_postgres_big_batched_seed(self):
         self.test_big_batched_seed()
     
-    @use_profile('snowflake')
-    def test_snowflake_big_batched_seed(self):
-        self.test_big_batched_seed()
-    

test/integration/007_graph_selection_tests/test_graph_selection.py

@@ -109,20 +109,6 @@ class TestGraphSelection(DBTIntegrationTest):
         self.assertNotIn('users_rollup_dependency', created_models)
         self.assert_correct_schemas()
 
-    @use_profile('snowflake')
-    def test__snowflake__specific_model(self):
-        self.run_sql_file("seed.sql")
-
-        results = self.run_dbt(['run', '--select', 'users'])
-        self.assertEqual(len(results),  1)
-
-        self.assertTablesEqual("SEED", "USERS")
-        created_models = self.get_models_in_schema()
-        self.assertFalse('USERS_ROLLUP' in created_models)
-        self.assertFalse('BASE_USERS' in created_models)
-        self.assertFalse('EMAILS' in created_models)
-        self.assert_correct_schemas()
-
     @use_profile('postgres')
     def test__postgres__specific_model_and_children(self):
         self.run_sql_file("seed.sql")
@@ -139,21 +125,6 @@ class TestGraphSelection(DBTIntegrationTest):
         self.assertNotIn('emails', created_models)
         self.assert_correct_schemas()
 
-    @use_profile('snowflake')
-    def test__snowflake__specific_model_and_children(self):
-        self.run_sql_file("seed.sql")
-
-        results = self.run_dbt(['run', '--select', 'users+'])
-        self.assertEqual(len(results),  4)
-
-        self.assertManyTablesEqual(
-            ["SEED", "USERS"],
-            ["SUMMARY_EXPECTED", "USERS_ROLLUP"]
-        )
-        created_models = self.get_models_in_schema()
-        self.assertFalse('BASE_USERS' in created_models)
-        self.assertFalse('EMAILS' in created_models)
-
     @use_profile('postgres')
     def test__postgres__specific_model_and_children_limited(self):
         self.run_sql_file("seed.sql")
@@ -184,22 +155,6 @@ class TestGraphSelection(DBTIntegrationTest):
         self.assertFalse('emails' in created_models)
         self.assert_correct_schemas()
 
-    @use_profile('snowflake')
-    def test__snowflake__specific_model_and_parents(self):
-        self.run_sql_file("seed.sql")
-
-        results = self.run_dbt(['run', '--select', '+users_rollup'])
-        self.assertEqual(len(results),  2)
-
-        self.assertManyTablesEqual(
-            ["SEED", "USERS"],
-            ["SUMMARY_EXPECTED", "USERS_ROLLUP"]
-        )
-
-        created_models = self.get_models_in_schema()
-        self.assertFalse('BASE_USERS' in created_models)
-        self.assertFalse('EMAILS' in created_models)
-
     @use_profile('postgres')
     def test__postgres__specific_model_and_parents_limited(self):
         self.run_sql_file("seed.sql")
@@ -230,21 +185,6 @@ class TestGraphSelection(DBTIntegrationTest):
         self.assertFalse('emails' in created_models)
         self.assert_correct_schemas()
 
-    @use_profile('snowflake')
-    def test__snowflake__specific_model_with_exclusion(self):
-        self.run_sql_file("seed.sql")
-
-        results = self.run_dbt(
-            ['run', '--select', '+users_rollup', '--exclude', 'users_rollup']
-        )
-        self.assertEqual(len(results),  1)
-
-        self.assertManyTablesEqual(["SEED", "USERS"])
-        created_models = self.get_models_in_schema()
-        self.assertFalse('BASE_USERS' in created_models)
-        self.assertFalse('USERS_ROLLUP' in created_models)
-        self.assertFalse('EMAILS' in created_models)
-
     @use_profile('postgres')
     def test__postgres__locally_qualified_name(self):
         results = self.run_dbt(['run', '--select', 'test.subdir'])
@@ -326,28 +266,6 @@ class TestGraphSelection(DBTIntegrationTest):
         self.assertEqual(len(results), 2)
         assert sorted([r.node.name for r in results]) == ['unique_users_id', 'unique_users_rollup_gender']
 
-
-    @use_profile('snowflake')
-    def test__snowflake__skip_intermediate(self):
-        self.run_sql_file("seed.sql")
-        results = self.run_dbt(['run', '--select', '@models/users.sql'])
-        # base_users, emails, users_rollup, users_rollup_dependency
-        self.assertEqual(len(results), 4)
-
-        # now re-run, skipping users_rollup
-        results = self.run_dbt(['run', '--select', '@users', '--exclude', 'users_rollup'])
-        self.assertEqual(len(results), 3)
-
-        # make sure that users_rollup_dependency and users don't interleave
-        users = [r for r in results if r.node.name == 'users'][0]
-        dep = [r for r in results if r.node.name == 'users_rollup_dependency'][0]
-        user_last_end = users.timing[1].completed_at
-        dep_first_start = dep.timing[0].started_at
-        self.assertTrue(
-            user_last_end <= dep_first_start,
-            'dependency started before its transitive parent ({} > {})'.format(user_last_end, dep_first_start)
-        )
-
     @use_profile('postgres')
     def test__postgres__concat(self):
         self.run_sql_file("seed.sql")

test/integration/009_data_tests_test/test_data_tests.py

@@ -56,26 +56,3 @@ class TestDataTests(DBTIntegrationTest):
         defined_tests = os.listdir(self.test_path)
         self.assertNotEqual(len(test_results), 0)
         self.assertEqual(len(test_results), len(defined_tests))
-
-    @use_profile('snowflake')
-    def test_snowflake_data_tests(self):
-        self.use_profile('snowflake')
-
-        self.run_sql_file("seed.sql")
-
-        results = self.run_dbt()
-        self.assertEqual(len(results), 1)
-        test_results = self.run_data_validations()
-
-        for result in test_results:
-            # assert that all deliberately failing tests actually fail
-            if 'fail' in result.node.name:
-                self.assertEqual(result.status, 'fail')
-                self.assertFalse(result.skipped)
-                self.assertTrue(result.failures > 0)
-
-            # assert that actual tests pass
-            else:
-                self.assertEqual(result.status, 'pass')
-                self.assertFalse(result.skipped)
-                self.assertEqual(result.failures, 0)

test/integration/012_deprecation_tests/test_deprecations.py

@@ -62,14 +62,6 @@ class TestAdapterMacroDeprecation(BaseTestDeprecations):
         exc_str = ' '.join(str(exc.exception).split())  # flatten all whitespace
         assert 'The "adapter_macro" macro has been deprecated' in exc_str
 
-    @use_profile('redshift')
-    def test_redshift_adapter_macro(self):
-        self.assertEqual(deprecations.active_deprecations, set())
-        # pick up the postgres macro
-        self.run_dbt()
-        expected = {'adapter-macro'}
-        self.assertEqual(expected, deprecations.active_deprecations)
-        
     @use_profile('bigquery')
     def test_bigquery_adapter_macro(self):
         self.assertEqual(deprecations.active_deprecations, set())
@@ -107,15 +99,6 @@ class TestAdapterMacroDeprecationPackages(BaseTestDeprecations):
         exc_str = ' '.join(str(exc.exception).split())  # flatten all whitespace
         assert 'The "adapter_macro" macro has been deprecated' in exc_str
 
-    @use_profile('redshift')
-    def test_redshift_adapter_macro_pkg(self):
-        self.assertEqual(deprecations.active_deprecations, set())
-        # pick up the postgres macro
-        self.assertEqual(deprecations.active_deprecations, set())
-        self.run_dbt()
-        expected = {'adapter-macro'}
-        self.assertEqual(expected, deprecations.active_deprecations)
-
     @use_profile('bigquery')
     def test_bigquery_adapter_macro_pkg(self):
         self.assertEqual(deprecations.active_deprecations, set())

test/integration/016_macro_tests/test_macros.py

@@ -95,20 +95,6 @@ class TestAdapterMacroNoDestination(DBTIntegrationTest):
         assert "In dispatch: No macro named 'dispatch_to_nowhere' found" in str(exc.value)
 
 
-class TestDispatchMacroUseParent(DBTIntegrationTest):
-    @property
-    def schema(self):
-        return "test_macros_016"
-
-    @property
-    def models(self):
-        return "dispatch-inheritance-models"
-
-    @use_profile('redshift')
-    def test_redshift_inherited_macro(self):
-        self.run_dbt(['run'])
-
-
 class TestMacroOverrideBuiltin(DBTIntegrationTest):
     @property
     def schema(self):
@@ -125,7 +111,6 @@ class TestMacroOverrideBuiltin(DBTIntegrationTest):
             'macro-paths': ['override-get-columns-macros'],
         }
 
-
     @use_profile('postgres')
     def test_postgres_overrides(self):
         # the first time, the model doesn't exist

test/integration/020_ephemeral_test/test_ephemeral.py

@@ -43,17 +43,6 @@ class TestEphemeralMulti(DBTIntegrationTest):
         expected_sql = "".join(expected_sql.split())
         self.assertEqual(sql_file, expected_sql)
 
-    @use_profile('snowflake')
-    def test__snowflake(self):
-        self.run_sql_file("seed.sql")
-
-        results = self.run_dbt()
-        self.assertEqual(len(results), 3)
-
-        self.assertManyTablesEqual(
-            ["SEED", "DEPENDENT", "DOUBLE_DEPENDENT", "SUPER_DEPENDENT"]
-        )
-
 
 class TestEphemeralNested(DBTIntegrationTest):
     @property

test/integration/021_concurrency_test/test_concurrency.py

@@ -37,19 +37,3 @@ class TestConcurrency(DBTIntegrationTest):
         self.assertTableDoesNotExist("skip")
 
         self.assertIn('PASS=5 WARN=0 ERROR=1 SKIP=1 TOTAL=7', output)
-
-    @use_profile('snowflake')
-    def test__snowflake__concurrency(self):
-        self.run_sql_file("seed.sql")
-
-        results = self.run_dbt(expect_pass=False)
-        self.assertEqual(len(results), 7)
-
-        self.assertManyTablesEqual(["SEED", "VIEW_MODEL", "DEP", "TABLE_A", "TABLE_B"])
-
-        self.run_sql_file("update.sql")
-
-        results = self.run_dbt(expect_pass=False)
-        self.assertEqual(len(results), 7)
-
-        self.assertManyTablesEqual(["SEED", "VIEW_MODEL", "DEP", "TABLE_A", "TABLE_B"])

test/integration/024_custom_schema_test/custom-db-macros/custom_db.sql

@@ -1,10 +0,0 @@
-
-{% macro generate_database_name(database_name, node) %}
-    {% if database_name == 'alt' %}
-        {{ env_var('SNOWFLAKE_TEST_ALT_DATABASE') }}
-    {% elif database_name %}
-        {{ database_name }}
-    {% else %}
-        {{ target.database }}
-    {% endif %}
-{% endmacro %}

test/integration/024_custom_schema_test/db-models/view_1.sql

@@ -1,3 +0,0 @@
-
-
-select * from {{ target.schema }}.seed

test/integration/024_custom_schema_test/db-models/view_2.sql

@@ -1,2 +0,0 @@
-{{ config(database='alt') }}
-select * from {{ ref('view_1') }}

test/integration/024_custom_schema_test/db-models/view_3.sql

@@ -1,30 +0,0 @@
-
-{{ config(database='alt', materialized='table') }}
-
-
-with v1 as (
-
-    select * from {{ ref('view_1') }}
-
-),
-
-v2 as (
-
-    select * from {{ ref('view_2') }}
-
-),
-
-combined as (
-
-    select last_name from v1
-    union all
-    select last_name from v2
-
-)
-
-select
-    last_name,
-    count(*) as count
-
-from combined
-group by 1

test/integration/024_custom_schema_test/test_custom_database.py

@@ -1,39 +0,0 @@
-from test.integration.base import DBTIntegrationTest, use_profile
-
-
-class TestOverrideDatabase(DBTIntegrationTest):
-    setup_alternate_db = True
-
-    @property
-    def schema(self):
-        return "custom_schema_024"
-
-    @property
-    def models(self):
-        return "db-models"
-
-    @property
-    def project_config(self):
-        return {
-            'config-version': 2,
-            'macro-paths': ['custom-db-macros'],
-        }
-
-    @use_profile('snowflake')
-    def test_snowflake_override_generate_db_name(self):
-        self.run_sql_file('seed.sql')
-        self.assertTableDoesExist('SEED', schema=self.unique_schema(), database=self.default_database)
-        self.assertTableDoesExist('AGG', schema=self.unique_schema(), database=self.default_database)
-
-        results = self.run_dbt()
-        self.assertEqual(len(results), 3)
-
-        self.assertTableDoesExist('VIEW_1', schema=self.unique_schema(), database=self.default_database)
-        self.assertTableDoesExist('VIEW_2', schema=self.unique_schema(), database=self.alternative_database)
-        self.assertTableDoesExist('VIEW_3', schema=self.unique_schema(), database=self.alternative_database)
-
-        # not overridden
-        self.assertTablesEqual('SEED', 'VIEW_1', table_b_db=self.default_database)
-        # overridden
-        self.assertTablesEqual('SEED', 'VIEW_2', table_b_db=self.alternative_database)
-        self.assertTablesEqual('AGG', 'VIEW_3', table_b_db=self.alternative_database)

test/integration/024_custom_schema_test/test_custom_schema.py

@@ -131,59 +131,6 @@ class TestCustomProjectSchemaWithPrefix(DBTIntegrationTest):
         self.assertTablesEqual("agg", "view_3", schema, self.xf_schema())
 
 
-class TestCustomProjectSchemaWithPrefixSnowflake(DBTIntegrationTest):
-    def setUp(self):
-        super().setUp()
-        self._created_schemas.add(
-            self._get_schema_fqn(self.default_database, self.v1_schema())
-        )
-        self._created_schemas.add(
-            self._get_schema_fqn(self.default_database, self.v2_schema())
-        )
-        self._created_schemas.add(
-            self._get_schema_fqn(self.default_database, self.xf_schema())
-        )
-
-    @property
-    def schema(self):
-        return "sf_custom_prefix_024"
-
-    @property
-    def models(self):
-        return "models"
-
-    @property
-    def project_config(self):
-        return {
-            'config-version': 2,
-            "models": {
-                "schema": "dbt_test"
-            }
-        }
-
-    def v1_schema(self):
-        return f"{self.unique_schema()}_DBT_TEST"
-
-    def v2_schema(self):
-        return f"{self.unique_schema()}_CUSTOM"
-
-    def xf_schema(self):
-        return f"{self.unique_schema()}_TEST"
-
-    @use_profile('snowflake')
-    def test__snowflake__custom_schema_with_prefix(self):
-        self.run_sql_file("seed.sql")
-
-        results = self.run_dbt()
-        self.assertEqual(len(results), 3)
-
-        schema = self.unique_schema().upper()
-
-        self.assertTablesEqual("SEED", "VIEW_1", schema, self.v1_schema())
-        self.assertTablesEqual("SEED", "VIEW_2", schema, self.v2_schema())
-        self.assertTablesEqual("AGG", "VIEW_3", schema, self.xf_schema())
-
-
 class TestCustomSchemaWithCustomMacro(DBTIntegrationTest):
     def setUp(self):
         super().setUp()

test/integration/026_aliases_test/test_aliases.py

@@ -39,12 +39,6 @@ class TestAliases(DBTIntegrationTest):
         self.assertEqual(len(results), 4)
         self.run_dbt(['test'])
 
-    @use_profile('snowflake')
-    def test__alias_model_name_snowflake(self):
-        results = self.run_dbt(['run'])
-        self.assertEqual(len(results), 4)
-        self.run_dbt(['test'])
-
 
 class TestAliasErrors(DBTIntegrationTest):
     @property

test/integration/029_docs_generate_tests/models/second_model.sql

@@ -1,12 +1,7 @@
-{%- if adapter.type() == 'snowflake' -%}
-    {%- set schema_suffix = 'TEST' -%}
-{%- else -%}
-    {%- set schema_suffix = 'test' -%}
-{%- endif -%}
 {{
     config(
         materialized='view',
-        schema=schema_suffix,
+        schema='test',
     )
 }}
 

test/integration/029_docs_generate_tests/test_docs_generate.py

@@ -99,16 +99,12 @@ class TestDocsGenerate(DBTIntegrationTest):
     setup_alternate_db = True
 
     def adapter_case(self, value):
-        if self.adapter_type == 'snowflake':
-            return value.upper()
-        else:
         return value.lower()
 
     def setUp(self):
         super().setUp()
         self.maxDiff = None
         self.alternate_schema = self.unique_schema() + '_test'
-        if self.adapter_type == 'snowflake':
         self.alternate_schema = self.alternate_schema.upper()
 
         self._created_schemas.add(self.alternate_schema)
@@ -181,105 +177,6 @@ class TestDocsGenerate(DBTIntegrationTest):
             },
         }
 
-    def _redshift_stats(self):
-        return {
-            "has_stats": {
-                "id": "has_stats",
-                "label": "Has Stats?",
-                "value": True,
-                "description": "Indicates whether there are statistics for this table",
-                "include": False
-            },
-            "encoded": {
-                "id": "encoded",
-                "label": "Encoded",
-                "value": AnyStringWith('Y'),
-                "description": "Indicates whether any column in the table has compression encoding defined.",
-                "include": True
-            },
-            "diststyle": {
-                "id": "diststyle",
-                "label": "Dist Style",
-                "value": AnyStringWith('AUTO'),
-                "description": "Distribution style or distribution key column, if key distribution is defined.",
-                "include": True
-            },
-            "max_varchar": {
-                "id": "max_varchar",
-                "label": "Max Varchar",
-                "value": AnyFloat(),
-                "description": "Size of the largest column that uses a VARCHAR data type.",
-                "include": True
-            },
-            "size": {
-                "id": "size",
-                "label": "Approximate Size",
-                "value": AnyFloat(),
-                "description": "Approximate size of the table, calculated from a count of 1MB blocks",
-                "include": True
-            },
-            'sortkey1': {
-                'id': 'sortkey1',
-                'label': 'Sort Key 1',
-                'value': AnyStringWith('AUTO'),
-                'description': 'First column in the sort key.',
-                'include': True,
-            },
-            "pct_used": {
-                "id": "pct_used",
-                "label": "Disk Utilization",
-                "value": AnyFloat(),
-                "description": "Percent of available space that is used by the table.",
-                "include": True
-            },
-            "stats_off": {
-                "id": "stats_off",
-                "label": "Stats Off",
-                "value": AnyFloat(),
-                "description": "Number that indicates how stale the table statistics are; 0 is current, 100 is out of date.",
-                "include": True
-            },
-            "rows": {
-                "id": "rows",
-                "label": "Approximate Row Count",
-                "value": AnyFloat(),
-                "description": "Approximate number of rows in the table. This value includes rows marked for deletion, but not yet vacuumed.",
-                "include": True
-            },
-        }
-
-    def _snowflake_stats(self):
-        return {
-            'has_stats': {
-                'id': 'has_stats',
-                'label': 'Has Stats?',
-                'value': True,
-                'description': 'Indicates whether there are statistics for this table',
-                'include': False,
-            },
-            'bytes': {
-                'id': 'bytes',
-                'label': 'Approximate Size',
-                'value': AnyFloat(),
-                'description': 'Approximate size of the table as reported by Snowflake',
-                'include': True,
-            },
-            'last_modified': {
-                'id': 'last_modified',
-                'label': 'Last Modified',
-                'value': AnyString(),
-                'description': 'The timestamp for last update/change',
-                'include': True,
-            },
-            'row_count': {
-                'id': 'row_count',
-                'label': 'Row Count',
-                'value': 1.0,
-                'description': 'An approximate count of rows in this table',
-                'include': True,
-            }
-        }
-
     def _bigquery_stats(self, is_table, partition=None, cluster=None):
         stats = {}
 
@@ -450,13 +347,10 @@ class TestDocsGenerate(DBTIntegrationTest):
         )
 
     def get_role(self):
-        if self.adapter_type in {'postgres', 'redshift'}:
+        if self.adapter_type in {'postgres'}:
             profile = self.get_profile(self.adapter_type)
             target_name = profile['test']['target']
             return profile['test']['outputs'][target_name]['user']
-        elif self.adapter_type == 'snowflake':
-            return self.run_sql('select current_role()', fetch='one')[0]
-        else:  # bigquery, presto, other dbs that have no 'role'
         return None
 
     def expected_postgres_references_catalog(self):
@@ -570,20 +464,6 @@ class TestDocsGenerate(DBTIntegrationTest):
             },
         }
 
-    def expected_snowflake_catalog(self, case_columns=False):
-        return self._expected_catalog(
-            id_type='NUMBER',
-            text_type='TEXT',
-            time_type='TIMESTAMP_NTZ',
-            view_type='VIEW',
-            table_type='BASE TABLE',
-            model_stats=self._no_stats(),
-            seed_stats=self._snowflake_stats(),
-            case=lambda x: x.upper(),
-            model_database=self.alternative_database,
-            case_columns=case_columns,
-        )
-
     def expected_bigquery_catalog(self):
         return self._expected_catalog(
             id_type='INT64',
@@ -761,115 +641,6 @@ class TestDocsGenerate(DBTIntegrationTest):
             'sources': {},
         }
 
-    def expected_redshift_catalog(self):
-        return self._expected_catalog(
-            id_type='integer',
-            text_type=AnyStringWith('character varying'),
-            time_type='timestamp without time zone',
-            view_type='VIEW',
-            table_type='BASE TABLE',
-            model_stats=self._no_stats(),
-            seed_stats=self._redshift_stats(),
-        )
-
-    def expected_redshift_incremental_catalog(self):
-        my_schema_name = self.unique_schema()
-        role = self.get_role()
-        return {
-            'nodes': {
-                'model.test.model': {
-                    'unique_id': 'model.test.model',
-                    'metadata': {
-                        'schema': my_schema_name,
-                        'database': self.default_database,
-                        'name': 'model',
-                        'type': 'LATE BINDING VIEW',
-                        'comment': None,
-                        'owner': role,
-                    },
-                    # incremental views have no stats
-                    'stats': self._no_stats(),
-                    'columns': {
-                        'id': {
-                            'name': 'id',
-                            'index': 1,
-                            'type': 'integer',
-                            'comment': None,
-                        },
-                        'first_name': {
-                            'name': 'first_name',
-                            'index': 2,
-                            'type': 'character varying(5)',
-                            'comment': None,
-                        },
-                        'email': {
-                            'name': 'email',
-                            'index': 3,
-                            'type': 'character varying(23)',
-                            'comment': None,
-                        },
-                        'ip_address': {
-                            'name': 'ip_address',
-                            'index': 4,
-                            'type': 'character varying(14)',
-                            'comment': None,
-                        },
-                        'updated_at': {
-                            'name': 'updated_at',
-                            'index': 5,
-                            'type': 'timestamp without time zone',
-                            'comment': None,
-                        },
-                    },
-                },
-                'seed.test.seed': {
-                    'unique_id': 'seed.test.seed',
-                    'metadata': {
-                        'schema': my_schema_name,
-                        'database': self.default_database,
-                        'name': 'seed',
-                        'type': 'BASE TABLE',
-                        'comment': None,
-                        'owner': role,
-                    },
-                    'stats': self._redshift_stats(),
-                    'columns': {
-                        'id': {
-                            'name': 'id',
-                            'index': 1,
-                            'type': 'integer',
-                            'comment': None,
-                        },
-                        'first_name': {
-                            'name': 'first_name',
-                            'index': 2,
-                            'type': 'character varying(5)',
-                            'comment': None,
-                        },
-                        'email': {
-                            'name': 'email',
-                            'index': 3,
-                            'type': 'character varying(23)',
-                            'comment': None,
-                        },
-                        'ip_address': {
-                            'name': 'ip_address',
-                            'index': 4,
-                            'type': 'character varying(14)',
-                            'comment': None,
-                        },
-                        'updated_at': {
-                            'name': 'updated_at',
-                            'index': 5,
-                            'type': 'timestamp without time zone',
-                            'comment': None,
-                        },
-                    },
-                },
-            },
-            'sources': {},
-        }
-
     def verify_catalog(self, expected):
         self.assertTrue(os.path.exists('./target/catalog.json'))
 
@@ -1084,7 +855,7 @@ class TestDocsGenerate(DBTIntegrationTest):
             target_schema=self.alternate_schema
         )
 
-        quote_database = quote_schema = self.adapter_type != 'snowflake'
+        quote_database = quote_schema = True
         relation_name_node_format = self._relation_name_format(
             quote_database, quote_schema, quote_model
         )
@@ -2688,242 +2459,6 @@ class TestDocsGenerate(DBTIntegrationTest):
             self._quote("{2}") if quote_identifier else '{2}',
         ))
 
-    def expected_redshift_incremental_view_manifest(self):
-        model_sql_path = self.dir('rs_models/model.sql')
-        my_schema_name = self.unique_schema()
-        seed_path = self.dir('seed/seed.csv')
-        snapshot_path = self.dir('snapshot/snapshot_seed.sql')
-
-        return {
-            'dbt_schema_version': 'https://schemas.getdbt.com/dbt/manifest/v3.json',
-            'dbt_version': dbt.version.__version__,
-            'nodes': {
-                'model.test.model': {
-                    'compiled_path': Normalized('target/compiled/test/rs_models/model.sql'),
-                    'build_path': None,
-                    'created_at': ANY,
-                    'name': 'model',
-                    'root_path': self.test_root_realpath,
-                    'relation_name': '"{0}"."{1}".model'.format(
-                        self.default_database, my_schema_name
-                    ),
-                    'resource_type': 'model',
-                    'path': 'model.sql',
-                    'original_file_path': model_sql_path,
-                    'package_name': 'test',
-                    'raw_sql': LineIndifferent(_read_file(model_sql_path).rstrip('\r\n')),
-                    'refs': [['seed']],
-                    'sources': [],
-                    'depends_on': {
-                        'nodes': ['seed.test.seed'],
-                        'macros': [],
-                    },
-                    'unique_id': 'model.test.model',
-                    'fqn': ['test', 'model'],
-                    'tags': [],
-                    'meta': {},
-                    'config': self.rendered_model_config(bind=False),
-                    'schema': my_schema_name,
-                    'database': self.default_database,
-                    'alias': 'model',
-                    'deferred': False,
-                    'description': 'The test model',
-                    'columns': {
-                        'id': {
-                            'name': 'id',
-                            'description': 'The user ID number',
-                            'data_type': None,
-                            'meta': {},
-                            'quote': None,
-                            'tags': [],
-                        },
-                        'first_name': {
-                            'name': 'first_name',
-                            'description': "The user's first name",
-                            'data_type': None,
-                            'meta': {},
-                            'quote': None,
-                            'tags': [],
-                        },
-                        'email': {
-                            'name': 'email',
-                            'description': "The user's email",
-                            'data_type': None,
-                            'meta': {},
-                            'quote': None,
-                            'tags': [],
-                        },
-                        'ip_address': {
-                            'name': 'ip_address',
-                            'description': "The user's IP address",
-                            'data_type': None,
-                            'meta': {},
-                            'quote': None,
-                            'tags': [],
-                        },
-                        'updated_at': {
-                            'name': 'updated_at',
-                            'description': "The last time this user's email was updated",
-                            'data_type': None,
-                            'meta': {},
-                            'quote': None,
-                            'tags': [],
-                        },
-                    },
-                    'patch_path': 'test://' + self.dir('rs_models/schema.yml'),
-                    'docs': {'show': True},
-                    'compiled': True,
-                    'compiled_sql': ANY,
-                    'extra_ctes_injected': True,
-                    'extra_ctes': [],
-                    'checksum': self._checksum_file(model_sql_path),
-                    'unrendered_config': self.unrendered_model_config(bind=False, materialized='view'),
-                },
-                'seed.test.seed': {
-                    'compiled_path': None,
-                    'build_path': None,
-                    'created_at': ANY,
-                    'patch_path': 'test://' + self.dir('seed/schema.yml'),
-                    'path': 'seed.csv',
-                    'name': 'seed',
-                    'root_path': self.test_root_realpath,
-                    'relation_name': '"{0}"."{1}".seed'.format(
-                        self.default_database, my_schema_name
-                    ),
-                    'resource_type': 'seed',
-                    'raw_sql': '',
-                    'package_name': 'test',
-                    'original_file_path': seed_path,
-                    'refs': [],
-                    'sources': [],
-                    'depends_on': {
-                        'nodes': [],
-                        'macros': [],
-                    },
-                    'unique_id': 'seed.test.seed',
-                    'fqn': ['test', 'seed'],
-                    'tags': [],
-                    'meta': {},
-                    'config': self.rendered_seed_config(),
-                    'schema': my_schema_name,
-                    'database': self.default_database,
-                    'alias': 'seed',
-                    'columns': {
-                        'id': {
-                            'name': 'id',
-                            'description': 'The user ID number',
-                            'data_type': None,
-                            'meta': {},
-                            'quote': None,
-                            'tags': [],
-                        },
-                        'first_name': {
-                            'name': 'first_name',
-                            'description': "The user's first name",
-                            'data_type': None,
-                            'meta': {},
-                            'quote': None,
-                            'tags': [],
-                        },
-                        'email': {
-                            'name': 'email',
-                            'description': "The user's email",
-                            'data_type': None,
-                            'meta': {},
-                            'quote': None,
-                            'tags': [],
-                        },
-                        'ip_address': {
-                            'name': 'ip_address',
-                            'description': "The user's IP address",
-                            'data_type': None,
-                            'meta': {},
-                            'quote': None,
-                            'tags': [],
-                        },
-                        'updated_at': {
-                            'name': 'updated_at',
-                            'description': "The last time this user's email was updated",
-                            'data_type': None,
-                            'meta': {},
-                            'quote': None,
-                            'tags': [],
-                        },
-                    },
-                    'deferred': False,
-                    'description': 'The test seed',
-                    'docs': {'show': True},
-                    'compiled': True,
-                    'compiled_sql': ANY,
-                    'extra_ctes_injected': True,
-                    'extra_ctes': [],
-                    'checksum': self._checksum_file(seed_path),
-                    'unrendered_config': self.unrendered_seed_config(),
-                },
-                'snapshot.test.snapshot_seed': {
-                    'alias': 'snapshot_seed',
-                    'compiled_path': None,
-                    'build_path': None,
-                    'created_at': ANY,
-                    'checksum': self._checksum_file(snapshot_path),
-                    'columns': {},
-                    'compiled': True,
-                    'compiled_sql': ANY,
-                    'config': self.rendered_snapshot_config(
-                        target_schema=self.alternate_schema
-                    ),
-                    'database': self.default_database,
-                    'deferred': False,
-                    'depends_on': {'macros': [],
-                                   'nodes': ['seed.test.seed']},
-                    'description': '',
-                    'docs': {'show': True},
-                    'extra_ctes': [],
-                    'extra_ctes_injected': True,
-                    'fqn': ['test', 'snapshot_seed', 'snapshot_seed'],
-                    'meta': {},
-                    'name': 'snapshot_seed',
-                    'original_file_path': snapshot_path,
-                    'package_name': 'test',
-                    'patch_path': None,
-                    'path': 'snapshot_seed.sql',
-                    'raw_sql': ANY,
-                    'refs': [['seed']],
-                    'relation_name': '"{0}"."{1}".snapshot_seed'.format(
-                        self.default_database, self.alternate_schema
-                    ),
-                    'resource_type': 'snapshot',
-                    'root_path': self.test_root_realpath,
-                    'schema': self.alternate_schema,
-                    'sources': [],
-                    'tags': [],
-                    'unique_id': 'snapshot.test.snapshot_seed',
-                    'unrendered_config': self.unrendered_snapshot_config(
-                        target_schema=self.alternate_schema
-                    )}
-            },
-            'sources': {},
-            'exposures': {},
-            'selectors': {},
-            'parent_map': {
-                'model.test.model': ['seed.test.seed'],
-                'seed.test.seed': [],
-                'snapshot.test.snapshot_seed': ['seed.test.seed']
-            },
-            'child_map': {
-                'model.test.model': [],
-                'seed.test.seed': ['model.test.model',
-                                   'snapshot.test.snapshot_seed'],
-                'snapshot.test.snapshot_seed': []
-            },
-            'docs': {
-                'dbt.__overview__': ANY,
-                'test.macro_info': ANY,
-                'test.macro_arg_info': ANY,
-            },
-            'disabled': [],
-        }
-
     def verify_metadata(self, metadata, dbt_schema_version):
         assert 'generated_at' in metadata
         self.assertBetween(metadata['generated_at'],
@@ -3155,38 +2690,6 @@ class TestDocsGenerate(DBTIntegrationTest):
 
         assert not os.path.exists('./target/non-existent-assets')
 
-    @use_profile('snowflake')
-    def test__snowflake__run_and_generate(self):
-        self.run_and_generate()
-
-        self.verify_catalog(self.expected_snowflake_catalog())
-        self.verify_manifest(self.expected_seeded_manifest())
-        self.verify_run_results(self.expected_run_results())
-
-    @use_profile('snowflake')
-    def test__snowflake__run_and_generate_ignore_quoting_parameter(self):
-        # with optional adapters, this package could easily just not exist!
-        # accordingly, only run it when we think snowflake things should work
-        from dbt.adapters.snowflake import connections as snowflake_conn
-        old_connect = snowflake_conn.snowflake.connector.connect
-
-        def connect(*args, **kwargs):
-            kwargs['session_parameters'] = {
-                'QUOTED_IDENTIFIERS_IGNORE_CASE': True
-            }
-            return old_connect(*args, **kwargs)
-
-        with patch.object(snowflake_conn.snowflake.connector, 'connect', connect):
-            self.run_and_generate({
-                'quoting': {
-                    'identifier': True,
-                }
-            })
-
-        self.verify_catalog(self.expected_snowflake_catalog(case_columns=True))
-        self.verify_manifest(self.expected_seeded_manifest(quote_model=True))
-        self.verify_run_results(self.expected_run_results())
-
     @use_profile('bigquery')
     def test__bigquery__run_and_generate(self):
         self.run_and_generate()
@@ -3205,35 +2708,6 @@ class TestDocsGenerate(DBTIntegrationTest):
         self.verify_catalog(self.expected_bigquery_complex_catalog())
         self.verify_manifest(self.expected_bigquery_complex_manifest())
 
-    @use_profile('redshift')
-    def test__redshift__run_and_generate(self):
-        self.run_and_generate(alternate_db=self.default_database)
-        self.verify_catalog(self.expected_redshift_catalog())
-        self.verify_manifest(self.expected_seeded_manifest(
-            model_database=self.default_database
-        ))
-        self.verify_run_results(self.expected_run_results())
-
-    @use_profile('redshift')
-    def test__redshift__incremental_view(self):
-        self.run_and_generate(
-            {'source-paths': [self.dir('rs_models')]},
-            alternate_db=self.default_database,
-            model_count=1,
-        )
-        self.verify_catalog(self.expected_redshift_incremental_catalog())
-        self.verify_manifest(
-            self.expected_redshift_incremental_view_manifest())
-
-    @use_profile('presto')
-    def test__presto__run_and_generate(self):
-        self.run_and_generate(alternate_db=self.default_database)
-        self.verify_catalog(self.expected_presto_catalog())
-        self.verify_manifest(self.expected_seeded_manifest(
-            model_database=self.default_database
-        ))
-        self.verify_run_results(self.expected_run_results())
-
 
 class TestDocsGenerateMissingSchema(DBTIntegrationTest):
     @property