Compare commits

...

11 Commits

45 changed files with 201 additions and 1396 deletions

View File

@@ -0,0 +1,6 @@
kind: Features
body: Stand-alone Python module for PostgresColumn
time: 2023-01-27T16:28:12.212427-08:00
custom:
Author: nssalian
Issue: "6772"

View File

@@ -0,0 +1,6 @@
kind: Under the Hood
body: '[CT-1841] Convert custom target test to Pytest'
time: 2023-01-26T16:47:41.198714-08:00
custom:
Author: aranke
Issue: "6638"

View File

@@ -0,0 +1,6 @@
kind: Under the Hood
body: warn_error/warn_error_options mutual exclusivity in click
time: 2023-01-30T18:09:17.240662-05:00
custom:
Author: michelleark
Issue: "6579"

View File

@@ -88,7 +88,7 @@ custom:
footerFormat: |
{{- $contributorDict := dict }}
{{- /* any names added to this list should be all lowercase for later matching purposes */}}
{{- $core_team := list "michelleark" "peterallenwebb" "emmyoop" "nathaniel-may" "gshank" "leahwicz" "chenyulinx" "stu-k" "iknox-fa" "versusfacit" "mcknight-42" "jtcohen6" "aranke" "dependabot[bot]" "snyk-bot" "colin-rogers-dbt" }}
{{- $core_team := list "michelleark" "peterallenwebb" "emmyoop" "nathaniel-may" "gshank" "leahwicz" "chenyulinx" "stu-k" "iknox-fa" "versusfacit" "mcknight-42" "jtcohen6" "aranke" "dependabot[bot]" "snyk-bot" "colin-rogers-dbt" "nssalian" }}
{{- range $change := .Changes }}
{{- $authorList := splitList " " $change.Custom.Author }}
{{- /* loop through all authors for a single changelog */}}

View File

@@ -178,8 +178,8 @@ jobs:
nightly_release: ${{ inputs.nightly_release }}
secrets:
AWS_ACCESS_KEY_ID: ${{ secrets.PRODUCTION_AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.PRODUCTION_AWS_SECRET_ACCESS_KEY }}
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
github-release:
name: GitHub Release
@@ -212,7 +212,7 @@ jobs:
slack-notification:
name: Slack Notification
if: ${{ failure() }}
if: ${{ failure() && (!inputs.test_run || inputs.nightly_release) }}
needs:
[

View File

@@ -5,9 +5,9 @@ from dataclasses import dataclass
from importlib import import_module
from multiprocessing import get_context
from pprint import pformat as pf
from typing import Set
from typing import Set, List
from click import Context, get_current_context
from click import Context, get_current_context, BadOptionUsage
from click.core import ParameterSource
from dbt.config.profile import read_user_config
@@ -59,12 +59,15 @@ class Flags:
# Overwrite default assignments with user config if available
if user_config:
param_assigned_from_default_copy = params_assigned_from_default.copy()
for param_assigned_from_default in params_assigned_from_default:
user_config_param_value = getattr(user_config, param_assigned_from_default, None)
if user_config_param_value is not None:
object.__setattr__(
self, param_assigned_from_default.upper(), user_config_param_value
)
param_assigned_from_default_copy.remove(param_assigned_from_default)
params_assigned_from_default = param_assigned_from_default_copy
# Hard coded flags
object.__setattr__(self, "WHICH", invoked_subcommand_name or ctx.info_name)
@@ -78,6 +81,10 @@ class Flags:
if os.getenv("DO_NOT_TRACK", "").lower() in ("1", "t", "true", "y", "yes")
else True,
)
# Check mutual exclusivity once all flags are set
self._assert_mutually_exclusive(
params_assigned_from_default, ["WARN_ERROR", "WARN_ERROR_OPTIONS"]
)
# Support lower cased access for legacy code
params = set(
@@ -88,3 +95,20 @@ class Flags:
def __str__(self) -> str:
return str(pf(self.__dict__))
def _assert_mutually_exclusive(
self, params_assigned_from_default: Set[str], group: List[str]
) -> None:
"""
Ensure no elements from group are simultaneously provided by a user, as inferred from params_assigned_from_default.
Raises click.UsageError if any two elements from group are simultaneously provided by a user.
"""
set_flag = None
for flag in group:
flag_set_by_user = flag.lower() not in params_assigned_from_default
if flag_set_by_user and set_flag:
raise BadOptionUsage(
flag.lower(), f"{flag.lower()}: not allowed with argument {set_flag.lower()}"
)
elif flag_set_by_user:
set_flag = flag

View File

@@ -398,7 +398,7 @@ warn_error = click.option(
envvar="DBT_WARN_ERROR",
help="If dbt would normally warn, instead raise an exception. Examples include --select that selects nothing, deprecations, configurations with no associated models, invalid test configurations, and missing sources/refs in tests.",
default=None,
flag_value=True,
is_flag=True,
)
warn_error_options = click.option(

View File

@@ -168,11 +168,7 @@ def msg_to_dict(msg: EventMsg) -> dict:
def warn_or_error(event, node=None):
# TODO: resolve this circular import when flags.WARN_ERROR_OPTIONS is WarnErrorOptions type via click CLI.
from dbt.helper_types import WarnErrorOptions
warn_error_options = WarnErrorOptions.from_yaml_string(flags.WARN_ERROR_OPTIONS)
if flags.WARN_ERROR or warn_error_options.includes(type(event).__name__):
if flags.WARN_ERROR or flags.WARN_ERROR_OPTIONS.includes(type(event).__name__):
# TODO: resolve this circular import when at top
from dbt.exceptions import EventCompilationError

View File

@@ -9,6 +9,8 @@ if os_name != "nt":
from pathlib import Path
from typing import Optional
from dbt.helper_types import WarnErrorOptions
# PROFILES_DIR must be set before the other flags
# It also gets set in main.py and in set_from_args because the rpc server
# doesn't go through exactly the same main arg processing.
@@ -46,7 +48,7 @@ USE_COLORS = None
USE_EXPERIMENTAL_PARSER = None
VERSION_CHECK = None
WARN_ERROR = None
WARN_ERROR_OPTIONS = None
WARN_ERROR_OPTIONS = WarnErrorOptions(include=[])
WHICH = None
WRITE_JSON = None
@@ -170,7 +172,13 @@ def set_from_args(args, user_config):
USE_EXPERIMENTAL_PARSER = get_flag_value("USE_EXPERIMENTAL_PARSER", args, user_config)
VERSION_CHECK = get_flag_value("VERSION_CHECK", args, user_config)
WARN_ERROR = get_flag_value("WARN_ERROR", args, user_config)
WARN_ERROR_OPTIONS = get_flag_value("WARN_ERROR_OPTIONS", args, user_config)
warn_error_options_str = get_flag_value("WARN_ERROR_OPTIONS", args, user_config)
from dbt.cli.option_types import WarnErrorOptionsType
# Converting to WarnErrorOptions for consistency with dbt/cli/flags.py
WARN_ERROR_OPTIONS = WarnErrorOptionsType().convert(warn_error_options_str, None, None)
WRITE_JSON = get_flag_value("WRITE_JSON", args, user_config)
_check_mutually_exclusive(["WARN_ERROR", "WARN_ERROR_OPTIONS"], args, user_config)

View File

@@ -123,22 +123,6 @@ class IncludeExclude(dbtClassMixin):
class WarnErrorOptions(IncludeExclude):
# TODO: this method can be removed once the click CLI is in use
@classmethod
def from_yaml_string(cls, warn_error_options_str: Optional[str]):
# TODO: resolve circular import
from dbt.config.utils import parse_cli_yaml_string
warn_error_options_str = (
str(warn_error_options_str) if warn_error_options_str is not None else "{}"
)
warn_error_options = parse_cli_yaml_string(warn_error_options_str, "warn-error-options")
return cls(
include=warn_error_options.get("include", []),
exclude=warn_error_options.get("exclude", []),
)
def _validate_items(self, items: List[str]):
valid_exception_names = set(
[name for name, cls in dbt_event_types.__dict__.items() if isinstance(cls, type)]

View File

@@ -1,7 +1,7 @@
# these are mostly just exports, #noqa them so flake8 will be happy
from dbt.adapters.postgres.connections import PostgresConnectionManager # noqa
from dbt.adapters.postgres.connections import PostgresCredentials
from dbt.adapters.postgres.relation import PostgresColumn # noqa
from dbt.adapters.postgres.column import PostgresColumn # noqa
from dbt.adapters.postgres.relation import PostgresRelation # noqa: F401
from dbt.adapters.postgres.impl import PostgresAdapter

View File

@@ -0,0 +1,12 @@
from dbt.adapters.base import Column
class PostgresColumn(Column):
@property
def data_type(self):
# on postgres, do not convert 'text' or 'varchar' to 'varchar()'
if self.dtype.lower() == "text" or (
self.dtype.lower() == "character varying" and self.char_size is None
):
return self.dtype
return super().data_type

View File

@@ -5,7 +5,7 @@ from dbt.adapters.base.meta import available
from dbt.adapters.base.impl import AdapterConfig
from dbt.adapters.sql import SQLAdapter
from dbt.adapters.postgres import PostgresConnectionManager
from dbt.adapters.postgres import PostgresColumn
from dbt.adapters.postgres.column import PostgresColumn
from dbt.adapters.postgres import PostgresRelation
from dbt.dataclass_schema import dbtClassMixin, ValidationError
from dbt.exceptions import (

View File

@@ -1,4 +1,3 @@
from dbt.adapters.base import Column
from dataclasses import dataclass
from dbt.adapters.base.relation import BaseRelation
from dbt.exceptions import DbtRuntimeError
@@ -21,14 +20,3 @@ class PostgresRelation(BaseRelation):
def relation_max_name_length(self):
return 63
class PostgresColumn(Column):
@property
def data_type(self):
# on postgres, do not convert 'text' or 'varchar' to 'varchar()'
if self.dtype.lower() == "text" or (
self.dtype.lower() == "character varying" and self.char_size is None
):
return self.dtype
return super().data_type

View File

@@ -6,5 +6,4 @@ env_files =
test.env
testpaths =
test/unit
test/integration
tests/functional

View File

@@ -1,4 +0,0 @@
create table {schema}.incremental__dbt_tmp as (
select 1 as id
);

View File

@@ -1,17 +0,0 @@
{% docs my_model_doc %}
Alt text about the model
{% enddocs %}
{% docs my_model_doc__id %}
The user ID number with alternative text
{% enddocs %}
The following doc is never used, which should be fine.
{% docs my_model_doc__first_name %}
The user's first name - don't show this text!
{% enddocs %}
This doc is referenced by its full name
{% docs my_model_doc__last_name %}
The user's last name in this other file
{% enddocs %}

View File

@@ -1,7 +0,0 @@
{% docs my_model_doc %}
a doc string
{% enddocs %}
{% docs my_model_doc %}
duplicate doc string
{% enddocs %}

View File

@@ -1 +0,0 @@
select 1 as id, 'joe' as first_name

View File

@@ -1,5 +0,0 @@
version: 2
models:
- name: model
description: "{{ doc('my_model_doc') }}"

View File

@@ -1,12 +0,0 @@
{% docs my_model_doc %}
My model is just a copy of the seed
{% enddocs %}
{% docs my_model_doc__id %}
The user ID number
{% enddocs %}
The following doc is never used, which should be fine.
{% docs my_model_doc__first_name %}
The user's first name
{% enddocs %}

View File

@@ -1 +0,0 @@
select 1 as id, 'joe' as first_name

View File

@@ -1,10 +0,0 @@
version: 2
models:
- name: model
description: "{{ doc('my_model_doc') }}"
columns:
- name: id
description: "{{ doc('my_model_doc__id') }}"
- name: first_name
description: "{{ doc('foo.bar.my_model_doc__id') }}"

View File

@@ -1,7 +0,0 @@
{% docs my_model_doc %}
My model is just a copy of the seed
{% enddocs %}
{% docs my_model_doc__id %}
The user ID number
{% enddocs %}

View File

@@ -1 +0,0 @@
select 1 as id, 'joe' as first_name

View File

@@ -1,11 +0,0 @@
version: 2
models:
- name: model
description: "{{ doc('my_model_doc') }}"
columns:
- name: id
description: "{{ doc('my_model_doc__id') }}"
- name: first_name
# invalid reference
description: "{{ doc('my_model_doc__first_name') }}"

View File

@@ -1,17 +0,0 @@
{% docs my_model_doc %}
My model is just a copy of the seed
{% enddocs %}
{% docs my_model_doc__id %}
The user ID number
{% enddocs %}
The following doc is never used, which should be fine.
{% docs my_model_doc__first_name %}
The user's first name (should not be shown!)
{% enddocs %}
This doc is referenced by its full name
{% docs my_model_doc__last_name %}
The user's last name
{% enddocs %}

View File

@@ -1 +0,0 @@
select 1 as id, 'joe' as first_name, 'smith' as last_name

View File

@@ -1,12 +0,0 @@
version: 2
models:
- name: model
description: "{{ doc('my_model_doc') }}"
columns:
- name: id
description: "{{ doc('my_model_doc__id') }}"
- name: first_name
description: The user's first name
- name: last_name
description: "{{ doc('test', 'my_model_doc__last_name') }}"

View File

@@ -1,6 +0,0 @@
{{
config(
materialized='table'
)
}}
select 1 as id

View File

@@ -1,7 +0,0 @@
{{
config(
materialized='table',
schema='another_schema'
)
}}
select 1 as id

View File

@@ -1,6 +0,0 @@
{{
config(
materialized='table'
)
}}
select 1 as id

View File

@@ -1,6 +0,0 @@
{{
config(
materialized='table'
)
}}
select 1 as id

View File

@@ -1 +0,0 @@
select 1 as x

View File

@@ -1 +0,0 @@
select 1 as y

View File

@@ -1 +0,0 @@
select 1 as z

View File

@@ -1,44 +0,0 @@
import os
from unittest import mock
from test.integration.base import DBTIntegrationTest, use_profile
class TestTargetPathFromProjectConfig(DBTIntegrationTest):
@property
def project_config(self):
return {"config-version": 2, "target-path": "project_target"}
@property
def schema(self):
return "target_path_tests_075"
@property
def models(self):
return "models"
@use_profile("postgres")
def test_postgres_overriden_target_path(self):
results = self.run_dbt(args=["run"])
self.assertFalse(os.path.exists("./target"))
self.assertTrue(os.path.exists("./project_target"))
class TestTargetPathOverridenEnv(TestTargetPathFromProjectConfig):
@use_profile("postgres")
def test_postgres_overriden_target_path(self):
with mock.patch.dict(os.environ, {"DBT_TARGET_PATH": "env_target"}):
results = self.run_dbt(args=["run"])
self.assertFalse(os.path.exists("./target"))
self.assertFalse(os.path.exists("./project_target"))
self.assertTrue(os.path.exists("./env_target"))
class TestTargetPathOverridenEnvironment(TestTargetPathFromProjectConfig):
@use_profile("postgres")
def test_postgres_overriden_target_path(self):
with mock.patch.dict(os.environ, {"DBT_TARGET_PATH": "env_target"}):
results = self.run_dbt(args=["run", "--target-path", "cli_target"])
self.assertFalse(os.path.exists("./target"))
self.assertFalse(os.path.exists("./project_target"))
self.assertFalse(os.path.exists("./env_target"))
self.assertTrue(os.path.exists("./cli_target"))

View File

@@ -1 +0,0 @@
# Integration test README

File diff suppressed because it is too large Load Diff

View File

@@ -6,6 +6,7 @@ import pytest
from dbt import flags
from dbt.contracts.project import UserConfig
from dbt.graph.selector_spec import IndirectSelection
from dbt.helper_types import WarnErrorOptions
class TestFlags(TestCase):
@@ -66,13 +67,13 @@ class TestFlags(TestCase):
# warn_error_options
self.user_config.warn_error_options = '{"include": "all"}'
flags.set_from_args(self.args, self.user_config)
self.assertEqual(flags.WARN_ERROR_OPTIONS, '{"include": "all"}')
self.assertEqual(flags.WARN_ERROR_OPTIONS, WarnErrorOptions(include="all"))
os.environ['DBT_WARN_ERROR_OPTIONS'] = '{"include": []}'
flags.set_from_args(self.args, self.user_config)
self.assertEqual(flags.WARN_ERROR_OPTIONS, '{"include": []}')
self.assertEqual(flags.WARN_ERROR_OPTIONS, WarnErrorOptions(include=[]))
setattr(self.args, 'warn_error_options', '{"include": "all"}')
flags.set_from_args(self.args, self.user_config)
self.assertEqual(flags.WARN_ERROR_OPTIONS, '{"include": "all"}')
self.assertEqual(flags.WARN_ERROR_OPTIONS, WarnErrorOptions(include="all"))
# cleanup
os.environ.pop('DBT_WARN_ERROR_OPTIONS')
delattr(self.args, 'warn_error_options')
@@ -283,7 +284,7 @@ class TestFlags(TestCase):
def test__flags_are_mutually_exclusive(self):
# options from user config
self.user_config.warn_error = False
self.user_config.warn_error_options = '{"include":"all}'
self.user_config.warn_error_options = '{"include":"all"}'
with pytest.raises(ValueError):
flags.set_from_args(self.args, self.user_config)
#cleanup
@@ -292,7 +293,7 @@ class TestFlags(TestCase):
# options from args
setattr(self.args, 'warn_error', False)
setattr(self.args, 'warn_error_options', '{"include":"all}')
setattr(self.args, 'warn_error_options', '{"include":"all"}')
with pytest.raises(ValueError):
flags.set_from_args(self.args, self.user_config)
# cleanup
@@ -310,7 +311,7 @@ class TestFlags(TestCase):
# options from user config + args
self.user_config.warn_error = False
setattr(self.args, 'warn_error_options', '{"include":"all}')
setattr(self.args, 'warn_error_options', '{"include":"all"}')
with pytest.raises(ValueError):
flags.set_from_args(self.args, self.user_config)
# cleanup

View File

@@ -0,0 +1,35 @@
from pathlib import Path
import pytest
from dbt.tests.util import run_dbt
class TestTargetPathConfig:
@pytest.fixture(scope="class")
def project_config_update(self):
return {"config-version": 2, "target-path": "project_target"}
def test_target_path(self, project):
run_dbt(["run"])
assert Path("project_target").is_dir()
assert not Path("target").is_dir()
class TestTargetPathEnvVar:
def test_target_path(self, project, monkeypatch):
monkeypatch.setenv("DBT_TARGET_PATH", "env_target")
run_dbt(["run"])
assert Path("env_target").is_dir()
assert not Path("project_target").is_dir()
assert not Path("target").is_dir()
class TestTargetPathCliArg:
def test_target_path(self, project, monkeypatch):
monkeypatch.setenv("DBT_TARGET_PATH", "env_target")
run_dbt(["run", "--target-path", "cli_target"])
assert Path("cli_target").is_dir()
assert not Path("env_target").is_dir()
assert not Path("project_target").is_dir()
assert not Path("target").is_dir()

View File

@@ -7,6 +7,7 @@ from typing import List
from dbt.cli.main import cli
from dbt.contracts.project import UserConfig
from dbt.cli.flags import Flags
from dbt.helper_types import WarnErrorOptions
class TestFlags:
@@ -18,6 +19,10 @@ class TestFlags:
def run_context(self) -> click.Context:
return self.make_dbt_context("run", ["run"])
@pytest.fixture
def user_config(self) -> UserConfig:
return UserConfig()
def test_which(self, run_context):
flags = Flags(run_context)
assert flags.WHICH == "run"
@@ -53,9 +58,7 @@ class TestFlags:
flags = Flags(run_context)
assert flags.ANONYMOUS_USAGE_STATS == expected_anonymous_usage_stats
def test_empty_user_config_uses_default(self, run_context):
user_config = UserConfig()
def test_empty_user_config_uses_default(self, run_context, user_config):
flags = Flags(run_context, user_config)
assert flags.USE_COLORS == run_context.params["use_colors"]
@@ -63,8 +66,8 @@ class TestFlags:
flags = Flags(run_context, None)
assert flags.USE_COLORS == run_context.params["use_colors"]
def test_prefer_user_config_to_default(self, run_context):
user_config = UserConfig(use_colors=False)
def test_prefer_user_config_to_default(self, run_context, user_config):
user_config.use_colors = False
# ensure default value is not the same as user config
assert run_context.params["use_colors"] is not user_config.use_colors
@@ -78,10 +81,81 @@ class TestFlags:
flags = Flags(context, user_config)
assert flags.USE_COLORS
def test_prefer_env_to_user_config(self, monkeypatch):
user_config = UserConfig(use_colors=False)
def test_prefer_env_to_user_config(self, monkeypatch, user_config):
user_config.use_colors = False
monkeypatch.setenv("DBT_USE_COLORS", "True")
context = self.make_dbt_context("run", ["run"])
flags = Flags(context, user_config)
assert flags.USE_COLORS
def test_mutually_exclusive_options_passed_separately(self):
"""Assert options that are mutually exclusive can be passed separately without error"""
warn_error_context = self.make_dbt_context("run", ["--warn-error", "run"])
flags = Flags(warn_error_context)
assert flags.WARN_ERROR
warn_error_options_context = self.make_dbt_context(
"run", ["--warn-error-options", '{"include": "all"}', "run"]
)
flags = Flags(warn_error_options_context)
assert flags.WARN_ERROR_OPTIONS == WarnErrorOptions(include="all")
def test_mutually_exclusive_options_from_cli(self):
context = self.make_dbt_context(
"run", ["--warn-error", "--warn-error-options", '{"include": "all"}', "run"]
)
with pytest.raises(click.BadOptionUsage):
Flags(context)
@pytest.mark.parametrize("warn_error", [True, False])
def test_mutually_exclusive_options_from_user_config(self, warn_error, user_config):
user_config.warn_error = warn_error
context = self.make_dbt_context(
"run", ["--warn-error-options", '{"include": "all"}', "run"]
)
with pytest.raises(click.BadOptionUsage):
Flags(context, user_config)
@pytest.mark.parametrize("warn_error", ["True", "False"])
def test_mutually_exclusive_options_from_envvar(self, warn_error, monkeypatch):
monkeypatch.setenv("DBT_WARN_ERROR", warn_error)
monkeypatch.setenv("DBT_WARN_ERROR_OPTIONS", '{"include":"all"}')
context = self.make_dbt_context("run", ["run"])
with pytest.raises(click.BadOptionUsage):
Flags(context)
@pytest.mark.parametrize("warn_error", [True, False])
def test_mutually_exclusive_options_from_cli_and_user_config(self, warn_error, user_config):
user_config.warn_error = warn_error
context = self.make_dbt_context(
"run", ["--warn-error-options", '{"include": "all"}', "run"]
)
with pytest.raises(click.BadOptionUsage):
Flags(context, user_config)
@pytest.mark.parametrize("warn_error", ["True", "False"])
def test_mutually_exclusive_options_from_cli_and_envvar(self, warn_error, monkeypatch):
monkeypatch.setenv("DBT_WARN_ERROR", warn_error)
context = self.make_dbt_context(
"run", ["--warn-error-options", '{"include": "all"}', "run"]
)
with pytest.raises(click.BadOptionUsage):
Flags(context)
@pytest.mark.parametrize("warn_error", ["True", "False"])
def test_mutually_exclusive_options_from_user_config_and_envvar(
self, user_config, warn_error, monkeypatch
):
user_config.warn_error = warn_error
monkeypatch.setenv("DBT_WARN_ERROR_OPTIONS", '{"include": "all"}')
context = self.make_dbt_context("run", ["run"])
with pytest.raises(click.BadOptionUsage):
Flags(context, user_config)

View File

@@ -16,6 +16,10 @@ class TestDbtRunner:
with pytest.raises(dbtUsageException):
dbt.invoke(["deps", "--invalid-option"])
def test_command_mutually_exclusive_option(self, dbt: dbtRunner) -> None:
with pytest.raises(dbtUsageException):
dbt.invoke(["--warn-error", "--warn-error-options", '{"include": "all"}', "deps"])
def test_invalid_command(self, dbt: dbtRunner) -> None:
with pytest.raises(dbtUsageException):
dbt.invoke(["invalid-command"])

View File

@@ -25,7 +25,6 @@ passenv =
POSTGRES_TEST_*
PYTEST_ADDOPTS
commands =
{envpython} -m pytest --cov=core -m profile_postgres {posargs} test/integration
{envpython} -m pytest --cov=core {posargs} tests/functional
{envpython} -m pytest --cov=core {posargs} tests/adapter