Compare commits

...

5 Commits

Author SHA1 Message Date
Michelle Ark
699fba83b9 restore missing types from merge 2023-10-25 14:33:11 -04:00
Colin
e9f81a99f6 merge main 2023-10-25 10:37:59 -07:00
Colin
a65dc24274 merge main 2023-10-25 10:29:27 -07:00
colin-rogers-dbt
29f734df9f Move events to common (#8676)
* Move events to common

* More Type Annotations (#8536)

* Extend use of type annotations in the events module.

* Add return type of None to more __init__ definitions.

* Still more type annotations adding -> None to __init__

* Tweak per review

* Allow adapters to include python package logging in dbt logs (#8643)

* add set_package_log_level functionality

* set package handler

* set package handler

* add logging about stting up logging

* test event log handler

* add event log handler

* add event log level

* rename package and add unit tests

* revert logfile config change

* cleanup and add code comments

* add changie

* swap function for dict

* add additional unit tests

* fix unit test

* update README and protos

* fix formatting

* update precommit

---------

Co-authored-by: Peter Webb <peter.webb@dbtlabs.com>
2023-09-21 16:48:41 -07:00
Michelle Ark
5fcd2b83f3 remove dbt.contracts.connection imports from adapter module 2023-09-19 00:04:37 +01:00
148 changed files with 1438 additions and 1630 deletions

View File

@@ -0,0 +1,6 @@
kind: Features
body: 'Allow adapters to include package logs in dbt standard logging '
time: 2023-09-15T12:37:33.862862-07:00
custom:
Author: colin-rogers-dbt
Issue: "7859"

View File

@@ -0,0 +1,6 @@
kind: Under the Hood
body: Added more type annotations.
time: 2023-08-31T16:44:35.737954-04:00
custom:
Author: peterallenwebb
Issue: "8537"

View File

@@ -1,7 +1,7 @@
# Configuration for pre-commit hooks (see https://pre-commit.com/).
# Eventually the hooks described here will be run as tests before merging each PR.
exclude: ^(core/dbt/docs/build/|core/dbt/events/types_pb2.py)
exclude: ^(core/dbt/docs/build/|core/dbt/common/events/types_pb2.py)
# Force all unspecified python hooks to run python 3.8
default_language_version:

View File

@@ -40,7 +40,7 @@ dev: dev_req ## Installs dbt-* packages in develop mode along with development d
.PHONY: proto_types
proto_types: ## generates google protobuf python file from types.proto
protoc -I=./core/dbt/events --python_out=./core/dbt/events ./core/dbt/events/types.proto
protoc -I=./core/dbt/common/events --python_out=./core/dbt/common/events ./core/dbt/common/events/types.proto
.PHONY: mypy
mypy: .env ## Runs mypy against staged changes for static type checking.

View File

@@ -1,7 +1,7 @@
# these are all just exports, #noqa them so flake8 will be happy
# TODO: Should we still include this in the `adapters` namespace?
from dbt.contracts.connection import Credentials # noqa: F401
from dbt.adapters.contracts.connection import Credentials # noqa: F401
from dbt.adapters.base.meta import available # noqa: F401
from dbt.adapters.base.connections import BaseConnectionManager # noqa: F401
from dbt.adapters.base.relation import ( # noqa: F401

View File

@@ -24,7 +24,7 @@ from typing import (
import agate
import dbt.exceptions
from dbt.contracts.connection import (
from dbt.adapters.contracts.connection import (
Connection,
Identifier,
ConnectionState,
@@ -36,9 +36,9 @@ from dbt.contracts.graph.manifest import Manifest
from dbt.adapters.base.query_headers import (
MacroQueryStringSetter,
)
from dbt.events import AdapterLogger
from dbt.events.functions import fire_event
from dbt.events.types import (
from dbt.common.events import AdapterLogger
from dbt.common.events.functions import fire_event
from dbt.common.events.types import (
NewConnection,
ConnectionReused,
ConnectionLeftOpenInCleanup,
@@ -48,7 +48,7 @@ from dbt.events.types import (
Rollback,
RollbackFailed,
)
from dbt.events.contextvars import get_node_info
from dbt.common.events.contextvars import get_node_info
from dbt import flags
from dbt.utils import cast_to_str

View File

@@ -17,11 +17,9 @@ from typing import (
Set,
Tuple,
Type,
TypedDict,
Union,
)
from dbt.adapters.capability import Capability, CapabilityDict
from dbt.contracts.graph.nodes import ColumnLevelConstraint, ConstraintType, ModelLevelConstraint
import agate
@@ -46,17 +44,12 @@ from dbt.exceptions import (
)
from dbt.adapters.protocol import AdapterConfig
from dbt.clients.agate_helper import (
empty_table,
get_column_value_uncased,
merge_tables,
table_from_rows,
)
from dbt.clients.agate_helper import empty_table, merge_tables, table_from_rows
from dbt.clients.jinja import MacroGenerator
from dbt.contracts.graph.manifest import Manifest, MacroManifest
from dbt.contracts.graph.nodes import ResultNode
from dbt.events.functions import fire_event, warn_or_error
from dbt.events.types import (
from dbt.common.events.functions import fire_event, warn_or_error
from dbt.common.events.types import (
CacheMiss,
ListRelations,
CodeExecution,
@@ -81,9 +74,7 @@ from dbt.adapters.cache import RelationsCache, _make_ref_key_dict
from dbt import deprecations
GET_CATALOG_MACRO_NAME = "get_catalog"
GET_CATALOG_RELATIONS_MACRO_NAME = "get_catalog_relations"
FRESHNESS_MACRO_NAME = "collect_freshness"
GET_RELATION_LAST_MODIFIED_MACRO_NAME = "get_relation_last_modified"
class ConstraintSupport(str, Enum):
@@ -118,7 +109,7 @@ def _catalog_filter_schemas(manifest: Manifest) -> Callable[[agate.Row], bool]:
return test
def _utc(dt: Optional[datetime], source: Optional[BaseRelation], field_name: str) -> datetime:
def _utc(dt: Optional[datetime], source: BaseRelation, field_name: str) -> datetime:
"""If dt has a timezone, return a new datetime that's in UTC. Otherwise,
assume the datetime is already for UTC and add the timezone.
"""
@@ -170,12 +161,6 @@ class PythonJobHelper:
raise NotImplementedError("PythonJobHelper submit function is not implemented yet")
class FreshnessResponse(TypedDict):
max_loaded_at: datetime
snapshotted_at: datetime
age: float # age in seconds
class BaseAdapter(metaclass=AdapterMeta):
"""The BaseAdapter provides an abstract base class for adapters.
@@ -237,10 +222,6 @@ class BaseAdapter(metaclass=AdapterMeta):
ConstraintType.foreign_key: ConstraintSupport.ENFORCED,
}
# This static member variable can be overriden in concrete adapter
# implementations to indicate adapter support for optional capabilities.
_capabilities = CapabilityDict({})
def __init__(self, config) -> None:
self.config = config
self.cache = RelationsCache()
@@ -434,8 +415,16 @@ class BaseAdapter(metaclass=AdapterMeta):
lowercase strings.
"""
info_schema_name_map = SchemaSearchMap()
relations = self._get_catalog_relations(manifest)
for relation in relations:
nodes: Iterator[ResultNode] = chain(
[
node
for node in manifest.nodes.values()
if (node.is_relational and not node.is_ephemeral_model)
],
manifest.sources.values(),
)
for node in nodes:
relation = self.Relation.create_from(self.config, node)
info_schema_name_map.add(relation)
# result is a map whose keys are information_schema Relations without
# identifiers that have appropriate database prefixes, and whose values
@@ -443,46 +432,6 @@ class BaseAdapter(metaclass=AdapterMeta):
# databases
return info_schema_name_map
def _get_catalog_relations_by_info_schema(
self, relations
) -> Dict[InformationSchema, List[BaseRelation]]:
relations_by_info_schema: Dict[InformationSchema, List[BaseRelation]] = dict()
for relation in relations:
info_schema = relation.information_schema_only()
if info_schema not in relations_by_info_schema:
relations_by_info_schema[info_schema] = []
relations_by_info_schema[info_schema].append(relation)
return relations_by_info_schema
def _get_catalog_relations(
self, manifest: Manifest, selected_nodes: Optional[Set] = None
) -> List[BaseRelation]:
nodes: Iterator[ResultNode]
if selected_nodes:
selected: List[ResultNode] = []
for unique_id in selected_nodes:
if unique_id in manifest.nodes:
node = manifest.nodes[unique_id]
if node.is_relational and not node.is_ephemeral_model:
selected.append(node)
elif unique_id in manifest.sources:
source = manifest.sources[unique_id]
selected.append(source)
nodes = iter(selected)
else:
nodes = chain(
[
node
for node in manifest.nodes.values()
if (node.is_relational and not node.is_ephemeral_model)
],
manifest.sources.values(),
)
relations = [self.Relation.create_from(self.config, n) for n in nodes]
return relations
def _relations_cache_for_schemas(
self, manifest: Manifest, cache_schemas: Optional[Set[BaseRelation]] = None
) -> None:
@@ -1144,60 +1093,20 @@ class BaseAdapter(metaclass=AdapterMeta):
results = self._catalog_filter_table(table, manifest) # type: ignore[arg-type]
return results
def _get_one_catalog_by_relations(
self,
information_schema: InformationSchema,
relations: List[BaseRelation],
manifest: Manifest,
) -> agate.Table:
kwargs = {
"information_schema": information_schema,
"relations": relations,
}
table = self.execute_macro(
GET_CATALOG_RELATIONS_MACRO_NAME,
kwargs=kwargs,
# pass in the full manifest, so we get any local project
# overrides
manifest=manifest,
)
results = self._catalog_filter_table(table, manifest) # type: ignore[arg-type]
return results
def get_catalog(
self, manifest: Manifest, selected_nodes: Optional[Set] = None
) -> Tuple[agate.Table, List[Exception]]:
def get_catalog(self, manifest: Manifest) -> Tuple[agate.Table, List[Exception]]:
schema_map = self._get_catalog_schemas(manifest)
with executor(self.config) as tpe:
futures: List[Future[agate.Table]] = []
catalog_relations = self._get_catalog_relations(manifest, selected_nodes)
relation_count = len(catalog_relations)
if relation_count <= 100 and self.supports(Capability.SchemaMetadataByRelations):
relations_by_schema = self._get_catalog_relations_by_info_schema(catalog_relations)
for info_schema in relations_by_schema:
name = ".".join([str(info_schema.database), "information_schema"])
relations = relations_by_schema[info_schema]
fut = tpe.submit_connected(
self,
name,
self._get_one_catalog_by_relations,
info_schema,
relations,
manifest,
)
futures.append(fut)
else:
schema_map: SchemaSearchMap = self._get_catalog_schemas(manifest)
for info, schemas in schema_map.items():
if len(schemas) == 0:
continue
name = ".".join([str(info.database), "information_schema"])
fut = tpe.submit_connected(
self, name, self._get_one_catalog, info, schemas, manifest
)
futures.append(fut)
for info, schemas in schema_map.items():
if len(schemas) == 0:
continue
name = ".".join([str(info.database), "information_schema"])
fut = tpe.submit_connected(
self, name, self._get_one_catalog, info, schemas, manifest
)
futures.append(fut)
catalogs, exceptions = catch_as_completed(futures)
@@ -1213,7 +1122,7 @@ class BaseAdapter(metaclass=AdapterMeta):
loaded_at_field: str,
filter: Optional[str],
manifest: Optional[Manifest] = None,
) -> Tuple[Optional[AdapterResponse], FreshnessResponse]:
) -> Tuple[Optional[AdapterResponse], Dict[str, Any]]:
"""Calculate the freshness of sources in dbt, and return it"""
kwargs: Dict[str, Any] = {
"source": source,
@@ -1248,52 +1157,13 @@ class BaseAdapter(metaclass=AdapterMeta):
snapshotted_at = _utc(table[0][1], source, loaded_at_field)
age = (snapshotted_at - max_loaded_at).total_seconds()
freshness: FreshnessResponse = {
freshness = {
"max_loaded_at": max_loaded_at,
"snapshotted_at": snapshotted_at,
"age": age,
}
return adapter_response, freshness
def calculate_freshness_from_metadata(
self,
source: BaseRelation,
manifest: Optional[Manifest] = None,
) -> Tuple[Optional[AdapterResponse], FreshnessResponse]:
kwargs: Dict[str, Any] = {
"information_schema": source.information_schema_only(),
"relations": [source],
}
result = self.execute_macro(
GET_RELATION_LAST_MODIFIED_MACRO_NAME, kwargs=kwargs, manifest=manifest
)
adapter_response, table = result.response, result.table # type: ignore[attr-defined]
try:
row = table[0]
last_modified_val = get_column_value_uncased("last_modified", row)
snapshotted_at_val = get_column_value_uncased("snapshotted_at", row)
except Exception:
raise MacroResultError(GET_RELATION_LAST_MODIFIED_MACRO_NAME, table)
if last_modified_val is None:
# Interpret missing value as "infinitely long ago"
max_loaded_at = datetime(1, 1, 1, 0, 0, 0, tzinfo=pytz.UTC)
else:
max_loaded_at = _utc(last_modified_val, None, "last_modified")
snapshotted_at = _utc(snapshotted_at_val, None, "snapshotted_at")
age = (snapshotted_at - max_loaded_at).total_seconds()
freshness: FreshnessResponse = {
"max_loaded_at": max_loaded_at,
"snapshotted_at": snapshotted_at,
"age": age,
}
return adapter_response, freshness
def pre_model_hook(self, config: Mapping[str, Any]) -> Any:
"""A hook for running some operation before the model materialization
runs. The hook can assume it has a connection available.
@@ -1567,14 +1437,6 @@ class BaseAdapter(metaclass=AdapterMeta):
else:
return None
@classmethod
def capabilities(cls) -> CapabilityDict:
return cls._capabilities
@classmethod
def supports(cls, capability: Capability) -> bool:
return bool(cls.capabilities()[capability])
COLUMNS_EQUAL_SQL = """
with diff_count as (

View File

@@ -4,7 +4,7 @@ from typing import Optional, Callable, Dict, Any
from dbt.clients.jinja import QueryStringGenerator
from dbt.context.manifest import generate_query_header_context
from dbt.contracts.connection import AdapterRequiredConfig, QueryComment
from dbt.adapters.contracts.connection import AdapterRequiredConfig, QueryComment
from dbt.contracts.graph.nodes import ResultNode
from dbt.contracts.graph.manifest import Manifest
from dbt.exceptions import DbtRuntimeError

View File

@@ -14,8 +14,8 @@ from dbt.exceptions import (
ReferencedLinkNotCachedError,
TruncatedModelNameCausedCollisionError,
)
from dbt.events.functions import fire_event, fire_event_if
from dbt.events.types import CacheAction, CacheDumpGraph
from dbt.common.events.functions import fire_event, fire_event_if
from dbt.common.events.types import CacheAction, CacheDumpGraph
from dbt.flags import get_flags
from dbt.utils import lowercase

View File

@@ -11,20 +11,27 @@ from typing import (
List,
Callable,
)
from dbt.exceptions import DbtInternalError
from dbt.utils import translate_aliases, md5
from dbt.events.functions import fire_event
from dbt.events.types import NewConnectionOpening
from dbt.events.contextvars import get_node_info
from typing_extensions import Protocol, Annotated
from dbt.dataclass_schema import (
from mashumaro.jsonschema.annotations import Pattern
from dbt.adapters.utils import translate_aliases
from dbt.common.exceptions import DbtInternalError
from dbt.common.dataclass_schema import (
dbtClassMixin,
StrEnum,
ExtensibleDbtClassMixin,
ValidatedStringMixin,
)
from dbt.contracts.util import Replaceable
from mashumaro.jsonschema.annotations import Pattern
from dbt.common.contracts.util import Replaceable
from dbt.common.util import md5
# TODO: dbt.common.events dependency
from dbt.common.events.functions import fire_event
from dbt.common.events.types import NewConnectionOpening
# TODO: this is a very bad dependency - shared global state
from dbt.common.events.contextvars import get_node_info
class Identifier(ValidatedStringMixin):

View File

@@ -0,0 +1,24 @@
from typing import Mapping, Any
from dbt.exceptions import DbtValidationError
class AliasError(DbtValidationError):
pass
# core level exceptions
class DuplicateAliasError(AliasError):
def __init__(self, kwargs: Mapping[str, Any], aliases: Mapping[str, str], canonical_key: str):
self.kwargs = kwargs
self.aliases = aliases
self.canonical_key = canonical_key
super().__init__(msg=self.get_message())
def get_message(self) -> str:
# dupe found: go through the dict so we can have a nice-ish error
key_names = ", ".join(
"{}".format(k) for k in self.kwargs if self.aliases.get(k) == self.canonical_key
)
msg = f'Got duplicate keys: ({key_names}) all map to "{self.canonical_key}"'
return msg

View File

@@ -7,9 +7,9 @@ from typing import Any, Dict, List, Optional, Set, Type
from dbt.adapters.base.plugin import AdapterPlugin
from dbt.adapters.protocol import AdapterConfig, AdapterProtocol, RelationProtocol
from dbt.contracts.connection import AdapterRequiredConfig, Credentials
from dbt.events.functions import fire_event
from dbt.events.types import AdapterImportError, PluginLoadError, AdapterRegistered
from dbt.adapters.contracts.connection import AdapterRequiredConfig, Credentials
from dbt.common.events.functions import fire_event
from dbt.common.events.types import AdapterImportError, PluginLoadError, AdapterRegistered
from dbt.exceptions import DbtInternalError, DbtRuntimeError
from dbt.include.global_project import PACKAGE_PATH as GLOBAL_PROJECT_PATH
from dbt.include.global_project import PROJECT_NAME as GLOBAL_PROJECT_NAME

View File

@@ -15,7 +15,7 @@ from typing_extensions import Protocol
import agate
from dbt.contracts.connection import Connection, AdapterRequiredConfig, AdapterResponse
from dbt.adapters.contracts.connection import Connection, AdapterRequiredConfig, AdapterResponse
from dbt.contracts.graph.nodes import ResultNode, ManifestNode
from dbt.contracts.graph.model_config import BaseConfig
from dbt.contracts.graph.manifest import Manifest

View File

@@ -3,7 +3,7 @@ from dataclasses import dataclass
from typing import Hashable
from dbt.adapters.relation_configs.config_base import RelationConfigBase
from dbt.dataclass_schema import StrEnum
from dbt.common.dataclass_schema import StrEnum
class RelationConfigChangeAction(StrEnum):

View File

@@ -7,10 +7,10 @@ import agate
import dbt.clients.agate_helper
import dbt.exceptions
from dbt.adapters.base import BaseConnectionManager
from dbt.contracts.connection import Connection, ConnectionState, AdapterResponse
from dbt.events.functions import fire_event
from dbt.events.types import ConnectionUsed, SQLQuery, SQLCommit, SQLQueryStatus
from dbt.events.contextvars import get_node_info
from dbt.adapters.contracts.connection import Connection, ConnectionState, AdapterResponse
from dbt.common.events.functions import fire_event
from dbt.common.events.types import ConnectionUsed, SQLQuery, SQLCommit, SQLQueryStatus
from dbt.common.events.contextvars import get_node_info
from dbt.utils import cast_to_str

View File

@@ -1,13 +1,13 @@
import agate
from typing import Any, Optional, Tuple, Type, List
from dbt.contracts.connection import Connection, AdapterResponse
from dbt.adapters.contracts.connection import Connection, AdapterResponse
from dbt.exceptions import RelationTypeNullError
from dbt.adapters.base import BaseAdapter, available
from dbt.adapters.cache import _make_ref_key_dict
from dbt.adapters.sql import SQLConnectionManager
from dbt.events.functions import fire_event
from dbt.events.types import ColTypeChange, SchemaCreation, SchemaDrop
from dbt.common.events.functions import fire_event
from dbt.common.events.types import ColTypeChange, SchemaCreation, SchemaDrop
from dbt.adapters.base.relation import BaseRelation

View File

@@ -0,0 +1,55 @@
from typing import Mapping, Sequence, Any, Dict, List
from dbt.adapters.exceptions import DuplicateAliasError
class Translator:
def __init__(self, aliases: Mapping[str, str], recursive: bool = False):
self.aliases = aliases
self.recursive = recursive
def translate_mapping(self, kwargs: Mapping[str, Any]) -> Dict[str, Any]:
result: Dict[str, Any] = {}
for key, value in kwargs.items():
canonical_key = self.aliases.get(key, key)
if canonical_key in result:
raise DuplicateAliasError(kwargs, self.aliases, canonical_key)
result[canonical_key] = self.translate_value(value)
return result
def translate_sequence(self, value: Sequence[Any]) -> List[Any]:
return [self.translate_value(v) for v in value]
def translate_value(self, value: Any) -> Any:
if self.recursive:
if isinstance(value, Mapping):
return self.translate_mapping(value)
elif isinstance(value, (list, tuple)):
return self.translate_sequence(value)
return value
def translate(self, value: Mapping[str, Any]) -> Dict[str, Any]:
try:
return self.translate_mapping(value)
except RuntimeError as exc:
if "maximum recursion depth exceeded" in str(exc):
raise RecursionError("Cycle detected in a value passed to translate!")
raise
def translate_aliases(
kwargs: Dict[str, Any],
aliases: Dict[str, str],
recurse: bool = False,
) -> Dict[str, Any]:
"""Given a dict of keyword arguments and a dict mapping aliases to their
canonical values, canonicalize the keys in the kwargs dict.
If recurse is True, perform this operation recursively.
:returns: A dict containing all the values in kwargs referenced by their
canonical key.
:raises: `AliasError`, if a canonical key is defined more than once.
"""
translator = Translator(aliases, recurse)
return translator.translate(kwargs)

View File

@@ -21,7 +21,7 @@ from dbt.contracts.results import (
CatalogArtifact,
RunExecutionResult,
)
from dbt.events.base_types import EventMsg
from dbt.common.events.base_types import EventMsg
from dbt.task.build import BuildTask
from dbt.task.clean import CleanTask
from dbt.task.clone import CloneTask

View File

@@ -9,17 +9,20 @@ from dbt.cli.exceptions import (
from dbt.cli.flags import Flags
from dbt.config import RuntimeConfig
from dbt.config.runtime import load_project, load_profile, UnsetProfile
from dbt.events.base_types import EventLevel
from dbt.events.functions import fire_event, LOG_VERSION, set_invocation_id, setup_event_logger
from dbt.events.types import (
from dbt.common.events.functions import (
fire_event,
LOG_VERSION,
set_invocation_id,
setup_event_logger,
)
from dbt.common.events.types import (
CommandCompleted,
MainReportVersion,
MainReportArgs,
MainTrackingUserState,
ResourceReport,
)
from dbt.events.helpers import get_json_string_utcnow
from dbt.events.types import MainEncounteredError, MainStackTrace
from dbt.common.events.helpers import get_json_string_utcnow
from dbt.common.events.types import MainEncounteredError, MainStackTrace
from dbt.exceptions import Exception as DbtException, DbtProjectError, FailFastError
from dbt.parser.manifest import ManifestLoader, write_manifest
from dbt.profiler import profiler
@@ -29,7 +32,6 @@ from dbt.plugins import set_up_plugin_manager, get_plugin_manager
from click import Context
from functools import update_wrapper
import importlib.util
import time
import traceback
@@ -99,28 +101,6 @@ def postflight(func):
fire_event(MainStackTrace(stack_trace=traceback.format_exc()))
raise ExceptionExit(e)
finally:
# Fire ResourceReport, but only on systems which support the resource
# module. (Skip it on Windows).
if importlib.util.find_spec("resource") is not None:
import resource
rusage = resource.getrusage(resource.RUSAGE_SELF)
fire_event(
ResourceReport(
command_name=ctx.command.name,
command_success=success,
command_wall_clock_time=time.perf_counter() - start_func,
process_user_time=rusage.ru_utime,
process_kernel_time=rusage.ru_stime,
process_mem_max_rss=rusage.ru_maxrss,
process_in_blocks=rusage.ru_inblock,
process_out_blocks=rusage.ru_oublock,
),
EventLevel.INFO
if "flags" in ctx.obj and ctx.obj["flags"].SHOW_RESOURCE_REPORT
else None,
)
fire_event(
CommandCompleted(
command=ctx.command_path,

View File

@@ -2,8 +2,8 @@ import re
import os.path
from dbt.clients.system import run_cmd, rmdir
from dbt.events.functions import fire_event
from dbt.events.types import (
from dbt.common.events.functions import fire_event
from dbt.common.events.types import (
GitSparseCheckoutSubdirectory,
GitProgressCheckoutRevision,
GitProgressUpdatingExistingDependency,

View File

@@ -1,8 +1,8 @@
import functools
from typing import Any, Dict, List
import requests
from dbt.events.functions import fire_event
from dbt.events.types import (
from dbt.common.events.functions import fire_event
from dbt.common.events.types import (
RegistryProgressGETRequest,
RegistryProgressGETResponse,
RegistryIndexProgressGETRequest,

View File

@@ -15,8 +15,8 @@ from typing import Any, Callable, Dict, List, NoReturn, Optional, Tuple, Type, U
import dbt.exceptions
import requests
from dbt.events.functions import fire_event
from dbt.events.types import (
from dbt.common.events.functions import fire_event
from dbt.common.events.types import (
SystemCouldNotWrite,
SystemExecutingCmd,
SystemStdOut,

View File

View File

@@ -0,0 +1 @@
SECRET_ENV_PREFIX = "DBT_ENV_SECRET_"

View File

View File

View File

@@ -0,0 +1,7 @@
import dataclasses
# TODO: remove from dbt.contracts.util:: Replaceable + references
class Replaceable:
def replace(self, **kwargs):
return dataclasses.replace(self, **kwargs)

View File

@@ -43,7 +43,7 @@ from dbt.logger import GLOBAL_LOGGER as logger
Simply change it to these two lines with your adapter's database name, and all your existing call sites will now use the new system for v1.0:
```python
from dbt.events import AdapterLogger
from dbt.common.events import AdapterLogger
logger = AdapterLogger("<database name>")
# e.g. AdapterLogger("Snowflake")
```
@@ -52,4 +52,4 @@ logger = AdapterLogger("<database name>")
After adding a new message in `types.proto`, either:
- In the repository root directory: `make proto_types`
- In the `core/dbt/events` directory: `protoc -I=. --python_out=. types.proto`
- In the `core/dbt/common/events` directory: `protoc -I=. --python_out=. types.proto`

View File

@@ -1,9 +1,10 @@
import traceback
from dataclasses import dataclass
from dbt.events.functions import fire_event, EVENT_MANAGER
from dbt.events.contextvars import get_node_info
from dbt.events.event_handler import set_package_logging
from dbt.events.types import (
from dbt.common.events.event_handler import set_package_logging
from dbt.common.events.functions import fire_event, EVENT_MANAGER
from dbt.common.events.contextvars import get_node_info
from dbt.common.events.types import (
AdapterEventDebug,
AdapterEventInfo,
AdapterEventWarning,

View File

@@ -1,11 +1,11 @@
from enum import Enum
import os
import threading
from dbt.events import types_pb2
from dbt.common.events import types_pb2
import sys
from google.protobuf.json_format import ParseDict, MessageToDict, MessageToJson
from google.protobuf.message import Message
from dbt.events.helpers import get_json_string_utcnow
from dbt.common.events.helpers import get_json_string_utcnow
from typing import Optional
if sys.version_info >= (3, 8):
@@ -21,13 +21,13 @@ else:
def get_global_metadata_vars() -> dict:
from dbt.events.functions import get_metadata_vars
from dbt.common.events.functions import get_metadata_vars
return get_metadata_vars()
def get_invocation_id() -> str:
from dbt.events.functions import get_invocation_id
from dbt.common.events.functions import get_invocation_id
return get_invocation_id()
@@ -72,8 +72,8 @@ class BaseEvent:
self.pb_msg = ParseDict(kwargs, msg_cls())
except Exception:
# Imports need to be here to avoid circular imports
from dbt.events.types import Note
from dbt.events.functions import fire_event
from dbt.common.events.types import Note
from dbt.common.events.functions import fire_event
error_msg = f"[{class_name}]: Unable to parse dict {kwargs}"
# If we're testing throw an error so that we notice failures
@@ -102,10 +102,7 @@ class BaseEvent:
def to_json(self) -> str:
return MessageToJson(
self.pb_msg,
preserving_proto_field_name=True,
including_default_value_fields=True,
indent=None,
self.pb_msg, preserving_proto_field_name=True, including_default_value_fields=True
)
def level_tag(self) -> EventLevel:

View File

@@ -1,10 +1,10 @@
import logging
from typing import Union
from dbt.events.base_types import EventLevel
from dbt.events.types import Note
from dbt.common.events.base_types import EventLevel
from dbt.common.events.types import Note
from dbt.events.eventmgr import IEventManager
from dbt.common.events.eventmgr import IEventManager
_log_level_to_event_level_map = {
logging.DEBUG: EventLevel.DEBUG,

View File

@@ -3,8 +3,8 @@ import traceback
from typing import Callable, List, Optional, Protocol, Tuple
from uuid import uuid4
from dbt.events.base_types import BaseEvent, EventLevel, msg_from_base_event, EventMsg
from dbt.events.logger import LoggerConfig, _Logger, _TextLogger, _JsonLogger, LineFormat
from dbt.common.events.base_types import BaseEvent, EventLevel, msg_from_base_event, EventMsg
from dbt.common.events.logger import LoggerConfig, _Logger, _TextLogger, _JsonLogger, LineFormat
class EventManager:

View File

@@ -1,9 +1,9 @@
from dbt.constants import METADATA_ENV_PREFIX
from dbt.events.base_types import BaseEvent, EventLevel, EventMsg
from dbt.events.eventmgr import EventManager, IEventManager
from dbt.events.logger import LoggerConfig, NoFilter, LineFormat
from dbt.common.events.base_types import BaseEvent, EventLevel, EventMsg
from dbt.common.events.eventmgr import EventManager, IEventManager
from dbt.common.events.logger import LoggerConfig, NoFilter, LineFormat
from dbt.exceptions import scrub_secrets, env_secrets
from dbt.events.types import Note
from dbt.common.events.types import Note
from dbt.flags import get_flags, ENABLE_LEGACY_LOGGER
from dbt.logger import GLOBAL_LOGGER, make_log_dir_if_missing
from functools import partial

View File

@@ -10,8 +10,8 @@ from typing import Optional, TextIO, Any, Callable
from colorama import Style
import dbt.utils
from dbt.events.base_types import EventLevel, EventMsg
from dbt.events.format import timestamp_to_datetime_string
from dbt.common.events.base_types import EventLevel, EventMsg
from dbt.common.events.format import timestamp_to_datetime_string
# A Filter is a function which takes a BaseEvent and returns True if the event
# should be logged, False otherwise.
@@ -172,7 +172,7 @@ class _TextLogger(_Logger):
class _JsonLogger(_Logger):
def create_line(self, msg: EventMsg) -> str:
from dbt.events.functions import msg_to_dict
from dbt.common.events.functions import msg_to_dict
msg_dict = msg_to_dict(msg)
raw_log_line = json.dumps(msg_dict, sort_keys=True, cls=dbt.utils.ForgivingJSONEncoder)

View File

@@ -1,7 +1,7 @@
import json
from dbt.constants import MAXIMUM_SEED_SIZE_NAME, PIN_PACKAGE_URL
from dbt.events.base_types import (
from dbt.common.events.base_types import (
DynamicLevel,
DebugLevel,
InfoLevel,
@@ -9,7 +9,11 @@ from dbt.events.base_types import (
ErrorLevel,
EventLevel,
)
from dbt.events.format import format_fancy_output_line, pluralize, timestamp_to_datetime_string
from dbt.common.events.format import (
format_fancy_output_line,
pluralize,
timestamp_to_datetime_string,
)
from dbt.node_types import NodeType
from dbt.ui import line_wrap_message, warning_tag, red, green, yellow
@@ -1239,17 +1243,15 @@ class UnversionedBreakingChange(WarnLevel):
def message(self) -> str:
reasons = "\n - ".join(self.breaking_changes)
msg = (
return (
f"Breaking change to contracted, unversioned model {self.model_name} ({self.model_file_path})"
"\nWhile comparing to previous project state, dbt detected a breaking change to an unversioned model."
f"\n - {reasons}\n"
)
return warning_tag(msg)
class WarnStateTargetEqual(WarnLevel):
def code(self) -> str:
def code(self):
return "I072"
def message(self) -> str:

File diff suppressed because one or more lines are too long

View File

@@ -0,0 +1,58 @@
from typing import List
import os
from dbt.common.constants import SECRET_ENV_PREFIX
def env_secrets() -> List[str]:
return [v for k, v in os.environ.items() if k.startswith(SECRET_ENV_PREFIX) and v.strip()]
def scrub_secrets(msg: str, secrets: List[str]) -> str:
scrubbed = str(msg)
for secret in secrets:
scrubbed = scrubbed.replace(secret, "*****")
return scrubbed
class DbtInternalError(Exception):
def __init__(self, msg: str):
self.stack: List = []
self.msg = scrub_secrets(msg, env_secrets())
@property
def type(self):
return "Internal"
def process_stack(self):
lines = []
stack = self.stack
first = True
if len(stack) > 1:
lines.append("")
for item in stack:
msg = "called by"
if first:
msg = "in"
first = False
lines.append(f"> {msg}")
return lines
def __str__(self):
if hasattr(self.msg, "split"):
split_msg = self.msg.split("\n")
else:
split_msg = str(self.msg).split("\n")
lines = ["{}".format(self.type + " Error")] + split_msg
lines += self.process_stack()
return lines[0] + "\n" + "\n".join([" " + line for line in lines[1:]])

9
core/dbt/common/util.py Normal file
View File

@@ -0,0 +1,9 @@
import hashlib
import sys
def md5(string, charset="utf-8"):
if sys.version_info >= (3, 9):
return hashlib.md5(string.encode(charset), usedforsecurity=False).hexdigest()
else:
return hashlib.md5(string.encode(charset)).hexdigest()

View File

@@ -28,11 +28,11 @@ from dbt.exceptions import (
DbtRuntimeError,
)
from dbt.graph import Graph
from dbt.events.functions import fire_event, get_invocation_id
from dbt.events.types import FoundStats, Note, WritingInjectedSQLForNode
from dbt.events.contextvars import get_node_info
from dbt.common.events.functions import fire_event, get_invocation_id
from dbt.common.events.types import FoundStats, Note, WritingInjectedSQLForNode
from dbt.common.events.contextvars import get_node_info
from dbt.node_types import NodeType, ModelLanguage
from dbt.events.format import pluralize
from dbt.common.events.format import pluralize
import dbt.tracking
import dbt.task.list as list_task
import sqlparse

View File

@@ -2,12 +2,12 @@ from dataclasses import dataclass
from typing import Any, Dict, Optional, Tuple
import os
from dbt.dataclass_schema import ValidationError
from dbt.common.dataclass_schema import ValidationError
from dbt.flags import get_flags
from dbt.clients.system import load_file_contents
from dbt.clients.yaml_helper import load_yaml_text
from dbt.contracts.connection import Credentials, HasCredentials
from dbt.adapters.contracts.connection import Credentials, HasCredentials
from dbt.contracts.project import ProfileConfig, UserConfig
from dbt.exceptions import (
CompilationError,
@@ -17,8 +17,8 @@ from dbt.exceptions import (
DbtRuntimeError,
ProfileConfigError,
)
from dbt.events.types import MissingProfileTarget
from dbt.events.functions import fire_event
from dbt.common.events.types import MissingProfileTarget
from dbt.common.events.functions import fire_event
from dbt.utils import coerce_dict_str
from .renderer import ProfileRenderer

View File

@@ -23,7 +23,7 @@ from dbt.constants import (
)
from dbt.clients.system import path_exists, load_file_contents
from dbt.clients.yaml_helper import load_yaml_text
from dbt.contracts.connection import QueryComment
from dbt.adapters.contracts.connection import QueryComment
from dbt.exceptions import (
DbtProjectError,
SemverError,
@@ -43,7 +43,7 @@ from dbt.contracts.project import (
SemverString,
)
from dbt.contracts.project import PackageConfig, ProjectPackageMetadata
from dbt.dataclass_schema import ValidationError
from dbt.common.dataclass_schema import ValidationError
from .renderer import DbtProjectYamlRenderer, PackageRenderer
from .selectors import (
selector_config_from_data,

View File

@@ -8,7 +8,7 @@ from dbt.constants import SECRET_ENV_PREFIX, DEPENDENCIES_FILE_NAME
from dbt.context.target import TargetContext
from dbt.context.secret import SecretContext, SECRET_PLACEHOLDER
from dbt.context.base import BaseContext
from dbt.contracts.connection import HasCredentials
from dbt.adapters.contracts.connection import HasCredentials
from dbt.exceptions import DbtProjectError, CompilationError, RecursionError
from dbt.utils import deep_map_render

View File

@@ -18,13 +18,13 @@ from typing import (
from dbt.flags import get_flags
from dbt.adapters.factory import get_include_paths, get_relation_class_by_name
from dbt.config.project import load_raw_project
from dbt.contracts.connection import AdapterRequiredConfig, Credentials, HasCredentials
from dbt.adapters.contracts.connection import AdapterRequiredConfig, Credentials, HasCredentials
from dbt.contracts.graph.manifest import ManifestMetadata
from dbt.contracts.project import Configuration, UserConfig
from dbt.contracts.relation import ComponentName
from dbt.dataclass_schema import ValidationError
from dbt.events.functions import warn_or_error
from dbt.events.types import UnusedResourceConfigPath
from dbt.common.dataclass_schema import ValidationError
from dbt.common.events.functions import warn_or_error
from dbt.common.events.types import UnusedResourceConfigPath
from dbt.exceptions import (
ConfigContractBrokenError,
DbtProjectError,

View File

@@ -2,7 +2,7 @@ from pathlib import Path
from copy import deepcopy
from typing import Dict, Any, Union
from dbt.clients.yaml_helper import yaml, Loader, Dumper, load_yaml_text # noqa: F401
from dbt.dataclass_schema import ValidationError
from dbt.common.dataclass_schema import ValidationError
from .renderer import BaseRenderer

View File

@@ -2,8 +2,8 @@ from typing import Any, Dict
from dbt.clients import yaml_helper
from dbt.events.functions import fire_event
from dbt.events.types import InvalidOptionYAML
from dbt.common.events.functions import fire_event
from dbt.common.events.types import InvalidOptionYAML
from dbt.exceptions import DbtValidationError, OptionNotYamlDictError

View File

@@ -1,3 +1,4 @@
# TODO: remove SECRET_ENV_PREFIX and import from dbt.common
SECRET_ENV_PREFIX = "DBT_ENV_SECRET_"
DEFAULT_ENV_PLACEHOLDER = "DBT_DEFAULT_PLACEHOLDER"
METADATA_ENV_PREFIX = "DBT_ENV_CUSTOM_ENV_"

View File

@@ -21,9 +21,9 @@ from dbt.exceptions import (
SetStrictWrongTypeError,
ZipStrictWrongTypeError,
)
from dbt.events.functions import fire_event, get_invocation_id
from dbt.events.types import JinjaLogInfo, JinjaLogDebug
from dbt.events.contextvars import get_node_info
from dbt.common.events.functions import fire_event, get_invocation_id
from dbt.common.events.types import JinjaLogInfo, JinjaLogDebug
from dbt.common.events.contextvars import get_node_info
from dbt.version import __version__ as dbt_version
# These modules are added to the context. Consider alternative

View File

@@ -2,7 +2,7 @@ import os
from typing import Any, Dict, Optional
from dbt.constants import SECRET_ENV_PREFIX, DEFAULT_ENV_PLACEHOLDER
from dbt.contracts.connection import AdapterRequiredConfig
from dbt.adapters.contracts.connection import AdapterRequiredConfig
from dbt.node_types import NodeType
from dbt.utils import MultiDict

View File

@@ -1,8 +1,8 @@
import functools
from typing import NoReturn
from dbt.events.functions import warn_or_error
from dbt.events.types import JinjaLogWarning
from dbt.common.events.functions import warn_or_error
from dbt.common.events.types import JinjaLogWarning
from dbt.exceptions import (
DbtRuntimeError,

View File

@@ -1,7 +1,7 @@
from typing import List
from dbt.clients.jinja import MacroStack
from dbt.contracts.connection import AdapterRequiredConfig
from dbt.adapters.contracts.connection import AdapterRequiredConfig
from dbt.contracts.graph.manifest import Manifest
from dbt.context.macro_resolver import TestMacroNamespace
from .base import contextproperty

View File

@@ -27,7 +27,7 @@ from dbt.context.exceptions_jinja import wrapped_exports
from dbt.context.macro_resolver import MacroResolver, TestMacroNamespace
from dbt.context.macros import MacroNamespaceBuilder, MacroNamespace
from dbt.context.manifest import ManifestContext
from dbt.contracts.connection import AdapterResponse
from dbt.adapters.contracts.connection import AdapterResponse
from dbt.contracts.graph.manifest import Manifest, Disabled
from dbt.contracts.graph.nodes import (
Macro,
@@ -42,7 +42,7 @@ from dbt.contracts.graph.nodes import (
)
from dbt.contracts.graph.metrics import MetricReference, ResolvedMetricReference
from dbt.contracts.graph.unparsed import NodeVersion
from dbt.events.functions import get_metadata_vars
from dbt.common.events.functions import get_metadata_vars
from dbt.exceptions import (
CompilationError,
ConflictingConfigKeysError,

View File

@@ -6,7 +6,7 @@ from mashumaro.types import SerializableType
from typing import List, Optional, Union, Dict, Any
from dbt.constants import MAXIMUM_SEED_SIZE
from dbt.dataclass_schema import dbtClassMixin, StrEnum
from dbt.common.dataclass_schema import dbtClassMixin, StrEnum
from .util import SourceKey

View File

@@ -46,7 +46,7 @@ from dbt.contracts.graph.unparsed import SourcePatch, NodeVersion, UnparsedVersi
from dbt.contracts.graph.manifest_upgrade import upgrade_manifest_json
from dbt.contracts.files import SourceFile, SchemaSourceFile, FileHash, AnySourceFile
from dbt.contracts.util import BaseArtifactMetadata, SourceKey, ArtifactMixin, schema_version
from dbt.dataclass_schema import dbtClassMixin
from dbt.common.dataclass_schema import dbtClassMixin
from dbt.exceptions import (
CompilationError,
DuplicateResourceNameError,
@@ -55,9 +55,9 @@ from dbt.exceptions import (
AmbiguousResourceNameRefError,
)
from dbt.helper_types import PathSet
from dbt.events.functions import fire_event
from dbt.events.types import MergedFromState, UnpinnedRefNewVersionAvailable
from dbt.events.contextvars import get_node_info
from dbt.common.events.functions import fire_event
from dbt.common.events.types import MergedFromState, UnpinnedRefNewVersionAvailable
from dbt.common.events.contextvars import get_node_info
from dbt.node_types import NodeType, AccessType
from dbt.flags import get_flags, MP_CONTEXT
from dbt import tracking

View File

@@ -4,7 +4,7 @@ from itertools import chain
from typing import Any, List, Optional, Dict, Union, Type, TypeVar, Callable
from typing_extensions import Annotated
from dbt.dataclass_schema import (
from dbt.common.dataclass_schema import (
dbtClassMixin,
ValidationError,
StrEnum,

View File

@@ -8,7 +8,7 @@ import hashlib
from mashumaro.types import SerializableType
from typing import Optional, Union, List, Dict, Any, Sequence, Tuple, Iterator, Literal
from dbt.dataclass_schema import dbtClassMixin, ExtensibleDbtClassMixin
from dbt.common.dataclass_schema import dbtClassMixin, ExtensibleDbtClassMixin
from dbt.clients.system import write_file
from dbt.contracts.files import FileHash
@@ -37,16 +37,16 @@ from dbt.contracts.graph.unparsed import (
)
from dbt.contracts.graph.node_args import ModelNodeArgs
from dbt.contracts.util import Replaceable, AdditionalPropertiesMixin
from dbt.events.functions import warn_or_error
from dbt.common.events.functions import warn_or_error
from dbt.exceptions import ParsingError, ContractBreakingChangeError
from dbt.events.types import (
from dbt.common.events.types import (
SeedIncreased,
SeedExceedsLimitSamePath,
SeedExceedsLimitAndPathChanged,
SeedExceedsLimitChecksumChanged,
UnversionedBreakingChange,
)
from dbt.events.contextvars import set_log_contextvars
from dbt.common.events.contextvars import set_log_contextvars
from dbt.flags import get_flags
from dbt.node_types import ModelLanguage, NodeType, AccessType
from dbt_semantic_interfaces.call_parameter_sets import FilterCallParameterSets

View File

@@ -14,9 +14,9 @@ from dbt_semantic_interfaces.validations.semantic_manifest_validator import (
)
from dbt.clients.system import write_file
from dbt.events.base_types import EventLevel
from dbt.events.functions import fire_event
from dbt.events.types import SemanticValidationFailure
from dbt.common.events.base_types import EventLevel
from dbt.common.events.functions import fire_event
from dbt.common.events.types import SemanticValidationFailure
from dbt.exceptions import ParsingError

View File

@@ -1,5 +1,5 @@
from dataclasses import dataclass
from dbt.dataclass_schema import dbtClassMixin
from dbt.common.dataclass_schema import dbtClassMixin
from dbt_semantic_interfaces.references import (
DimensionReference,
EntityReference,

View File

@@ -18,7 +18,12 @@ from dbt.contracts.util import (
import dbt.helper_types # noqa:F401
from dbt.exceptions import CompilationError, ParsingError, DbtInternalError
from dbt.dataclass_schema import dbtClassMixin, StrEnum, ExtensibleDbtClassMixin, ValidationError
from dbt.common.dataclass_schema import (
dbtClassMixin,
StrEnum,
ExtensibleDbtClassMixin,
ValidationError,
)
from dataclasses import dataclass, field
from datetime import timedelta

View File

@@ -1,7 +1,7 @@
from dbt.contracts.util import Replaceable, Mergeable, list_str, Identifier
from dbt.contracts.connection import QueryComment, UserConfigContract
from dbt.adapters.contracts.connection import QueryComment, UserConfigContract
from dbt.helper_types import NoValue
from dbt.dataclass_schema import (
from dbt.common.dataclass_schema import (
dbtClassMixin,
ValidationError,
ExtensibleDbtClassMixin,

View File

@@ -6,7 +6,7 @@ from typing import (
)
from typing_extensions import Protocol
from dbt.dataclass_schema import dbtClassMixin, StrEnum
from dbt.common.dataclass_schema import dbtClassMixin, StrEnum
from dbt.contracts.util import Replaceable
from dbt.exceptions import CompilationError, DataclassNotDictError

View File

@@ -10,13 +10,13 @@ from dbt.contracts.util import (
schema_version,
)
from dbt.exceptions import DbtInternalError
from dbt.events.functions import fire_event
from dbt.events.types import TimingInfoCollected
from dbt.events.contextvars import get_node_info
from dbt.events.helpers import datetime_to_json_string
from dbt.common.events.functions import fire_event
from dbt.common.events.types import TimingInfoCollected
from dbt.common.events.contextvars import get_node_info
from dbt.common.events.helpers import datetime_to_json_string
from dbt.logger import TimingProcessor
from dbt.utils import lowercase, cast_to_str, cast_to_int
from dbt.dataclass_schema import dbtClassMixin, StrEnum
from dbt.common.dataclass_schema import dbtClassMixin, StrEnum
import agate

View File

@@ -1,5 +1,5 @@
from dataclasses import dataclass
from dbt.dataclass_schema import dbtClassMixin
from dbt.common.dataclass_schema import dbtClassMixin
from typing import List, Dict, Any, Union

View File

@@ -3,7 +3,7 @@ from dataclasses import dataclass, field
from datetime import datetime
from typing import Optional, List, Any, Dict, Sequence
from dbt.dataclass_schema import dbtClassMixin
from dbt.common.dataclass_schema import dbtClassMixin
from dbt.contracts.graph.nodes import ResultNode
from dbt.contracts.results import (

View File

@@ -4,8 +4,8 @@ from typing import Optional
from dbt.contracts.graph.manifest import WritableManifest
from dbt.contracts.results import FreshnessExecutionResultArtifact
from dbt.contracts.results import RunResultsArtifact
from dbt.events.functions import fire_event
from dbt.events.types import WarnStateTargetEqual
from dbt.common.events.functions import fire_event
from dbt.common.events.types import WarnStateTargetEqual
from dbt.exceptions import IncompatibleSchemaError

View File

@@ -10,10 +10,10 @@ from dbt.exceptions import (
)
from dbt.version import __version__
from dbt.events.functions import get_invocation_id, get_metadata_vars
from dbt.dataclass_schema import dbtClassMixin
from dbt.common.events.functions import get_invocation_id, get_metadata_vars
from dbt.common.dataclass_schema import dbtClassMixin
from dbt.dataclass_schema import (
from dbt.common.dataclass_schema import (
ValidatedStringMixin,
ValidationError,
)

View File

@@ -23,7 +23,7 @@ class DBTDeprecation:
@property
def event(self) -> abc.ABCMeta:
if self._event is not None:
module_path = dbt.events.types
module_path = dbt.common.events.types
class_name = self._event
try:
@@ -36,7 +36,7 @@ class DBTDeprecation:
def show(self, *args, **kwargs) -> None:
if self.name not in active_deprecations:
event = self.event(**kwargs)
dbt.events.functions.warn_or_error(event)
dbt.common.events.functions.warn_or_error(event)
self.track_deprecation_warn()
active_deprecations.add(self.name)

View File

@@ -8,8 +8,8 @@ from typing import List, Optional, Generic, TypeVar
from dbt.clients import system
from dbt.contracts.project import ProjectPackageMetadata
from dbt.events.functions import fire_event
from dbt.events.types import DepsSetDownloadDirectory
from dbt.common.events.functions import fire_event
from dbt.common.events.types import DepsSetDownloadDirectory
from dbt.utils import _connection_exception_retry as connection_exception_retry
DOWNLOADS_PATH = None

View File

@@ -10,8 +10,8 @@ from dbt.contracts.project import (
)
from dbt.deps.base import PinnedPackage, UnpinnedPackage, get_downloads_path
from dbt.exceptions import ExecutableError, MultipleVersionGitDepsError
from dbt.events.functions import fire_event, warn_or_error
from dbt.events.types import EnsureGitInstalled, DepsUnpinned
from dbt.common.events.functions import fire_event, warn_or_error
from dbt.common.events.types import EnsureGitInstalled, DepsUnpinned
from dbt.utils import md5

View File

@@ -6,8 +6,8 @@ from dbt.contracts.project import (
ProjectPackageMetadata,
LocalPackage,
)
from dbt.events.functions import fire_event
from dbt.events.types import DepsCreatingLocalSymlink, DepsSymlinkNotAvailable
from dbt.common.events.functions import fire_event
from dbt.common.events.types import DepsCreatingLocalSymlink, DepsSymlinkNotAvailable
from dbt.config.project import PartialProject, Project
from dbt.config.renderer import PackageRenderer

View File

@@ -1,30 +1,16 @@
import builtins
import json
import os
import re
import io
import agate
from typing import Any, Dict, List, Mapping, Optional, Union
from dbt.constants import SECRET_ENV_PREFIX
from dbt.dataclass_schema import ValidationError
from dbt.node_types import NodeType, AccessType
from dbt.ui import line_wrap_message
import dbt.dataclass_schema
def env_secrets() -> List[str]:
return [v for k, v in os.environ.items() if k.startswith(SECRET_ENV_PREFIX) and v.strip()]
def scrub_secrets(msg: str, secrets: List[str]) -> str:
scrubbed = str(msg)
for secret in secrets:
scrubbed = scrubbed.replace(secret, "*****")
return scrubbed
import dbt.common.dataclass_schema
from dbt.common.dataclass_schema import ValidationError
from dbt.common.exceptions import env_secrets, scrub_secrets
class MacroReturn(builtins.BaseException):
@@ -147,7 +133,7 @@ class DbtRuntimeError(RuntimeError, Exception):
"""Given a dbt.dataclass_schema.ValidationError (which is basically a
jsonschema.ValidationError), return the relevant parts as a string
"""
if not isinstance(exc, dbt.dataclass_schema.ValidationError):
if not isinstance(exc, dbt.common.dataclass_schema.ValidationError):
return str(exc)
path = "[%s]" % "][".join(map(repr, exc.relative_path))
return f"at path {path}: {exc.message}"

View File

@@ -24,7 +24,7 @@ ENABLE_LEGACY_LOGGER = env_set_truthy("DBT_ENABLE_LEGACY_LOGGER")
MP_CONTEXT = get_context()
# this roughly follows the patten of EVENT_MANAGER in dbt/events/functions.py
# this roughly follows the patten of EVENT_MANAGER in dbt/common/events/functions.py
# During de-globlization, we'll need to handle both similarly
# Match USE_COLORS default with default in dbt.cli.params.use_colors for use in --version
GLOBAL_FLAGS = Namespace(USE_COLORS=True) # type: ignore

View File

@@ -5,8 +5,8 @@ from .queue import GraphQueue
from .selector_methods import MethodManager
from .selector_spec import SelectionCriteria, SelectionSpec, IndirectSelection
from dbt.events.functions import fire_event, warn_or_error
from dbt.events.types import SelectorReportInvalidSelector, NoNodesForSelectionCriteria
from dbt.common.events.functions import fire_event, warn_or_error
from dbt.common.events.types import SelectorReportInvalidSelector, NoNodesForSelectionCriteria
from dbt.node_types import NodeType
from dbt.exceptions import (
DbtInternalError,

View File

@@ -4,7 +4,7 @@ from itertools import chain
from pathlib import Path
from typing import Set, List, Dict, Iterator, Tuple, Any, Union, Type, Optional, Callable
from dbt.dataclass_schema import StrEnum
from dbt.common.dataclass_schema import StrEnum
from .graph import UniqueId
@@ -28,7 +28,7 @@ from dbt.exceptions import (
DbtRuntimeError,
)
from dbt.node_types import NodeType
from dbt.events.contextvars import get_project_root
from dbt.common.events.contextvars import get_project_root
SELECTOR_GLOB = "*"

View File

@@ -2,7 +2,7 @@ import os
import re
from abc import ABCMeta, abstractmethod
from dataclasses import dataclass
from dbt.dataclass_schema import StrEnum, dbtClassMixin
from dbt.common.dataclass_schema import StrEnum, dbtClassMixin
from typing import Set, Iterator, List, Optional, Dict, Union, Any, Iterable, Tuple
from .graph import UniqueId

View File

@@ -7,12 +7,12 @@ from dataclasses import dataclass, field
from typing import Tuple, AbstractSet, Union
from typing import Callable, cast, Generic, Optional, TypeVar, List, NewType
from dbt.dataclass_schema import (
from dbt.common.dataclass_schema import (
dbtClassMixin,
ValidationError,
StrEnum,
)
import dbt.events.types as dbt_event_types
import dbt.common.events.types as dbt_event_types
Port = NewType("Port", int)

View File

@@ -1,4 +1,4 @@
from dbt.dataclass_schema import StrEnum
from dbt.common.dataclass_schema import StrEnum
import json
from typing import Union, Dict, Any

View File

@@ -1,8 +1,8 @@
import functools
from typing import Optional
from dbt.events.functions import warn_or_error
from dbt.events.types import InternalDeprecation
from dbt.common.events.functions import warn_or_error
from dbt.common.events.types import InternalDeprecation
def deprecated(suggested_action: str, version: str, reason: Optional[str]):

View File

@@ -14,7 +14,7 @@ from typing import Optional, List, ContextManager, Callable, Dict, Any, Set
import colorama
import logbook
from dbt.constants import SECRET_ENV_PREFIX
from dbt.dataclass_schema import dbtClassMixin
from dbt.common.dataclass_schema import dbtClassMixin
# Colorama is needed for colored logs on Windows because we're using logger.info
# intead of print(). If the Windows env doesn't have a TERM var set or it is set to None

View File

@@ -1,6 +1,6 @@
from typing import List
from dbt.dataclass_schema import StrEnum
from dbt.common.dataclass_schema import StrEnum
class AccessType(StrEnum):

View File

@@ -3,7 +3,7 @@ import itertools
import os
from typing import List, Dict, Any, Generic, Optional, TypeVar
from dbt.dataclass_schema import ValidationError
from dbt.common.dataclass_schema import ValidationError
from dbt import utils
from dbt.clients.jinja import MacroGenerator

View File

@@ -20,7 +20,7 @@ from itertools import chain
import time
from dbt.contracts.graph.semantic_manifest import SemanticManifest
from dbt.events.base_types import EventLevel
from dbt.common.events.base_types import EventLevel
import json
import pprint
import msgpack
@@ -41,8 +41,8 @@ from dbt.constants import (
SEMANTIC_MANIFEST_FILE_NAME,
)
from dbt.helper_types import PathSet
from dbt.events.functions import fire_event, get_invocation_id, warn_or_error
from dbt.events.types import (
from dbt.common.events.functions import fire_event, get_invocation_id, warn_or_error
from dbt.common.events.types import (
PartialParsingErrorProcessingFile,
PartialParsingError,
ParsePerfInfoPath,
@@ -124,7 +124,7 @@ from dbt.parser.snapshots import SnapshotParser
from dbt.parser.sources import SourcePatcher
from dbt.version import __version__
from dbt.dataclass_schema import StrEnum, dbtClassMixin
from dbt.common.dataclass_schema import StrEnum, dbtClassMixin
from dbt import plugins
from dbt_semantic_interfaces.enum_extension import assert_values_exhausted

View File

@@ -1,9 +1,9 @@
from copy import deepcopy
from dbt.context.context_config import ContextConfig
from dbt.contracts.graph.nodes import ModelNode, RefArgs
from dbt.events.base_types import EventLevel
from dbt.events.types import Note
from dbt.events.functions import fire_event_if_test
from dbt.common.events.base_types import EventLevel
from dbt.common.events.types import Note
from dbt.common.events.functions import fire_event_if_test
from dbt.flags import get_flags
from dbt.node_types import NodeType, ModelLanguage
from dbt.parser.base import SimpleSQLParser
@@ -19,7 +19,7 @@ from typing import Any, Dict, Iterator, List, Optional, Tuple, Union
# New for Python models :p
import ast
from dbt.dataclass_schema import ValidationError
from dbt.common.dataclass_schema import ValidationError
from dbt.exceptions import (
ModelConfigError,
ParsingError,

View File

@@ -8,9 +8,9 @@ from dbt.contracts.files import (
parse_file_type_to_parser,
SchemaSourceFile,
)
from dbt.events.functions import fire_event
from dbt.events.base_types import EventLevel
from dbt.events.types import (
from dbt.common.events.functions import fire_event
from dbt.common.events.base_types import EventLevel
from dbt.common.events.types import (
PartialParsingEnabled,
PartialParsingFile,
)

View File

@@ -12,13 +12,13 @@ from dbt.contracts.files import (
SchemaSourceFile,
)
from dbt.config import Project
from dbt.dataclass_schema import dbtClassMixin
from dbt.common.dataclass_schema import dbtClassMixin
from dbt.parser.schemas import yaml_from_file, schema_file_keys
from dbt.exceptions import ParsingError
from dbt.parser.search import filesystem_search
from typing import Optional, Dict, List, Mapping, MutableMapping
from dbt.events.types import InputFileDiffError
from dbt.events.functions import fire_event
from dbt.common.events.types import InputFileDiffError
from dbt.common.events.functions import fire_event
from typing import Protocol

View File

@@ -24,7 +24,7 @@ from dbt.contracts.graph.nodes import (
)
from dbt.context.context_config import ContextConfig
from dbt.context.configured import generate_schema_yml_context, SchemaYamlVars
from dbt.dataclass_schema import ValidationError
from dbt.common.dataclass_schema import ValidationError
from dbt.exceptions import SchemaConfigError, CompilationError, ParsingError, TestConfigError
from dbt.contracts.files import FileHash
from dbt.utils import md5, get_pseudo_test_path

View File

@@ -46,7 +46,7 @@ from dbt.context.context_config import (
UnrenderedConfigGenerator,
)
from dbt.clients.jinja import get_rendered
from dbt.dataclass_schema import ValidationError
from dbt.common.dataclass_schema import ValidationError
from dbt_semantic_interfaces.type_enums import (
AggregationType,
DimensionType,

View File

@@ -5,7 +5,7 @@ from abc import ABCMeta, abstractmethod
from typing import Any, Callable, Dict, Generic, Iterable, List, Optional, Type, TypeVar
from dataclasses import dataclass, field
from dbt.dataclass_schema import ValidationError, dbtClassMixin
from dbt.common.dataclass_schema import ValidationError, dbtClassMixin
from dbt.clients.yaml_helper import load_yaml_text
from dbt.parser.schema_renderer import SchemaYamlRenderer
@@ -44,8 +44,8 @@ from dbt.exceptions import (
YamlParseListError,
InvalidAccessTypeError,
)
from dbt.events.functions import warn_or_error
from dbt.events.types import (
from dbt.common.events.functions import warn_or_error
from dbt.common.events.types import (
MacroNotFoundForPatch,
NoNodeForYamlKey,
ValidationWarning,

View File

@@ -1,7 +1,7 @@
import os
from typing import List
from dbt.dataclass_schema import ValidationError
from dbt.common.dataclass_schema import ValidationError
from dbt.contracts.graph.nodes import IntermediateSnapshotNode, SnapshotNode
from dbt.exceptions import SnapshopConfigError

View File

@@ -1,8 +1,6 @@
import itertools
from pathlib import Path
from typing import Iterable, Dict, Optional, Set, Any, List
from dbt.adapters.capability import Capability
from dbt.adapters.factory import get_adapter
from dbt.config import RuntimeConfig
from dbt.context.context_config import (
@@ -26,9 +24,8 @@ from dbt.contracts.graph.unparsed import (
UnparsedColumn,
Time,
)
from dbt.events.functions import warn_or_error, fire_event
from dbt.events.types import UnusedTables, FreshnessConfigProblem
from dbt.common.events.functions import warn_or_error
from dbt.common.events.types import UnusedTables
from dbt.exceptions import DbtInternalError
from dbt.node_types import NodeType
@@ -187,21 +184,6 @@ class SourcePatcher:
unrendered_config=unrendered_config,
)
if (
parsed_source.freshness
and not parsed_source.loaded_at_field
and not get_adapter(self.root_project).supports(Capability.TableLastModifiedMetadata)
):
# Metadata-based freshness is being used by default for this node,
# but is not available through the configured adapter, so warn the
# user that freshness info will not be collected for this node at
# runtime.
fire_event(
FreshnessConfigProblem(
msg=f"The configured adapter does not support metadata-based freshness. A loaded_at_field must be specified for source '{source.name}'."
)
)
# relation name is added after instantiation because the adapter does
# not provide the relation name for a UnpatchedSourceDefinition object
parsed_source.relation_name = self._get_relation_name(parsed_source)

View File

@@ -3,7 +3,7 @@ from typing import Dict
# just exports, they need "noqa" so flake8 will not complain.
from dbt.contracts.util import ArtifactMixin as PluginArtifact, schema_version # noqa
from dbt.contracts.util import BaseArtifactMetadata, AdditionalPropertiesMixin # noqa
from dbt.dataclass_schema import dbtClassMixin, ExtensibleDbtClassMixin # noqa
from dbt.common.dataclass_schema import dbtClassMixin, ExtensibleDbtClassMixin # noqa
PluginArtifacts = Dict[str, PluginArtifact]

View File

@@ -5,7 +5,7 @@ from typing import List
from dbt.exceptions import VersionsNotCompatibleError
import dbt.utils
from dbt.dataclass_schema import dbtClassMixin, StrEnum
from dbt.common.dataclass_schema import dbtClassMixin, StrEnum
from typing import Optional

View File

@@ -5,7 +5,6 @@ import traceback
from abc import ABCMeta, abstractmethod
from contextlib import nullcontext
from datetime import datetime
from pathlib import Path
from typing import Any, Dict, List, Optional, Type, Union
import dbt.exceptions
@@ -13,7 +12,6 @@ from dbt import tracking
from dbt.adapters.factory import get_adapter
from dbt.config import RuntimeConfig, Project
from dbt.config.profile import read_profile
from dbt.constants import DBT_PROJECT_FILE_NAME
from dbt.contracts.graph.manifest import Manifest
from dbt.contracts.results import (
NodeStatus,
@@ -23,9 +21,9 @@ from dbt.contracts.results import (
RunningStatus,
TimingInfo,
)
from dbt.events.contextvars import get_node_info
from dbt.events.functions import fire_event
from dbt.events.types import (
from dbt.common.events.contextvars import get_node_info
from dbt.common.events.functions import fire_event
from dbt.common.events.types import (
LogDbtProjectError,
LogDbtProfileError,
CatchableExceptionOnRun,
@@ -47,7 +45,7 @@ from dbt.exceptions import (
from dbt.flags import get_flags
from dbt.graph import Graph
from dbt.logger import log_manager
from dbt.task.printer import print_run_result_error
from .printer import print_run_result_error
class NoneConfig:
@@ -119,32 +117,35 @@ class BaseTask(metaclass=ABCMeta):
return True
def get_nearest_project_dir(project_dir: Optional[str]) -> Path:
def get_nearest_project_dir(project_dir: Optional[str]) -> str:
# If the user provides an explicit project directory, use that
# but don't look at parent directories.
if project_dir:
cur_dir = Path(project_dir)
project_file = Path(project_dir) / DBT_PROJECT_FILE_NAME
if project_file.is_file():
return cur_dir
project_file = os.path.join(project_dir, "dbt_project.yml")
if os.path.exists(project_file):
return project_dir
else:
raise dbt.exceptions.DbtRuntimeError(
"fatal: Invalid --project-dir flag. Not a dbt project. "
"Missing dbt_project.yml file"
)
cur_dir = Path.cwd()
project_file = cur_dir / DBT_PROJECT_FILE_NAME
if project_file.is_file():
return cur_dir
else:
raise dbt.exceptions.DbtRuntimeError(
"fatal: Not a dbt project (or any of the parent directories). "
"Missing dbt_project.yml file"
)
root_path = os.path.abspath(os.sep)
cwd = os.getcwd()
while cwd != root_path:
project_file = os.path.join(cwd, "dbt_project.yml")
if os.path.exists(project_file):
return cwd
cwd = os.path.dirname(cwd)
raise dbt.exceptions.DbtRuntimeError(
"fatal: Not a dbt project (or any of the parent directories). "
"Missing dbt_project.yml file"
)
def move_to_nearest_project_dir(project_dir: Optional[str]) -> Path:
def move_to_nearest_project_dir(project_dir: Optional[str]) -> str:
nearest_project_dir = get_nearest_project_dir(project_dir)
os.chdir(nearest_project_dir)
return nearest_project_dir

View File

@@ -2,8 +2,8 @@ from pathlib import Path
from shutil import rmtree
from dbt import deprecations
from dbt.events.functions import fire_event
from dbt.events.types import (
from dbt.common.events.functions import fire_event
from dbt.common.events.types import (
CheckCleanPath,
ConfirmCleanPath,
FinishedCleanPaths,

View File

@@ -5,7 +5,7 @@ from dbt.adapters.base import BaseRelation
from dbt.clients.jinja import MacroGenerator
from dbt.context.providers import generate_runtime_model_context
from dbt.contracts.results import RunStatus, RunResult
from dbt.dataclass_schema import dbtClassMixin
from dbt.common.dataclass_schema import dbtClassMixin
from dbt.exceptions import DbtInternalError, CompilationError
from dbt.graph import ResourceTypeSelector
from dbt.node_types import NodeType

Some files were not shown because too many files have changed in this diff Show More