mirror of
https://github.com/dbt-labs/dbt-core
synced 2025-12-22 08:51:28 +00:00
Compare commits
5 Commits
enable-pos
...
spike/deco
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
699fba83b9 | ||
|
|
e9f81a99f6 | ||
|
|
a65dc24274 | ||
|
|
29f734df9f | ||
|
|
5fcd2b83f3 |
6
.changes/unreleased/Features-20230915-123733.yaml
Normal file
6
.changes/unreleased/Features-20230915-123733.yaml
Normal file
@@ -0,0 +1,6 @@
|
||||
kind: Features
|
||||
body: 'Allow adapters to include package logs in dbt standard logging '
|
||||
time: 2023-09-15T12:37:33.862862-07:00
|
||||
custom:
|
||||
Author: colin-rogers-dbt
|
||||
Issue: "7859"
|
||||
6
.changes/unreleased/Under the Hood-20230831-164435.yaml
Normal file
6
.changes/unreleased/Under the Hood-20230831-164435.yaml
Normal file
@@ -0,0 +1,6 @@
|
||||
kind: Under the Hood
|
||||
body: Added more type annotations.
|
||||
time: 2023-08-31T16:44:35.737954-04:00
|
||||
custom:
|
||||
Author: peterallenwebb
|
||||
Issue: "8537"
|
||||
@@ -1,7 +1,7 @@
|
||||
# Configuration for pre-commit hooks (see https://pre-commit.com/).
|
||||
# Eventually the hooks described here will be run as tests before merging each PR.
|
||||
|
||||
exclude: ^(core/dbt/docs/build/|core/dbt/events/types_pb2.py)
|
||||
exclude: ^(core/dbt/docs/build/|core/dbt/common/events/types_pb2.py)
|
||||
|
||||
# Force all unspecified python hooks to run python 3.8
|
||||
default_language_version:
|
||||
|
||||
2
Makefile
2
Makefile
@@ -40,7 +40,7 @@ dev: dev_req ## Installs dbt-* packages in develop mode along with development d
|
||||
|
||||
.PHONY: proto_types
|
||||
proto_types: ## generates google protobuf python file from types.proto
|
||||
protoc -I=./core/dbt/events --python_out=./core/dbt/events ./core/dbt/events/types.proto
|
||||
protoc -I=./core/dbt/common/events --python_out=./core/dbt/common/events ./core/dbt/common/events/types.proto
|
||||
|
||||
.PHONY: mypy
|
||||
mypy: .env ## Runs mypy against staged changes for static type checking.
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
# these are all just exports, #noqa them so flake8 will be happy
|
||||
|
||||
# TODO: Should we still include this in the `adapters` namespace?
|
||||
from dbt.contracts.connection import Credentials # noqa: F401
|
||||
from dbt.adapters.contracts.connection import Credentials # noqa: F401
|
||||
from dbt.adapters.base.meta import available # noqa: F401
|
||||
from dbt.adapters.base.connections import BaseConnectionManager # noqa: F401
|
||||
from dbt.adapters.base.relation import ( # noqa: F401
|
||||
|
||||
@@ -24,7 +24,7 @@ from typing import (
|
||||
import agate
|
||||
|
||||
import dbt.exceptions
|
||||
from dbt.contracts.connection import (
|
||||
from dbt.adapters.contracts.connection import (
|
||||
Connection,
|
||||
Identifier,
|
||||
ConnectionState,
|
||||
@@ -36,9 +36,9 @@ from dbt.contracts.graph.manifest import Manifest
|
||||
from dbt.adapters.base.query_headers import (
|
||||
MacroQueryStringSetter,
|
||||
)
|
||||
from dbt.events import AdapterLogger
|
||||
from dbt.events.functions import fire_event
|
||||
from dbt.events.types import (
|
||||
from dbt.common.events import AdapterLogger
|
||||
from dbt.common.events.functions import fire_event
|
||||
from dbt.common.events.types import (
|
||||
NewConnection,
|
||||
ConnectionReused,
|
||||
ConnectionLeftOpenInCleanup,
|
||||
@@ -48,7 +48,7 @@ from dbt.events.types import (
|
||||
Rollback,
|
||||
RollbackFailed,
|
||||
)
|
||||
from dbt.events.contextvars import get_node_info
|
||||
from dbt.common.events.contextvars import get_node_info
|
||||
from dbt import flags
|
||||
from dbt.utils import cast_to_str
|
||||
|
||||
|
||||
@@ -17,11 +17,9 @@ from typing import (
|
||||
Set,
|
||||
Tuple,
|
||||
Type,
|
||||
TypedDict,
|
||||
Union,
|
||||
)
|
||||
|
||||
from dbt.adapters.capability import Capability, CapabilityDict
|
||||
from dbt.contracts.graph.nodes import ColumnLevelConstraint, ConstraintType, ModelLevelConstraint
|
||||
|
||||
import agate
|
||||
@@ -46,17 +44,12 @@ from dbt.exceptions import (
|
||||
)
|
||||
|
||||
from dbt.adapters.protocol import AdapterConfig
|
||||
from dbt.clients.agate_helper import (
|
||||
empty_table,
|
||||
get_column_value_uncased,
|
||||
merge_tables,
|
||||
table_from_rows,
|
||||
)
|
||||
from dbt.clients.agate_helper import empty_table, merge_tables, table_from_rows
|
||||
from dbt.clients.jinja import MacroGenerator
|
||||
from dbt.contracts.graph.manifest import Manifest, MacroManifest
|
||||
from dbt.contracts.graph.nodes import ResultNode
|
||||
from dbt.events.functions import fire_event, warn_or_error
|
||||
from dbt.events.types import (
|
||||
from dbt.common.events.functions import fire_event, warn_or_error
|
||||
from dbt.common.events.types import (
|
||||
CacheMiss,
|
||||
ListRelations,
|
||||
CodeExecution,
|
||||
@@ -81,9 +74,7 @@ from dbt.adapters.cache import RelationsCache, _make_ref_key_dict
|
||||
from dbt import deprecations
|
||||
|
||||
GET_CATALOG_MACRO_NAME = "get_catalog"
|
||||
GET_CATALOG_RELATIONS_MACRO_NAME = "get_catalog_relations"
|
||||
FRESHNESS_MACRO_NAME = "collect_freshness"
|
||||
GET_RELATION_LAST_MODIFIED_MACRO_NAME = "get_relation_last_modified"
|
||||
|
||||
|
||||
class ConstraintSupport(str, Enum):
|
||||
@@ -118,7 +109,7 @@ def _catalog_filter_schemas(manifest: Manifest) -> Callable[[agate.Row], bool]:
|
||||
return test
|
||||
|
||||
|
||||
def _utc(dt: Optional[datetime], source: Optional[BaseRelation], field_name: str) -> datetime:
|
||||
def _utc(dt: Optional[datetime], source: BaseRelation, field_name: str) -> datetime:
|
||||
"""If dt has a timezone, return a new datetime that's in UTC. Otherwise,
|
||||
assume the datetime is already for UTC and add the timezone.
|
||||
"""
|
||||
@@ -170,12 +161,6 @@ class PythonJobHelper:
|
||||
raise NotImplementedError("PythonJobHelper submit function is not implemented yet")
|
||||
|
||||
|
||||
class FreshnessResponse(TypedDict):
|
||||
max_loaded_at: datetime
|
||||
snapshotted_at: datetime
|
||||
age: float # age in seconds
|
||||
|
||||
|
||||
class BaseAdapter(metaclass=AdapterMeta):
|
||||
"""The BaseAdapter provides an abstract base class for adapters.
|
||||
|
||||
@@ -237,10 +222,6 @@ class BaseAdapter(metaclass=AdapterMeta):
|
||||
ConstraintType.foreign_key: ConstraintSupport.ENFORCED,
|
||||
}
|
||||
|
||||
# This static member variable can be overriden in concrete adapter
|
||||
# implementations to indicate adapter support for optional capabilities.
|
||||
_capabilities = CapabilityDict({})
|
||||
|
||||
def __init__(self, config) -> None:
|
||||
self.config = config
|
||||
self.cache = RelationsCache()
|
||||
@@ -434,8 +415,16 @@ class BaseAdapter(metaclass=AdapterMeta):
|
||||
lowercase strings.
|
||||
"""
|
||||
info_schema_name_map = SchemaSearchMap()
|
||||
relations = self._get_catalog_relations(manifest)
|
||||
for relation in relations:
|
||||
nodes: Iterator[ResultNode] = chain(
|
||||
[
|
||||
node
|
||||
for node in manifest.nodes.values()
|
||||
if (node.is_relational and not node.is_ephemeral_model)
|
||||
],
|
||||
manifest.sources.values(),
|
||||
)
|
||||
for node in nodes:
|
||||
relation = self.Relation.create_from(self.config, node)
|
||||
info_schema_name_map.add(relation)
|
||||
# result is a map whose keys are information_schema Relations without
|
||||
# identifiers that have appropriate database prefixes, and whose values
|
||||
@@ -443,46 +432,6 @@ class BaseAdapter(metaclass=AdapterMeta):
|
||||
# databases
|
||||
return info_schema_name_map
|
||||
|
||||
def _get_catalog_relations_by_info_schema(
|
||||
self, relations
|
||||
) -> Dict[InformationSchema, List[BaseRelation]]:
|
||||
relations_by_info_schema: Dict[InformationSchema, List[BaseRelation]] = dict()
|
||||
for relation in relations:
|
||||
info_schema = relation.information_schema_only()
|
||||
if info_schema not in relations_by_info_schema:
|
||||
relations_by_info_schema[info_schema] = []
|
||||
relations_by_info_schema[info_schema].append(relation)
|
||||
|
||||
return relations_by_info_schema
|
||||
|
||||
def _get_catalog_relations(
|
||||
self, manifest: Manifest, selected_nodes: Optional[Set] = None
|
||||
) -> List[BaseRelation]:
|
||||
nodes: Iterator[ResultNode]
|
||||
if selected_nodes:
|
||||
selected: List[ResultNode] = []
|
||||
for unique_id in selected_nodes:
|
||||
if unique_id in manifest.nodes:
|
||||
node = manifest.nodes[unique_id]
|
||||
if node.is_relational and not node.is_ephemeral_model:
|
||||
selected.append(node)
|
||||
elif unique_id in manifest.sources:
|
||||
source = manifest.sources[unique_id]
|
||||
selected.append(source)
|
||||
nodes = iter(selected)
|
||||
else:
|
||||
nodes = chain(
|
||||
[
|
||||
node
|
||||
for node in manifest.nodes.values()
|
||||
if (node.is_relational and not node.is_ephemeral_model)
|
||||
],
|
||||
manifest.sources.values(),
|
||||
)
|
||||
|
||||
relations = [self.Relation.create_from(self.config, n) for n in nodes]
|
||||
return relations
|
||||
|
||||
def _relations_cache_for_schemas(
|
||||
self, manifest: Manifest, cache_schemas: Optional[Set[BaseRelation]] = None
|
||||
) -> None:
|
||||
@@ -1144,60 +1093,20 @@ class BaseAdapter(metaclass=AdapterMeta):
|
||||
results = self._catalog_filter_table(table, manifest) # type: ignore[arg-type]
|
||||
return results
|
||||
|
||||
def _get_one_catalog_by_relations(
|
||||
self,
|
||||
information_schema: InformationSchema,
|
||||
relations: List[BaseRelation],
|
||||
manifest: Manifest,
|
||||
) -> agate.Table:
|
||||
|
||||
kwargs = {
|
||||
"information_schema": information_schema,
|
||||
"relations": relations,
|
||||
}
|
||||
table = self.execute_macro(
|
||||
GET_CATALOG_RELATIONS_MACRO_NAME,
|
||||
kwargs=kwargs,
|
||||
# pass in the full manifest, so we get any local project
|
||||
# overrides
|
||||
manifest=manifest,
|
||||
)
|
||||
|
||||
results = self._catalog_filter_table(table, manifest) # type: ignore[arg-type]
|
||||
return results
|
||||
|
||||
def get_catalog(
|
||||
self, manifest: Manifest, selected_nodes: Optional[Set] = None
|
||||
) -> Tuple[agate.Table, List[Exception]]:
|
||||
def get_catalog(self, manifest: Manifest) -> Tuple[agate.Table, List[Exception]]:
|
||||
schema_map = self._get_catalog_schemas(manifest)
|
||||
|
||||
with executor(self.config) as tpe:
|
||||
futures: List[Future[agate.Table]] = []
|
||||
catalog_relations = self._get_catalog_relations(manifest, selected_nodes)
|
||||
relation_count = len(catalog_relations)
|
||||
if relation_count <= 100 and self.supports(Capability.SchemaMetadataByRelations):
|
||||
relations_by_schema = self._get_catalog_relations_by_info_schema(catalog_relations)
|
||||
for info_schema in relations_by_schema:
|
||||
name = ".".join([str(info_schema.database), "information_schema"])
|
||||
relations = relations_by_schema[info_schema]
|
||||
fut = tpe.submit_connected(
|
||||
self,
|
||||
name,
|
||||
self._get_one_catalog_by_relations,
|
||||
info_schema,
|
||||
relations,
|
||||
manifest,
|
||||
)
|
||||
futures.append(fut)
|
||||
else:
|
||||
schema_map: SchemaSearchMap = self._get_catalog_schemas(manifest)
|
||||
for info, schemas in schema_map.items():
|
||||
if len(schemas) == 0:
|
||||
continue
|
||||
name = ".".join([str(info.database), "information_schema"])
|
||||
fut = tpe.submit_connected(
|
||||
self, name, self._get_one_catalog, info, schemas, manifest
|
||||
)
|
||||
futures.append(fut)
|
||||
for info, schemas in schema_map.items():
|
||||
if len(schemas) == 0:
|
||||
continue
|
||||
name = ".".join([str(info.database), "information_schema"])
|
||||
|
||||
fut = tpe.submit_connected(
|
||||
self, name, self._get_one_catalog, info, schemas, manifest
|
||||
)
|
||||
futures.append(fut)
|
||||
|
||||
catalogs, exceptions = catch_as_completed(futures)
|
||||
|
||||
@@ -1213,7 +1122,7 @@ class BaseAdapter(metaclass=AdapterMeta):
|
||||
loaded_at_field: str,
|
||||
filter: Optional[str],
|
||||
manifest: Optional[Manifest] = None,
|
||||
) -> Tuple[Optional[AdapterResponse], FreshnessResponse]:
|
||||
) -> Tuple[Optional[AdapterResponse], Dict[str, Any]]:
|
||||
"""Calculate the freshness of sources in dbt, and return it"""
|
||||
kwargs: Dict[str, Any] = {
|
||||
"source": source,
|
||||
@@ -1248,52 +1157,13 @@ class BaseAdapter(metaclass=AdapterMeta):
|
||||
|
||||
snapshotted_at = _utc(table[0][1], source, loaded_at_field)
|
||||
age = (snapshotted_at - max_loaded_at).total_seconds()
|
||||
freshness: FreshnessResponse = {
|
||||
freshness = {
|
||||
"max_loaded_at": max_loaded_at,
|
||||
"snapshotted_at": snapshotted_at,
|
||||
"age": age,
|
||||
}
|
||||
return adapter_response, freshness
|
||||
|
||||
def calculate_freshness_from_metadata(
|
||||
self,
|
||||
source: BaseRelation,
|
||||
manifest: Optional[Manifest] = None,
|
||||
) -> Tuple[Optional[AdapterResponse], FreshnessResponse]:
|
||||
kwargs: Dict[str, Any] = {
|
||||
"information_schema": source.information_schema_only(),
|
||||
"relations": [source],
|
||||
}
|
||||
result = self.execute_macro(
|
||||
GET_RELATION_LAST_MODIFIED_MACRO_NAME, kwargs=kwargs, manifest=manifest
|
||||
)
|
||||
adapter_response, table = result.response, result.table # type: ignore[attr-defined]
|
||||
|
||||
try:
|
||||
row = table[0]
|
||||
last_modified_val = get_column_value_uncased("last_modified", row)
|
||||
snapshotted_at_val = get_column_value_uncased("snapshotted_at", row)
|
||||
except Exception:
|
||||
raise MacroResultError(GET_RELATION_LAST_MODIFIED_MACRO_NAME, table)
|
||||
|
||||
if last_modified_val is None:
|
||||
# Interpret missing value as "infinitely long ago"
|
||||
max_loaded_at = datetime(1, 1, 1, 0, 0, 0, tzinfo=pytz.UTC)
|
||||
else:
|
||||
max_loaded_at = _utc(last_modified_val, None, "last_modified")
|
||||
|
||||
snapshotted_at = _utc(snapshotted_at_val, None, "snapshotted_at")
|
||||
|
||||
age = (snapshotted_at - max_loaded_at).total_seconds()
|
||||
|
||||
freshness: FreshnessResponse = {
|
||||
"max_loaded_at": max_loaded_at,
|
||||
"snapshotted_at": snapshotted_at,
|
||||
"age": age,
|
||||
}
|
||||
|
||||
return adapter_response, freshness
|
||||
|
||||
def pre_model_hook(self, config: Mapping[str, Any]) -> Any:
|
||||
"""A hook for running some operation before the model materialization
|
||||
runs. The hook can assume it has a connection available.
|
||||
@@ -1567,14 +1437,6 @@ class BaseAdapter(metaclass=AdapterMeta):
|
||||
else:
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
def capabilities(cls) -> CapabilityDict:
|
||||
return cls._capabilities
|
||||
|
||||
@classmethod
|
||||
def supports(cls, capability: Capability) -> bool:
|
||||
return bool(cls.capabilities()[capability])
|
||||
|
||||
|
||||
COLUMNS_EQUAL_SQL = """
|
||||
with diff_count as (
|
||||
|
||||
@@ -4,7 +4,7 @@ from typing import Optional, Callable, Dict, Any
|
||||
from dbt.clients.jinja import QueryStringGenerator
|
||||
|
||||
from dbt.context.manifest import generate_query_header_context
|
||||
from dbt.contracts.connection import AdapterRequiredConfig, QueryComment
|
||||
from dbt.adapters.contracts.connection import AdapterRequiredConfig, QueryComment
|
||||
from dbt.contracts.graph.nodes import ResultNode
|
||||
from dbt.contracts.graph.manifest import Manifest
|
||||
from dbt.exceptions import DbtRuntimeError
|
||||
|
||||
@@ -14,8 +14,8 @@ from dbt.exceptions import (
|
||||
ReferencedLinkNotCachedError,
|
||||
TruncatedModelNameCausedCollisionError,
|
||||
)
|
||||
from dbt.events.functions import fire_event, fire_event_if
|
||||
from dbt.events.types import CacheAction, CacheDumpGraph
|
||||
from dbt.common.events.functions import fire_event, fire_event_if
|
||||
from dbt.common.events.types import CacheAction, CacheDumpGraph
|
||||
from dbt.flags import get_flags
|
||||
from dbt.utils import lowercase
|
||||
|
||||
|
||||
@@ -11,20 +11,27 @@ from typing import (
|
||||
List,
|
||||
Callable,
|
||||
)
|
||||
from dbt.exceptions import DbtInternalError
|
||||
from dbt.utils import translate_aliases, md5
|
||||
from dbt.events.functions import fire_event
|
||||
from dbt.events.types import NewConnectionOpening
|
||||
from dbt.events.contextvars import get_node_info
|
||||
from typing_extensions import Protocol, Annotated
|
||||
from dbt.dataclass_schema import (
|
||||
|
||||
from mashumaro.jsonschema.annotations import Pattern
|
||||
|
||||
from dbt.adapters.utils import translate_aliases
|
||||
from dbt.common.exceptions import DbtInternalError
|
||||
from dbt.common.dataclass_schema import (
|
||||
dbtClassMixin,
|
||||
StrEnum,
|
||||
ExtensibleDbtClassMixin,
|
||||
ValidatedStringMixin,
|
||||
)
|
||||
from dbt.contracts.util import Replaceable
|
||||
from mashumaro.jsonschema.annotations import Pattern
|
||||
from dbt.common.contracts.util import Replaceable
|
||||
from dbt.common.util import md5
|
||||
|
||||
# TODO: dbt.common.events dependency
|
||||
from dbt.common.events.functions import fire_event
|
||||
from dbt.common.events.types import NewConnectionOpening
|
||||
|
||||
# TODO: this is a very bad dependency - shared global state
|
||||
from dbt.common.events.contextvars import get_node_info
|
||||
|
||||
|
||||
class Identifier(ValidatedStringMixin):
|
||||
24
core/dbt/adapters/exceptions.py
Normal file
24
core/dbt/adapters/exceptions.py
Normal file
@@ -0,0 +1,24 @@
|
||||
from typing import Mapping, Any
|
||||
|
||||
from dbt.exceptions import DbtValidationError
|
||||
|
||||
|
||||
class AliasError(DbtValidationError):
|
||||
pass
|
||||
|
||||
|
||||
# core level exceptions
|
||||
class DuplicateAliasError(AliasError):
|
||||
def __init__(self, kwargs: Mapping[str, Any], aliases: Mapping[str, str], canonical_key: str):
|
||||
self.kwargs = kwargs
|
||||
self.aliases = aliases
|
||||
self.canonical_key = canonical_key
|
||||
super().__init__(msg=self.get_message())
|
||||
|
||||
def get_message(self) -> str:
|
||||
# dupe found: go through the dict so we can have a nice-ish error
|
||||
key_names = ", ".join(
|
||||
"{}".format(k) for k in self.kwargs if self.aliases.get(k) == self.canonical_key
|
||||
)
|
||||
msg = f'Got duplicate keys: ({key_names}) all map to "{self.canonical_key}"'
|
||||
return msg
|
||||
@@ -7,9 +7,9 @@ from typing import Any, Dict, List, Optional, Set, Type
|
||||
|
||||
from dbt.adapters.base.plugin import AdapterPlugin
|
||||
from dbt.adapters.protocol import AdapterConfig, AdapterProtocol, RelationProtocol
|
||||
from dbt.contracts.connection import AdapterRequiredConfig, Credentials
|
||||
from dbt.events.functions import fire_event
|
||||
from dbt.events.types import AdapterImportError, PluginLoadError, AdapterRegistered
|
||||
from dbt.adapters.contracts.connection import AdapterRequiredConfig, Credentials
|
||||
from dbt.common.events.functions import fire_event
|
||||
from dbt.common.events.types import AdapterImportError, PluginLoadError, AdapterRegistered
|
||||
from dbt.exceptions import DbtInternalError, DbtRuntimeError
|
||||
from dbt.include.global_project import PACKAGE_PATH as GLOBAL_PROJECT_PATH
|
||||
from dbt.include.global_project import PROJECT_NAME as GLOBAL_PROJECT_NAME
|
||||
|
||||
@@ -15,7 +15,7 @@ from typing_extensions import Protocol
|
||||
|
||||
import agate
|
||||
|
||||
from dbt.contracts.connection import Connection, AdapterRequiredConfig, AdapterResponse
|
||||
from dbt.adapters.contracts.connection import Connection, AdapterRequiredConfig, AdapterResponse
|
||||
from dbt.contracts.graph.nodes import ResultNode, ManifestNode
|
||||
from dbt.contracts.graph.model_config import BaseConfig
|
||||
from dbt.contracts.graph.manifest import Manifest
|
||||
|
||||
@@ -3,7 +3,7 @@ from dataclasses import dataclass
|
||||
from typing import Hashable
|
||||
|
||||
from dbt.adapters.relation_configs.config_base import RelationConfigBase
|
||||
from dbt.dataclass_schema import StrEnum
|
||||
from dbt.common.dataclass_schema import StrEnum
|
||||
|
||||
|
||||
class RelationConfigChangeAction(StrEnum):
|
||||
|
||||
@@ -7,10 +7,10 @@ import agate
|
||||
import dbt.clients.agate_helper
|
||||
import dbt.exceptions
|
||||
from dbt.adapters.base import BaseConnectionManager
|
||||
from dbt.contracts.connection import Connection, ConnectionState, AdapterResponse
|
||||
from dbt.events.functions import fire_event
|
||||
from dbt.events.types import ConnectionUsed, SQLQuery, SQLCommit, SQLQueryStatus
|
||||
from dbt.events.contextvars import get_node_info
|
||||
from dbt.adapters.contracts.connection import Connection, ConnectionState, AdapterResponse
|
||||
from dbt.common.events.functions import fire_event
|
||||
from dbt.common.events.types import ConnectionUsed, SQLQuery, SQLCommit, SQLQueryStatus
|
||||
from dbt.common.events.contextvars import get_node_info
|
||||
from dbt.utils import cast_to_str
|
||||
|
||||
|
||||
|
||||
@@ -1,13 +1,13 @@
|
||||
import agate
|
||||
from typing import Any, Optional, Tuple, Type, List
|
||||
|
||||
from dbt.contracts.connection import Connection, AdapterResponse
|
||||
from dbt.adapters.contracts.connection import Connection, AdapterResponse
|
||||
from dbt.exceptions import RelationTypeNullError
|
||||
from dbt.adapters.base import BaseAdapter, available
|
||||
from dbt.adapters.cache import _make_ref_key_dict
|
||||
from dbt.adapters.sql import SQLConnectionManager
|
||||
from dbt.events.functions import fire_event
|
||||
from dbt.events.types import ColTypeChange, SchemaCreation, SchemaDrop
|
||||
from dbt.common.events.functions import fire_event
|
||||
from dbt.common.events.types import ColTypeChange, SchemaCreation, SchemaDrop
|
||||
|
||||
|
||||
from dbt.adapters.base.relation import BaseRelation
|
||||
|
||||
55
core/dbt/adapters/utils.py
Normal file
55
core/dbt/adapters/utils.py
Normal file
@@ -0,0 +1,55 @@
|
||||
from typing import Mapping, Sequence, Any, Dict, List
|
||||
from dbt.adapters.exceptions import DuplicateAliasError
|
||||
|
||||
|
||||
class Translator:
|
||||
def __init__(self, aliases: Mapping[str, str], recursive: bool = False):
|
||||
self.aliases = aliases
|
||||
self.recursive = recursive
|
||||
|
||||
def translate_mapping(self, kwargs: Mapping[str, Any]) -> Dict[str, Any]:
|
||||
result: Dict[str, Any] = {}
|
||||
|
||||
for key, value in kwargs.items():
|
||||
canonical_key = self.aliases.get(key, key)
|
||||
if canonical_key in result:
|
||||
raise DuplicateAliasError(kwargs, self.aliases, canonical_key)
|
||||
result[canonical_key] = self.translate_value(value)
|
||||
return result
|
||||
|
||||
def translate_sequence(self, value: Sequence[Any]) -> List[Any]:
|
||||
return [self.translate_value(v) for v in value]
|
||||
|
||||
def translate_value(self, value: Any) -> Any:
|
||||
if self.recursive:
|
||||
if isinstance(value, Mapping):
|
||||
return self.translate_mapping(value)
|
||||
elif isinstance(value, (list, tuple)):
|
||||
return self.translate_sequence(value)
|
||||
return value
|
||||
|
||||
def translate(self, value: Mapping[str, Any]) -> Dict[str, Any]:
|
||||
try:
|
||||
return self.translate_mapping(value)
|
||||
except RuntimeError as exc:
|
||||
if "maximum recursion depth exceeded" in str(exc):
|
||||
raise RecursionError("Cycle detected in a value passed to translate!")
|
||||
raise
|
||||
|
||||
|
||||
def translate_aliases(
|
||||
kwargs: Dict[str, Any],
|
||||
aliases: Dict[str, str],
|
||||
recurse: bool = False,
|
||||
) -> Dict[str, Any]:
|
||||
"""Given a dict of keyword arguments and a dict mapping aliases to their
|
||||
canonical values, canonicalize the keys in the kwargs dict.
|
||||
|
||||
If recurse is True, perform this operation recursively.
|
||||
|
||||
:returns: A dict containing all the values in kwargs referenced by their
|
||||
canonical key.
|
||||
:raises: `AliasError`, if a canonical key is defined more than once.
|
||||
"""
|
||||
translator = Translator(aliases, recurse)
|
||||
return translator.translate(kwargs)
|
||||
@@ -21,7 +21,7 @@ from dbt.contracts.results import (
|
||||
CatalogArtifact,
|
||||
RunExecutionResult,
|
||||
)
|
||||
from dbt.events.base_types import EventMsg
|
||||
from dbt.common.events.base_types import EventMsg
|
||||
from dbt.task.build import BuildTask
|
||||
from dbt.task.clean import CleanTask
|
||||
from dbt.task.clone import CloneTask
|
||||
|
||||
@@ -9,17 +9,20 @@ from dbt.cli.exceptions import (
|
||||
from dbt.cli.flags import Flags
|
||||
from dbt.config import RuntimeConfig
|
||||
from dbt.config.runtime import load_project, load_profile, UnsetProfile
|
||||
from dbt.events.base_types import EventLevel
|
||||
from dbt.events.functions import fire_event, LOG_VERSION, set_invocation_id, setup_event_logger
|
||||
from dbt.events.types import (
|
||||
from dbt.common.events.functions import (
|
||||
fire_event,
|
||||
LOG_VERSION,
|
||||
set_invocation_id,
|
||||
setup_event_logger,
|
||||
)
|
||||
from dbt.common.events.types import (
|
||||
CommandCompleted,
|
||||
MainReportVersion,
|
||||
MainReportArgs,
|
||||
MainTrackingUserState,
|
||||
ResourceReport,
|
||||
)
|
||||
from dbt.events.helpers import get_json_string_utcnow
|
||||
from dbt.events.types import MainEncounteredError, MainStackTrace
|
||||
from dbt.common.events.helpers import get_json_string_utcnow
|
||||
from dbt.common.events.types import MainEncounteredError, MainStackTrace
|
||||
from dbt.exceptions import Exception as DbtException, DbtProjectError, FailFastError
|
||||
from dbt.parser.manifest import ManifestLoader, write_manifest
|
||||
from dbt.profiler import profiler
|
||||
@@ -29,7 +32,6 @@ from dbt.plugins import set_up_plugin_manager, get_plugin_manager
|
||||
|
||||
from click import Context
|
||||
from functools import update_wrapper
|
||||
import importlib.util
|
||||
import time
|
||||
import traceback
|
||||
|
||||
@@ -99,28 +101,6 @@ def postflight(func):
|
||||
fire_event(MainStackTrace(stack_trace=traceback.format_exc()))
|
||||
raise ExceptionExit(e)
|
||||
finally:
|
||||
# Fire ResourceReport, but only on systems which support the resource
|
||||
# module. (Skip it on Windows).
|
||||
if importlib.util.find_spec("resource") is not None:
|
||||
import resource
|
||||
|
||||
rusage = resource.getrusage(resource.RUSAGE_SELF)
|
||||
fire_event(
|
||||
ResourceReport(
|
||||
command_name=ctx.command.name,
|
||||
command_success=success,
|
||||
command_wall_clock_time=time.perf_counter() - start_func,
|
||||
process_user_time=rusage.ru_utime,
|
||||
process_kernel_time=rusage.ru_stime,
|
||||
process_mem_max_rss=rusage.ru_maxrss,
|
||||
process_in_blocks=rusage.ru_inblock,
|
||||
process_out_blocks=rusage.ru_oublock,
|
||||
),
|
||||
EventLevel.INFO
|
||||
if "flags" in ctx.obj and ctx.obj["flags"].SHOW_RESOURCE_REPORT
|
||||
else None,
|
||||
)
|
||||
|
||||
fire_event(
|
||||
CommandCompleted(
|
||||
command=ctx.command_path,
|
||||
|
||||
@@ -2,8 +2,8 @@ import re
|
||||
import os.path
|
||||
|
||||
from dbt.clients.system import run_cmd, rmdir
|
||||
from dbt.events.functions import fire_event
|
||||
from dbt.events.types import (
|
||||
from dbt.common.events.functions import fire_event
|
||||
from dbt.common.events.types import (
|
||||
GitSparseCheckoutSubdirectory,
|
||||
GitProgressCheckoutRevision,
|
||||
GitProgressUpdatingExistingDependency,
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
import functools
|
||||
from typing import Any, Dict, List
|
||||
import requests
|
||||
from dbt.events.functions import fire_event
|
||||
from dbt.events.types import (
|
||||
from dbt.common.events.functions import fire_event
|
||||
from dbt.common.events.types import (
|
||||
RegistryProgressGETRequest,
|
||||
RegistryProgressGETResponse,
|
||||
RegistryIndexProgressGETRequest,
|
||||
|
||||
@@ -15,8 +15,8 @@ from typing import Any, Callable, Dict, List, NoReturn, Optional, Tuple, Type, U
|
||||
|
||||
import dbt.exceptions
|
||||
import requests
|
||||
from dbt.events.functions import fire_event
|
||||
from dbt.events.types import (
|
||||
from dbt.common.events.functions import fire_event
|
||||
from dbt.common.events.types import (
|
||||
SystemCouldNotWrite,
|
||||
SystemExecutingCmd,
|
||||
SystemStdOut,
|
||||
|
||||
0
core/dbt/common/__init__.py
Normal file
0
core/dbt/common/__init__.py
Normal file
1
core/dbt/common/constants.py
Normal file
1
core/dbt/common/constants.py
Normal file
@@ -0,0 +1 @@
|
||||
SECRET_ENV_PREFIX = "DBT_ENV_SECRET_"
|
||||
0
core/dbt/common/contracts/__init__.py
Normal file
0
core/dbt/common/contracts/__init__.py
Normal file
0
core/dbt/common/contracts/connection.py
Normal file
0
core/dbt/common/contracts/connection.py
Normal file
7
core/dbt/common/contracts/util.py
Normal file
7
core/dbt/common/contracts/util.py
Normal file
@@ -0,0 +1,7 @@
|
||||
import dataclasses
|
||||
|
||||
|
||||
# TODO: remove from dbt.contracts.util:: Replaceable + references
|
||||
class Replaceable:
|
||||
def replace(self, **kwargs):
|
||||
return dataclasses.replace(self, **kwargs)
|
||||
@@ -43,7 +43,7 @@ from dbt.logger import GLOBAL_LOGGER as logger
|
||||
|
||||
Simply change it to these two lines with your adapter's database name, and all your existing call sites will now use the new system for v1.0:
|
||||
```python
|
||||
from dbt.events import AdapterLogger
|
||||
from dbt.common.events import AdapterLogger
|
||||
logger = AdapterLogger("<database name>")
|
||||
# e.g. AdapterLogger("Snowflake")
|
||||
```
|
||||
@@ -52,4 +52,4 @@ logger = AdapterLogger("<database name>")
|
||||
|
||||
After adding a new message in `types.proto`, either:
|
||||
- In the repository root directory: `make proto_types`
|
||||
- In the `core/dbt/events` directory: `protoc -I=. --python_out=. types.proto`
|
||||
- In the `core/dbt/common/events` directory: `protoc -I=. --python_out=. types.proto`
|
||||
@@ -1,9 +1,10 @@
|
||||
import traceback
|
||||
from dataclasses import dataclass
|
||||
from dbt.events.functions import fire_event, EVENT_MANAGER
|
||||
from dbt.events.contextvars import get_node_info
|
||||
from dbt.events.event_handler import set_package_logging
|
||||
from dbt.events.types import (
|
||||
|
||||
from dbt.common.events.event_handler import set_package_logging
|
||||
from dbt.common.events.functions import fire_event, EVENT_MANAGER
|
||||
from dbt.common.events.contextvars import get_node_info
|
||||
from dbt.common.events.types import (
|
||||
AdapterEventDebug,
|
||||
AdapterEventInfo,
|
||||
AdapterEventWarning,
|
||||
@@ -1,11 +1,11 @@
|
||||
from enum import Enum
|
||||
import os
|
||||
import threading
|
||||
from dbt.events import types_pb2
|
||||
from dbt.common.events import types_pb2
|
||||
import sys
|
||||
from google.protobuf.json_format import ParseDict, MessageToDict, MessageToJson
|
||||
from google.protobuf.message import Message
|
||||
from dbt.events.helpers import get_json_string_utcnow
|
||||
from dbt.common.events.helpers import get_json_string_utcnow
|
||||
from typing import Optional
|
||||
|
||||
if sys.version_info >= (3, 8):
|
||||
@@ -21,13 +21,13 @@ else:
|
||||
|
||||
|
||||
def get_global_metadata_vars() -> dict:
|
||||
from dbt.events.functions import get_metadata_vars
|
||||
from dbt.common.events.functions import get_metadata_vars
|
||||
|
||||
return get_metadata_vars()
|
||||
|
||||
|
||||
def get_invocation_id() -> str:
|
||||
from dbt.events.functions import get_invocation_id
|
||||
from dbt.common.events.functions import get_invocation_id
|
||||
|
||||
return get_invocation_id()
|
||||
|
||||
@@ -72,8 +72,8 @@ class BaseEvent:
|
||||
self.pb_msg = ParseDict(kwargs, msg_cls())
|
||||
except Exception:
|
||||
# Imports need to be here to avoid circular imports
|
||||
from dbt.events.types import Note
|
||||
from dbt.events.functions import fire_event
|
||||
from dbt.common.events.types import Note
|
||||
from dbt.common.events.functions import fire_event
|
||||
|
||||
error_msg = f"[{class_name}]: Unable to parse dict {kwargs}"
|
||||
# If we're testing throw an error so that we notice failures
|
||||
@@ -102,10 +102,7 @@ class BaseEvent:
|
||||
|
||||
def to_json(self) -> str:
|
||||
return MessageToJson(
|
||||
self.pb_msg,
|
||||
preserving_proto_field_name=True,
|
||||
including_default_value_fields=True,
|
||||
indent=None,
|
||||
self.pb_msg, preserving_proto_field_name=True, including_default_value_fields=True
|
||||
)
|
||||
|
||||
def level_tag(self) -> EventLevel:
|
||||
@@ -1,10 +1,10 @@
|
||||
import logging
|
||||
from typing import Union
|
||||
|
||||
from dbt.events.base_types import EventLevel
|
||||
from dbt.events.types import Note
|
||||
from dbt.common.events.base_types import EventLevel
|
||||
from dbt.common.events.types import Note
|
||||
|
||||
from dbt.events.eventmgr import IEventManager
|
||||
from dbt.common.events.eventmgr import IEventManager
|
||||
|
||||
_log_level_to_event_level_map = {
|
||||
logging.DEBUG: EventLevel.DEBUG,
|
||||
@@ -3,8 +3,8 @@ import traceback
|
||||
from typing import Callable, List, Optional, Protocol, Tuple
|
||||
from uuid import uuid4
|
||||
|
||||
from dbt.events.base_types import BaseEvent, EventLevel, msg_from_base_event, EventMsg
|
||||
from dbt.events.logger import LoggerConfig, _Logger, _TextLogger, _JsonLogger, LineFormat
|
||||
from dbt.common.events.base_types import BaseEvent, EventLevel, msg_from_base_event, EventMsg
|
||||
from dbt.common.events.logger import LoggerConfig, _Logger, _TextLogger, _JsonLogger, LineFormat
|
||||
|
||||
|
||||
class EventManager:
|
||||
@@ -1,9 +1,9 @@
|
||||
from dbt.constants import METADATA_ENV_PREFIX
|
||||
from dbt.events.base_types import BaseEvent, EventLevel, EventMsg
|
||||
from dbt.events.eventmgr import EventManager, IEventManager
|
||||
from dbt.events.logger import LoggerConfig, NoFilter, LineFormat
|
||||
from dbt.common.events.base_types import BaseEvent, EventLevel, EventMsg
|
||||
from dbt.common.events.eventmgr import EventManager, IEventManager
|
||||
from dbt.common.events.logger import LoggerConfig, NoFilter, LineFormat
|
||||
from dbt.exceptions import scrub_secrets, env_secrets
|
||||
from dbt.events.types import Note
|
||||
from dbt.common.events.types import Note
|
||||
from dbt.flags import get_flags, ENABLE_LEGACY_LOGGER
|
||||
from dbt.logger import GLOBAL_LOGGER, make_log_dir_if_missing
|
||||
from functools import partial
|
||||
@@ -10,8 +10,8 @@ from typing import Optional, TextIO, Any, Callable
|
||||
from colorama import Style
|
||||
|
||||
import dbt.utils
|
||||
from dbt.events.base_types import EventLevel, EventMsg
|
||||
from dbt.events.format import timestamp_to_datetime_string
|
||||
from dbt.common.events.base_types import EventLevel, EventMsg
|
||||
from dbt.common.events.format import timestamp_to_datetime_string
|
||||
|
||||
# A Filter is a function which takes a BaseEvent and returns True if the event
|
||||
# should be logged, False otherwise.
|
||||
@@ -172,7 +172,7 @@ class _TextLogger(_Logger):
|
||||
|
||||
class _JsonLogger(_Logger):
|
||||
def create_line(self, msg: EventMsg) -> str:
|
||||
from dbt.events.functions import msg_to_dict
|
||||
from dbt.common.events.functions import msg_to_dict
|
||||
|
||||
msg_dict = msg_to_dict(msg)
|
||||
raw_log_line = json.dumps(msg_dict, sort_keys=True, cls=dbt.utils.ForgivingJSONEncoder)
|
||||
@@ -1,7 +1,7 @@
|
||||
import json
|
||||
|
||||
from dbt.constants import MAXIMUM_SEED_SIZE_NAME, PIN_PACKAGE_URL
|
||||
from dbt.events.base_types import (
|
||||
from dbt.common.events.base_types import (
|
||||
DynamicLevel,
|
||||
DebugLevel,
|
||||
InfoLevel,
|
||||
@@ -9,7 +9,11 @@ from dbt.events.base_types import (
|
||||
ErrorLevel,
|
||||
EventLevel,
|
||||
)
|
||||
from dbt.events.format import format_fancy_output_line, pluralize, timestamp_to_datetime_string
|
||||
from dbt.common.events.format import (
|
||||
format_fancy_output_line,
|
||||
pluralize,
|
||||
timestamp_to_datetime_string,
|
||||
)
|
||||
from dbt.node_types import NodeType
|
||||
from dbt.ui import line_wrap_message, warning_tag, red, green, yellow
|
||||
|
||||
@@ -1239,17 +1243,15 @@ class UnversionedBreakingChange(WarnLevel):
|
||||
def message(self) -> str:
|
||||
reasons = "\n - ".join(self.breaking_changes)
|
||||
|
||||
msg = (
|
||||
return (
|
||||
f"Breaking change to contracted, unversioned model {self.model_name} ({self.model_file_path})"
|
||||
"\nWhile comparing to previous project state, dbt detected a breaking change to an unversioned model."
|
||||
f"\n - {reasons}\n"
|
||||
)
|
||||
|
||||
return warning_tag(msg)
|
||||
|
||||
|
||||
class WarnStateTargetEqual(WarnLevel):
|
||||
def code(self) -> str:
|
||||
def code(self):
|
||||
return "I072"
|
||||
|
||||
def message(self) -> str:
|
||||
File diff suppressed because one or more lines are too long
58
core/dbt/common/exceptions.py
Normal file
58
core/dbt/common/exceptions.py
Normal file
@@ -0,0 +1,58 @@
|
||||
from typing import List
|
||||
import os
|
||||
|
||||
from dbt.common.constants import SECRET_ENV_PREFIX
|
||||
|
||||
|
||||
def env_secrets() -> List[str]:
|
||||
return [v for k, v in os.environ.items() if k.startswith(SECRET_ENV_PREFIX) and v.strip()]
|
||||
|
||||
|
||||
def scrub_secrets(msg: str, secrets: List[str]) -> str:
|
||||
scrubbed = str(msg)
|
||||
|
||||
for secret in secrets:
|
||||
scrubbed = scrubbed.replace(secret, "*****")
|
||||
|
||||
return scrubbed
|
||||
|
||||
|
||||
class DbtInternalError(Exception):
|
||||
def __init__(self, msg: str):
|
||||
self.stack: List = []
|
||||
self.msg = scrub_secrets(msg, env_secrets())
|
||||
|
||||
@property
|
||||
def type(self):
|
||||
return "Internal"
|
||||
|
||||
def process_stack(self):
|
||||
lines = []
|
||||
stack = self.stack
|
||||
first = True
|
||||
|
||||
if len(stack) > 1:
|
||||
lines.append("")
|
||||
|
||||
for item in stack:
|
||||
msg = "called by"
|
||||
|
||||
if first:
|
||||
msg = "in"
|
||||
first = False
|
||||
|
||||
lines.append(f"> {msg}")
|
||||
|
||||
return lines
|
||||
|
||||
def __str__(self):
|
||||
if hasattr(self.msg, "split"):
|
||||
split_msg = self.msg.split("\n")
|
||||
else:
|
||||
split_msg = str(self.msg).split("\n")
|
||||
|
||||
lines = ["{}".format(self.type + " Error")] + split_msg
|
||||
|
||||
lines += self.process_stack()
|
||||
|
||||
return lines[0] + "\n" + "\n".join([" " + line for line in lines[1:]])
|
||||
9
core/dbt/common/util.py
Normal file
9
core/dbt/common/util.py
Normal file
@@ -0,0 +1,9 @@
|
||||
import hashlib
|
||||
import sys
|
||||
|
||||
|
||||
def md5(string, charset="utf-8"):
|
||||
if sys.version_info >= (3, 9):
|
||||
return hashlib.md5(string.encode(charset), usedforsecurity=False).hexdigest()
|
||||
else:
|
||||
return hashlib.md5(string.encode(charset)).hexdigest()
|
||||
@@ -28,11 +28,11 @@ from dbt.exceptions import (
|
||||
DbtRuntimeError,
|
||||
)
|
||||
from dbt.graph import Graph
|
||||
from dbt.events.functions import fire_event, get_invocation_id
|
||||
from dbt.events.types import FoundStats, Note, WritingInjectedSQLForNode
|
||||
from dbt.events.contextvars import get_node_info
|
||||
from dbt.common.events.functions import fire_event, get_invocation_id
|
||||
from dbt.common.events.types import FoundStats, Note, WritingInjectedSQLForNode
|
||||
from dbt.common.events.contextvars import get_node_info
|
||||
from dbt.node_types import NodeType, ModelLanguage
|
||||
from dbt.events.format import pluralize
|
||||
from dbt.common.events.format import pluralize
|
||||
import dbt.tracking
|
||||
import dbt.task.list as list_task
|
||||
import sqlparse
|
||||
|
||||
@@ -2,12 +2,12 @@ from dataclasses import dataclass
|
||||
from typing import Any, Dict, Optional, Tuple
|
||||
import os
|
||||
|
||||
from dbt.dataclass_schema import ValidationError
|
||||
from dbt.common.dataclass_schema import ValidationError
|
||||
|
||||
from dbt.flags import get_flags
|
||||
from dbt.clients.system import load_file_contents
|
||||
from dbt.clients.yaml_helper import load_yaml_text
|
||||
from dbt.contracts.connection import Credentials, HasCredentials
|
||||
from dbt.adapters.contracts.connection import Credentials, HasCredentials
|
||||
from dbt.contracts.project import ProfileConfig, UserConfig
|
||||
from dbt.exceptions import (
|
||||
CompilationError,
|
||||
@@ -17,8 +17,8 @@ from dbt.exceptions import (
|
||||
DbtRuntimeError,
|
||||
ProfileConfigError,
|
||||
)
|
||||
from dbt.events.types import MissingProfileTarget
|
||||
from dbt.events.functions import fire_event
|
||||
from dbt.common.events.types import MissingProfileTarget
|
||||
from dbt.common.events.functions import fire_event
|
||||
from dbt.utils import coerce_dict_str
|
||||
|
||||
from .renderer import ProfileRenderer
|
||||
|
||||
@@ -23,7 +23,7 @@ from dbt.constants import (
|
||||
)
|
||||
from dbt.clients.system import path_exists, load_file_contents
|
||||
from dbt.clients.yaml_helper import load_yaml_text
|
||||
from dbt.contracts.connection import QueryComment
|
||||
from dbt.adapters.contracts.connection import QueryComment
|
||||
from dbt.exceptions import (
|
||||
DbtProjectError,
|
||||
SemverError,
|
||||
@@ -43,7 +43,7 @@ from dbt.contracts.project import (
|
||||
SemverString,
|
||||
)
|
||||
from dbt.contracts.project import PackageConfig, ProjectPackageMetadata
|
||||
from dbt.dataclass_schema import ValidationError
|
||||
from dbt.common.dataclass_schema import ValidationError
|
||||
from .renderer import DbtProjectYamlRenderer, PackageRenderer
|
||||
from .selectors import (
|
||||
selector_config_from_data,
|
||||
|
||||
@@ -8,7 +8,7 @@ from dbt.constants import SECRET_ENV_PREFIX, DEPENDENCIES_FILE_NAME
|
||||
from dbt.context.target import TargetContext
|
||||
from dbt.context.secret import SecretContext, SECRET_PLACEHOLDER
|
||||
from dbt.context.base import BaseContext
|
||||
from dbt.contracts.connection import HasCredentials
|
||||
from dbt.adapters.contracts.connection import HasCredentials
|
||||
from dbt.exceptions import DbtProjectError, CompilationError, RecursionError
|
||||
from dbt.utils import deep_map_render
|
||||
|
||||
|
||||
@@ -18,13 +18,13 @@ from typing import (
|
||||
from dbt.flags import get_flags
|
||||
from dbt.adapters.factory import get_include_paths, get_relation_class_by_name
|
||||
from dbt.config.project import load_raw_project
|
||||
from dbt.contracts.connection import AdapterRequiredConfig, Credentials, HasCredentials
|
||||
from dbt.adapters.contracts.connection import AdapterRequiredConfig, Credentials, HasCredentials
|
||||
from dbt.contracts.graph.manifest import ManifestMetadata
|
||||
from dbt.contracts.project import Configuration, UserConfig
|
||||
from dbt.contracts.relation import ComponentName
|
||||
from dbt.dataclass_schema import ValidationError
|
||||
from dbt.events.functions import warn_or_error
|
||||
from dbt.events.types import UnusedResourceConfigPath
|
||||
from dbt.common.dataclass_schema import ValidationError
|
||||
from dbt.common.events.functions import warn_or_error
|
||||
from dbt.common.events.types import UnusedResourceConfigPath
|
||||
from dbt.exceptions import (
|
||||
ConfigContractBrokenError,
|
||||
DbtProjectError,
|
||||
|
||||
@@ -2,7 +2,7 @@ from pathlib import Path
|
||||
from copy import deepcopy
|
||||
from typing import Dict, Any, Union
|
||||
from dbt.clients.yaml_helper import yaml, Loader, Dumper, load_yaml_text # noqa: F401
|
||||
from dbt.dataclass_schema import ValidationError
|
||||
from dbt.common.dataclass_schema import ValidationError
|
||||
|
||||
from .renderer import BaseRenderer
|
||||
|
||||
|
||||
@@ -2,8 +2,8 @@ from typing import Any, Dict
|
||||
|
||||
|
||||
from dbt.clients import yaml_helper
|
||||
from dbt.events.functions import fire_event
|
||||
from dbt.events.types import InvalidOptionYAML
|
||||
from dbt.common.events.functions import fire_event
|
||||
from dbt.common.events.types import InvalidOptionYAML
|
||||
from dbt.exceptions import DbtValidationError, OptionNotYamlDictError
|
||||
|
||||
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
# TODO: remove SECRET_ENV_PREFIX and import from dbt.common
|
||||
SECRET_ENV_PREFIX = "DBT_ENV_SECRET_"
|
||||
DEFAULT_ENV_PLACEHOLDER = "DBT_DEFAULT_PLACEHOLDER"
|
||||
METADATA_ENV_PREFIX = "DBT_ENV_CUSTOM_ENV_"
|
||||
|
||||
@@ -21,9 +21,9 @@ from dbt.exceptions import (
|
||||
SetStrictWrongTypeError,
|
||||
ZipStrictWrongTypeError,
|
||||
)
|
||||
from dbt.events.functions import fire_event, get_invocation_id
|
||||
from dbt.events.types import JinjaLogInfo, JinjaLogDebug
|
||||
from dbt.events.contextvars import get_node_info
|
||||
from dbt.common.events.functions import fire_event, get_invocation_id
|
||||
from dbt.common.events.types import JinjaLogInfo, JinjaLogDebug
|
||||
from dbt.common.events.contextvars import get_node_info
|
||||
from dbt.version import __version__ as dbt_version
|
||||
|
||||
# These modules are added to the context. Consider alternative
|
||||
|
||||
@@ -2,7 +2,7 @@ import os
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
from dbt.constants import SECRET_ENV_PREFIX, DEFAULT_ENV_PLACEHOLDER
|
||||
from dbt.contracts.connection import AdapterRequiredConfig
|
||||
from dbt.adapters.contracts.connection import AdapterRequiredConfig
|
||||
from dbt.node_types import NodeType
|
||||
from dbt.utils import MultiDict
|
||||
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
import functools
|
||||
from typing import NoReturn
|
||||
|
||||
from dbt.events.functions import warn_or_error
|
||||
from dbt.events.types import JinjaLogWarning
|
||||
from dbt.common.events.functions import warn_or_error
|
||||
from dbt.common.events.types import JinjaLogWarning
|
||||
|
||||
from dbt.exceptions import (
|
||||
DbtRuntimeError,
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
from typing import List
|
||||
|
||||
from dbt.clients.jinja import MacroStack
|
||||
from dbt.contracts.connection import AdapterRequiredConfig
|
||||
from dbt.adapters.contracts.connection import AdapterRequiredConfig
|
||||
from dbt.contracts.graph.manifest import Manifest
|
||||
from dbt.context.macro_resolver import TestMacroNamespace
|
||||
from .base import contextproperty
|
||||
|
||||
@@ -27,7 +27,7 @@ from dbt.context.exceptions_jinja import wrapped_exports
|
||||
from dbt.context.macro_resolver import MacroResolver, TestMacroNamespace
|
||||
from dbt.context.macros import MacroNamespaceBuilder, MacroNamespace
|
||||
from dbt.context.manifest import ManifestContext
|
||||
from dbt.contracts.connection import AdapterResponse
|
||||
from dbt.adapters.contracts.connection import AdapterResponse
|
||||
from dbt.contracts.graph.manifest import Manifest, Disabled
|
||||
from dbt.contracts.graph.nodes import (
|
||||
Macro,
|
||||
@@ -42,7 +42,7 @@ from dbt.contracts.graph.nodes import (
|
||||
)
|
||||
from dbt.contracts.graph.metrics import MetricReference, ResolvedMetricReference
|
||||
from dbt.contracts.graph.unparsed import NodeVersion
|
||||
from dbt.events.functions import get_metadata_vars
|
||||
from dbt.common.events.functions import get_metadata_vars
|
||||
from dbt.exceptions import (
|
||||
CompilationError,
|
||||
ConflictingConfigKeysError,
|
||||
|
||||
@@ -6,7 +6,7 @@ from mashumaro.types import SerializableType
|
||||
from typing import List, Optional, Union, Dict, Any
|
||||
|
||||
from dbt.constants import MAXIMUM_SEED_SIZE
|
||||
from dbt.dataclass_schema import dbtClassMixin, StrEnum
|
||||
from dbt.common.dataclass_schema import dbtClassMixin, StrEnum
|
||||
|
||||
from .util import SourceKey
|
||||
|
||||
|
||||
@@ -46,7 +46,7 @@ from dbt.contracts.graph.unparsed import SourcePatch, NodeVersion, UnparsedVersi
|
||||
from dbt.contracts.graph.manifest_upgrade import upgrade_manifest_json
|
||||
from dbt.contracts.files import SourceFile, SchemaSourceFile, FileHash, AnySourceFile
|
||||
from dbt.contracts.util import BaseArtifactMetadata, SourceKey, ArtifactMixin, schema_version
|
||||
from dbt.dataclass_schema import dbtClassMixin
|
||||
from dbt.common.dataclass_schema import dbtClassMixin
|
||||
from dbt.exceptions import (
|
||||
CompilationError,
|
||||
DuplicateResourceNameError,
|
||||
@@ -55,9 +55,9 @@ from dbt.exceptions import (
|
||||
AmbiguousResourceNameRefError,
|
||||
)
|
||||
from dbt.helper_types import PathSet
|
||||
from dbt.events.functions import fire_event
|
||||
from dbt.events.types import MergedFromState, UnpinnedRefNewVersionAvailable
|
||||
from dbt.events.contextvars import get_node_info
|
||||
from dbt.common.events.functions import fire_event
|
||||
from dbt.common.events.types import MergedFromState, UnpinnedRefNewVersionAvailable
|
||||
from dbt.common.events.contextvars import get_node_info
|
||||
from dbt.node_types import NodeType, AccessType
|
||||
from dbt.flags import get_flags, MP_CONTEXT
|
||||
from dbt import tracking
|
||||
|
||||
@@ -4,7 +4,7 @@ from itertools import chain
|
||||
from typing import Any, List, Optional, Dict, Union, Type, TypeVar, Callable
|
||||
from typing_extensions import Annotated
|
||||
|
||||
from dbt.dataclass_schema import (
|
||||
from dbt.common.dataclass_schema import (
|
||||
dbtClassMixin,
|
||||
ValidationError,
|
||||
StrEnum,
|
||||
|
||||
@@ -8,7 +8,7 @@ import hashlib
|
||||
from mashumaro.types import SerializableType
|
||||
from typing import Optional, Union, List, Dict, Any, Sequence, Tuple, Iterator, Literal
|
||||
|
||||
from dbt.dataclass_schema import dbtClassMixin, ExtensibleDbtClassMixin
|
||||
from dbt.common.dataclass_schema import dbtClassMixin, ExtensibleDbtClassMixin
|
||||
|
||||
from dbt.clients.system import write_file
|
||||
from dbt.contracts.files import FileHash
|
||||
@@ -37,16 +37,16 @@ from dbt.contracts.graph.unparsed import (
|
||||
)
|
||||
from dbt.contracts.graph.node_args import ModelNodeArgs
|
||||
from dbt.contracts.util import Replaceable, AdditionalPropertiesMixin
|
||||
from dbt.events.functions import warn_or_error
|
||||
from dbt.common.events.functions import warn_or_error
|
||||
from dbt.exceptions import ParsingError, ContractBreakingChangeError
|
||||
from dbt.events.types import (
|
||||
from dbt.common.events.types import (
|
||||
SeedIncreased,
|
||||
SeedExceedsLimitSamePath,
|
||||
SeedExceedsLimitAndPathChanged,
|
||||
SeedExceedsLimitChecksumChanged,
|
||||
UnversionedBreakingChange,
|
||||
)
|
||||
from dbt.events.contextvars import set_log_contextvars
|
||||
from dbt.common.events.contextvars import set_log_contextvars
|
||||
from dbt.flags import get_flags
|
||||
from dbt.node_types import ModelLanguage, NodeType, AccessType
|
||||
from dbt_semantic_interfaces.call_parameter_sets import FilterCallParameterSets
|
||||
|
||||
@@ -14,9 +14,9 @@ from dbt_semantic_interfaces.validations.semantic_manifest_validator import (
|
||||
)
|
||||
|
||||
from dbt.clients.system import write_file
|
||||
from dbt.events.base_types import EventLevel
|
||||
from dbt.events.functions import fire_event
|
||||
from dbt.events.types import SemanticValidationFailure
|
||||
from dbt.common.events.base_types import EventLevel
|
||||
from dbt.common.events.functions import fire_event
|
||||
from dbt.common.events.types import SemanticValidationFailure
|
||||
from dbt.exceptions import ParsingError
|
||||
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
from dataclasses import dataclass
|
||||
from dbt.dataclass_schema import dbtClassMixin
|
||||
from dbt.common.dataclass_schema import dbtClassMixin
|
||||
from dbt_semantic_interfaces.references import (
|
||||
DimensionReference,
|
||||
EntityReference,
|
||||
|
||||
@@ -18,7 +18,12 @@ from dbt.contracts.util import (
|
||||
import dbt.helper_types # noqa:F401
|
||||
from dbt.exceptions import CompilationError, ParsingError, DbtInternalError
|
||||
|
||||
from dbt.dataclass_schema import dbtClassMixin, StrEnum, ExtensibleDbtClassMixin, ValidationError
|
||||
from dbt.common.dataclass_schema import (
|
||||
dbtClassMixin,
|
||||
StrEnum,
|
||||
ExtensibleDbtClassMixin,
|
||||
ValidationError,
|
||||
)
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import timedelta
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
from dbt.contracts.util import Replaceable, Mergeable, list_str, Identifier
|
||||
from dbt.contracts.connection import QueryComment, UserConfigContract
|
||||
from dbt.adapters.contracts.connection import QueryComment, UserConfigContract
|
||||
from dbt.helper_types import NoValue
|
||||
from dbt.dataclass_schema import (
|
||||
from dbt.common.dataclass_schema import (
|
||||
dbtClassMixin,
|
||||
ValidationError,
|
||||
ExtensibleDbtClassMixin,
|
||||
|
||||
@@ -6,7 +6,7 @@ from typing import (
|
||||
)
|
||||
from typing_extensions import Protocol
|
||||
|
||||
from dbt.dataclass_schema import dbtClassMixin, StrEnum
|
||||
from dbt.common.dataclass_schema import dbtClassMixin, StrEnum
|
||||
|
||||
from dbt.contracts.util import Replaceable
|
||||
from dbt.exceptions import CompilationError, DataclassNotDictError
|
||||
|
||||
@@ -10,13 +10,13 @@ from dbt.contracts.util import (
|
||||
schema_version,
|
||||
)
|
||||
from dbt.exceptions import DbtInternalError
|
||||
from dbt.events.functions import fire_event
|
||||
from dbt.events.types import TimingInfoCollected
|
||||
from dbt.events.contextvars import get_node_info
|
||||
from dbt.events.helpers import datetime_to_json_string
|
||||
from dbt.common.events.functions import fire_event
|
||||
from dbt.common.events.types import TimingInfoCollected
|
||||
from dbt.common.events.contextvars import get_node_info
|
||||
from dbt.common.events.helpers import datetime_to_json_string
|
||||
from dbt.logger import TimingProcessor
|
||||
from dbt.utils import lowercase, cast_to_str, cast_to_int
|
||||
from dbt.dataclass_schema import dbtClassMixin, StrEnum
|
||||
from dbt.common.dataclass_schema import dbtClassMixin, StrEnum
|
||||
|
||||
import agate
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
from dataclasses import dataclass
|
||||
from dbt.dataclass_schema import dbtClassMixin
|
||||
from dbt.common.dataclass_schema import dbtClassMixin
|
||||
|
||||
from typing import List, Dict, Any, Union
|
||||
|
||||
|
||||
@@ -3,7 +3,7 @@ from dataclasses import dataclass, field
|
||||
from datetime import datetime
|
||||
from typing import Optional, List, Any, Dict, Sequence
|
||||
|
||||
from dbt.dataclass_schema import dbtClassMixin
|
||||
from dbt.common.dataclass_schema import dbtClassMixin
|
||||
|
||||
from dbt.contracts.graph.nodes import ResultNode
|
||||
from dbt.contracts.results import (
|
||||
|
||||
@@ -4,8 +4,8 @@ from typing import Optional
|
||||
from dbt.contracts.graph.manifest import WritableManifest
|
||||
from dbt.contracts.results import FreshnessExecutionResultArtifact
|
||||
from dbt.contracts.results import RunResultsArtifact
|
||||
from dbt.events.functions import fire_event
|
||||
from dbt.events.types import WarnStateTargetEqual
|
||||
from dbt.common.events.functions import fire_event
|
||||
from dbt.common.events.types import WarnStateTargetEqual
|
||||
from dbt.exceptions import IncompatibleSchemaError
|
||||
|
||||
|
||||
|
||||
@@ -10,10 +10,10 @@ from dbt.exceptions import (
|
||||
)
|
||||
from dbt.version import __version__
|
||||
|
||||
from dbt.events.functions import get_invocation_id, get_metadata_vars
|
||||
from dbt.dataclass_schema import dbtClassMixin
|
||||
from dbt.common.events.functions import get_invocation_id, get_metadata_vars
|
||||
from dbt.common.dataclass_schema import dbtClassMixin
|
||||
|
||||
from dbt.dataclass_schema import (
|
||||
from dbt.common.dataclass_schema import (
|
||||
ValidatedStringMixin,
|
||||
ValidationError,
|
||||
)
|
||||
|
||||
@@ -23,7 +23,7 @@ class DBTDeprecation:
|
||||
@property
|
||||
def event(self) -> abc.ABCMeta:
|
||||
if self._event is not None:
|
||||
module_path = dbt.events.types
|
||||
module_path = dbt.common.events.types
|
||||
class_name = self._event
|
||||
|
||||
try:
|
||||
@@ -36,7 +36,7 @@ class DBTDeprecation:
|
||||
def show(self, *args, **kwargs) -> None:
|
||||
if self.name not in active_deprecations:
|
||||
event = self.event(**kwargs)
|
||||
dbt.events.functions.warn_or_error(event)
|
||||
dbt.common.events.functions.warn_or_error(event)
|
||||
self.track_deprecation_warn()
|
||||
active_deprecations.add(self.name)
|
||||
|
||||
|
||||
@@ -8,8 +8,8 @@ from typing import List, Optional, Generic, TypeVar
|
||||
|
||||
from dbt.clients import system
|
||||
from dbt.contracts.project import ProjectPackageMetadata
|
||||
from dbt.events.functions import fire_event
|
||||
from dbt.events.types import DepsSetDownloadDirectory
|
||||
from dbt.common.events.functions import fire_event
|
||||
from dbt.common.events.types import DepsSetDownloadDirectory
|
||||
from dbt.utils import _connection_exception_retry as connection_exception_retry
|
||||
|
||||
DOWNLOADS_PATH = None
|
||||
|
||||
@@ -10,8 +10,8 @@ from dbt.contracts.project import (
|
||||
)
|
||||
from dbt.deps.base import PinnedPackage, UnpinnedPackage, get_downloads_path
|
||||
from dbt.exceptions import ExecutableError, MultipleVersionGitDepsError
|
||||
from dbt.events.functions import fire_event, warn_or_error
|
||||
from dbt.events.types import EnsureGitInstalled, DepsUnpinned
|
||||
from dbt.common.events.functions import fire_event, warn_or_error
|
||||
from dbt.common.events.types import EnsureGitInstalled, DepsUnpinned
|
||||
from dbt.utils import md5
|
||||
|
||||
|
||||
|
||||
@@ -6,8 +6,8 @@ from dbt.contracts.project import (
|
||||
ProjectPackageMetadata,
|
||||
LocalPackage,
|
||||
)
|
||||
from dbt.events.functions import fire_event
|
||||
from dbt.events.types import DepsCreatingLocalSymlink, DepsSymlinkNotAvailable
|
||||
from dbt.common.events.functions import fire_event
|
||||
from dbt.common.events.types import DepsCreatingLocalSymlink, DepsSymlinkNotAvailable
|
||||
from dbt.config.project import PartialProject, Project
|
||||
from dbt.config.renderer import PackageRenderer
|
||||
|
||||
|
||||
@@ -1,30 +1,16 @@
|
||||
import builtins
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import io
|
||||
import agate
|
||||
from typing import Any, Dict, List, Mapping, Optional, Union
|
||||
|
||||
from dbt.constants import SECRET_ENV_PREFIX
|
||||
from dbt.dataclass_schema import ValidationError
|
||||
from dbt.node_types import NodeType, AccessType
|
||||
from dbt.ui import line_wrap_message
|
||||
|
||||
import dbt.dataclass_schema
|
||||
|
||||
|
||||
def env_secrets() -> List[str]:
|
||||
return [v for k, v in os.environ.items() if k.startswith(SECRET_ENV_PREFIX) and v.strip()]
|
||||
|
||||
|
||||
def scrub_secrets(msg: str, secrets: List[str]) -> str:
|
||||
scrubbed = str(msg)
|
||||
|
||||
for secret in secrets:
|
||||
scrubbed = scrubbed.replace(secret, "*****")
|
||||
|
||||
return scrubbed
|
||||
import dbt.common.dataclass_schema
|
||||
from dbt.common.dataclass_schema import ValidationError
|
||||
from dbt.common.exceptions import env_secrets, scrub_secrets
|
||||
|
||||
|
||||
class MacroReturn(builtins.BaseException):
|
||||
@@ -147,7 +133,7 @@ class DbtRuntimeError(RuntimeError, Exception):
|
||||
"""Given a dbt.dataclass_schema.ValidationError (which is basically a
|
||||
jsonschema.ValidationError), return the relevant parts as a string
|
||||
"""
|
||||
if not isinstance(exc, dbt.dataclass_schema.ValidationError):
|
||||
if not isinstance(exc, dbt.common.dataclass_schema.ValidationError):
|
||||
return str(exc)
|
||||
path = "[%s]" % "][".join(map(repr, exc.relative_path))
|
||||
return f"at path {path}: {exc.message}"
|
||||
|
||||
@@ -24,7 +24,7 @@ ENABLE_LEGACY_LOGGER = env_set_truthy("DBT_ENABLE_LEGACY_LOGGER")
|
||||
MP_CONTEXT = get_context()
|
||||
|
||||
|
||||
# this roughly follows the patten of EVENT_MANAGER in dbt/events/functions.py
|
||||
# this roughly follows the patten of EVENT_MANAGER in dbt/common/events/functions.py
|
||||
# During de-globlization, we'll need to handle both similarly
|
||||
# Match USE_COLORS default with default in dbt.cli.params.use_colors for use in --version
|
||||
GLOBAL_FLAGS = Namespace(USE_COLORS=True) # type: ignore
|
||||
|
||||
@@ -5,8 +5,8 @@ from .queue import GraphQueue
|
||||
from .selector_methods import MethodManager
|
||||
from .selector_spec import SelectionCriteria, SelectionSpec, IndirectSelection
|
||||
|
||||
from dbt.events.functions import fire_event, warn_or_error
|
||||
from dbt.events.types import SelectorReportInvalidSelector, NoNodesForSelectionCriteria
|
||||
from dbt.common.events.functions import fire_event, warn_or_error
|
||||
from dbt.common.events.types import SelectorReportInvalidSelector, NoNodesForSelectionCriteria
|
||||
from dbt.node_types import NodeType
|
||||
from dbt.exceptions import (
|
||||
DbtInternalError,
|
||||
|
||||
@@ -4,7 +4,7 @@ from itertools import chain
|
||||
from pathlib import Path
|
||||
from typing import Set, List, Dict, Iterator, Tuple, Any, Union, Type, Optional, Callable
|
||||
|
||||
from dbt.dataclass_schema import StrEnum
|
||||
from dbt.common.dataclass_schema import StrEnum
|
||||
|
||||
from .graph import UniqueId
|
||||
|
||||
@@ -28,7 +28,7 @@ from dbt.exceptions import (
|
||||
DbtRuntimeError,
|
||||
)
|
||||
from dbt.node_types import NodeType
|
||||
from dbt.events.contextvars import get_project_root
|
||||
from dbt.common.events.contextvars import get_project_root
|
||||
|
||||
|
||||
SELECTOR_GLOB = "*"
|
||||
|
||||
@@ -2,7 +2,7 @@ import os
|
||||
import re
|
||||
from abc import ABCMeta, abstractmethod
|
||||
from dataclasses import dataclass
|
||||
from dbt.dataclass_schema import StrEnum, dbtClassMixin
|
||||
from dbt.common.dataclass_schema import StrEnum, dbtClassMixin
|
||||
|
||||
from typing import Set, Iterator, List, Optional, Dict, Union, Any, Iterable, Tuple
|
||||
from .graph import UniqueId
|
||||
|
||||
@@ -7,12 +7,12 @@ from dataclasses import dataclass, field
|
||||
from typing import Tuple, AbstractSet, Union
|
||||
from typing import Callable, cast, Generic, Optional, TypeVar, List, NewType
|
||||
|
||||
from dbt.dataclass_schema import (
|
||||
from dbt.common.dataclass_schema import (
|
||||
dbtClassMixin,
|
||||
ValidationError,
|
||||
StrEnum,
|
||||
)
|
||||
import dbt.events.types as dbt_event_types
|
||||
import dbt.common.events.types as dbt_event_types
|
||||
|
||||
|
||||
Port = NewType("Port", int)
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
from dbt.dataclass_schema import StrEnum
|
||||
from dbt.common.dataclass_schema import StrEnum
|
||||
import json
|
||||
|
||||
from typing import Union, Dict, Any
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
import functools
|
||||
from typing import Optional
|
||||
|
||||
from dbt.events.functions import warn_or_error
|
||||
from dbt.events.types import InternalDeprecation
|
||||
from dbt.common.events.functions import warn_or_error
|
||||
from dbt.common.events.types import InternalDeprecation
|
||||
|
||||
|
||||
def deprecated(suggested_action: str, version: str, reason: Optional[str]):
|
||||
|
||||
@@ -14,7 +14,7 @@ from typing import Optional, List, ContextManager, Callable, Dict, Any, Set
|
||||
import colorama
|
||||
import logbook
|
||||
from dbt.constants import SECRET_ENV_PREFIX
|
||||
from dbt.dataclass_schema import dbtClassMixin
|
||||
from dbt.common.dataclass_schema import dbtClassMixin
|
||||
|
||||
# Colorama is needed for colored logs on Windows because we're using logger.info
|
||||
# intead of print(). If the Windows env doesn't have a TERM var set or it is set to None
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
from typing import List
|
||||
|
||||
from dbt.dataclass_schema import StrEnum
|
||||
from dbt.common.dataclass_schema import StrEnum
|
||||
|
||||
|
||||
class AccessType(StrEnum):
|
||||
|
||||
@@ -3,7 +3,7 @@ import itertools
|
||||
import os
|
||||
from typing import List, Dict, Any, Generic, Optional, TypeVar
|
||||
|
||||
from dbt.dataclass_schema import ValidationError
|
||||
from dbt.common.dataclass_schema import ValidationError
|
||||
|
||||
from dbt import utils
|
||||
from dbt.clients.jinja import MacroGenerator
|
||||
|
||||
@@ -20,7 +20,7 @@ from itertools import chain
|
||||
import time
|
||||
|
||||
from dbt.contracts.graph.semantic_manifest import SemanticManifest
|
||||
from dbt.events.base_types import EventLevel
|
||||
from dbt.common.events.base_types import EventLevel
|
||||
import json
|
||||
import pprint
|
||||
import msgpack
|
||||
@@ -41,8 +41,8 @@ from dbt.constants import (
|
||||
SEMANTIC_MANIFEST_FILE_NAME,
|
||||
)
|
||||
from dbt.helper_types import PathSet
|
||||
from dbt.events.functions import fire_event, get_invocation_id, warn_or_error
|
||||
from dbt.events.types import (
|
||||
from dbt.common.events.functions import fire_event, get_invocation_id, warn_or_error
|
||||
from dbt.common.events.types import (
|
||||
PartialParsingErrorProcessingFile,
|
||||
PartialParsingError,
|
||||
ParsePerfInfoPath,
|
||||
@@ -124,7 +124,7 @@ from dbt.parser.snapshots import SnapshotParser
|
||||
from dbt.parser.sources import SourcePatcher
|
||||
from dbt.version import __version__
|
||||
|
||||
from dbt.dataclass_schema import StrEnum, dbtClassMixin
|
||||
from dbt.common.dataclass_schema import StrEnum, dbtClassMixin
|
||||
from dbt import plugins
|
||||
|
||||
from dbt_semantic_interfaces.enum_extension import assert_values_exhausted
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
from copy import deepcopy
|
||||
from dbt.context.context_config import ContextConfig
|
||||
from dbt.contracts.graph.nodes import ModelNode, RefArgs
|
||||
from dbt.events.base_types import EventLevel
|
||||
from dbt.events.types import Note
|
||||
from dbt.events.functions import fire_event_if_test
|
||||
from dbt.common.events.base_types import EventLevel
|
||||
from dbt.common.events.types import Note
|
||||
from dbt.common.events.functions import fire_event_if_test
|
||||
from dbt.flags import get_flags
|
||||
from dbt.node_types import NodeType, ModelLanguage
|
||||
from dbt.parser.base import SimpleSQLParser
|
||||
@@ -19,7 +19,7 @@ from typing import Any, Dict, Iterator, List, Optional, Tuple, Union
|
||||
|
||||
# New for Python models :p
|
||||
import ast
|
||||
from dbt.dataclass_schema import ValidationError
|
||||
from dbt.common.dataclass_schema import ValidationError
|
||||
from dbt.exceptions import (
|
||||
ModelConfigError,
|
||||
ParsingError,
|
||||
|
||||
@@ -8,9 +8,9 @@ from dbt.contracts.files import (
|
||||
parse_file_type_to_parser,
|
||||
SchemaSourceFile,
|
||||
)
|
||||
from dbt.events.functions import fire_event
|
||||
from dbt.events.base_types import EventLevel
|
||||
from dbt.events.types import (
|
||||
from dbt.common.events.functions import fire_event
|
||||
from dbt.common.events.base_types import EventLevel
|
||||
from dbt.common.events.types import (
|
||||
PartialParsingEnabled,
|
||||
PartialParsingFile,
|
||||
)
|
||||
|
||||
@@ -12,13 +12,13 @@ from dbt.contracts.files import (
|
||||
SchemaSourceFile,
|
||||
)
|
||||
from dbt.config import Project
|
||||
from dbt.dataclass_schema import dbtClassMixin
|
||||
from dbt.common.dataclass_schema import dbtClassMixin
|
||||
from dbt.parser.schemas import yaml_from_file, schema_file_keys
|
||||
from dbt.exceptions import ParsingError
|
||||
from dbt.parser.search import filesystem_search
|
||||
from typing import Optional, Dict, List, Mapping, MutableMapping
|
||||
from dbt.events.types import InputFileDiffError
|
||||
from dbt.events.functions import fire_event
|
||||
from dbt.common.events.types import InputFileDiffError
|
||||
from dbt.common.events.functions import fire_event
|
||||
from typing import Protocol
|
||||
|
||||
|
||||
|
||||
@@ -24,7 +24,7 @@ from dbt.contracts.graph.nodes import (
|
||||
)
|
||||
from dbt.context.context_config import ContextConfig
|
||||
from dbt.context.configured import generate_schema_yml_context, SchemaYamlVars
|
||||
from dbt.dataclass_schema import ValidationError
|
||||
from dbt.common.dataclass_schema import ValidationError
|
||||
from dbt.exceptions import SchemaConfigError, CompilationError, ParsingError, TestConfigError
|
||||
from dbt.contracts.files import FileHash
|
||||
from dbt.utils import md5, get_pseudo_test_path
|
||||
|
||||
@@ -46,7 +46,7 @@ from dbt.context.context_config import (
|
||||
UnrenderedConfigGenerator,
|
||||
)
|
||||
from dbt.clients.jinja import get_rendered
|
||||
from dbt.dataclass_schema import ValidationError
|
||||
from dbt.common.dataclass_schema import ValidationError
|
||||
from dbt_semantic_interfaces.type_enums import (
|
||||
AggregationType,
|
||||
DimensionType,
|
||||
|
||||
@@ -5,7 +5,7 @@ from abc import ABCMeta, abstractmethod
|
||||
from typing import Any, Callable, Dict, Generic, Iterable, List, Optional, Type, TypeVar
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from dbt.dataclass_schema import ValidationError, dbtClassMixin
|
||||
from dbt.common.dataclass_schema import ValidationError, dbtClassMixin
|
||||
|
||||
from dbt.clients.yaml_helper import load_yaml_text
|
||||
from dbt.parser.schema_renderer import SchemaYamlRenderer
|
||||
@@ -44,8 +44,8 @@ from dbt.exceptions import (
|
||||
YamlParseListError,
|
||||
InvalidAccessTypeError,
|
||||
)
|
||||
from dbt.events.functions import warn_or_error
|
||||
from dbt.events.types import (
|
||||
from dbt.common.events.functions import warn_or_error
|
||||
from dbt.common.events.types import (
|
||||
MacroNotFoundForPatch,
|
||||
NoNodeForYamlKey,
|
||||
ValidationWarning,
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import os
|
||||
from typing import List
|
||||
|
||||
from dbt.dataclass_schema import ValidationError
|
||||
from dbt.common.dataclass_schema import ValidationError
|
||||
|
||||
from dbt.contracts.graph.nodes import IntermediateSnapshotNode, SnapshotNode
|
||||
from dbt.exceptions import SnapshopConfigError
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
import itertools
|
||||
from pathlib import Path
|
||||
from typing import Iterable, Dict, Optional, Set, Any, List
|
||||
|
||||
from dbt.adapters.capability import Capability
|
||||
from dbt.adapters.factory import get_adapter
|
||||
from dbt.config import RuntimeConfig
|
||||
from dbt.context.context_config import (
|
||||
@@ -26,9 +24,8 @@ from dbt.contracts.graph.unparsed import (
|
||||
UnparsedColumn,
|
||||
Time,
|
||||
)
|
||||
|
||||
from dbt.events.functions import warn_or_error, fire_event
|
||||
from dbt.events.types import UnusedTables, FreshnessConfigProblem
|
||||
from dbt.common.events.functions import warn_or_error
|
||||
from dbt.common.events.types import UnusedTables
|
||||
from dbt.exceptions import DbtInternalError
|
||||
from dbt.node_types import NodeType
|
||||
|
||||
@@ -187,21 +184,6 @@ class SourcePatcher:
|
||||
unrendered_config=unrendered_config,
|
||||
)
|
||||
|
||||
if (
|
||||
parsed_source.freshness
|
||||
and not parsed_source.loaded_at_field
|
||||
and not get_adapter(self.root_project).supports(Capability.TableLastModifiedMetadata)
|
||||
):
|
||||
# Metadata-based freshness is being used by default for this node,
|
||||
# but is not available through the configured adapter, so warn the
|
||||
# user that freshness info will not be collected for this node at
|
||||
# runtime.
|
||||
fire_event(
|
||||
FreshnessConfigProblem(
|
||||
msg=f"The configured adapter does not support metadata-based freshness. A loaded_at_field must be specified for source '{source.name}'."
|
||||
)
|
||||
)
|
||||
|
||||
# relation name is added after instantiation because the adapter does
|
||||
# not provide the relation name for a UnpatchedSourceDefinition object
|
||||
parsed_source.relation_name = self._get_relation_name(parsed_source)
|
||||
|
||||
@@ -3,7 +3,7 @@ from typing import Dict
|
||||
# just exports, they need "noqa" so flake8 will not complain.
|
||||
from dbt.contracts.util import ArtifactMixin as PluginArtifact, schema_version # noqa
|
||||
from dbt.contracts.util import BaseArtifactMetadata, AdditionalPropertiesMixin # noqa
|
||||
from dbt.dataclass_schema import dbtClassMixin, ExtensibleDbtClassMixin # noqa
|
||||
from dbt.common.dataclass_schema import dbtClassMixin, ExtensibleDbtClassMixin # noqa
|
||||
|
||||
|
||||
PluginArtifacts = Dict[str, PluginArtifact]
|
||||
|
||||
@@ -5,7 +5,7 @@ from typing import List
|
||||
from dbt.exceptions import VersionsNotCompatibleError
|
||||
import dbt.utils
|
||||
|
||||
from dbt.dataclass_schema import dbtClassMixin, StrEnum
|
||||
from dbt.common.dataclass_schema import dbtClassMixin, StrEnum
|
||||
from typing import Optional
|
||||
|
||||
|
||||
|
||||
@@ -5,7 +5,6 @@ import traceback
|
||||
from abc import ABCMeta, abstractmethod
|
||||
from contextlib import nullcontext
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Type, Union
|
||||
|
||||
import dbt.exceptions
|
||||
@@ -13,7 +12,6 @@ from dbt import tracking
|
||||
from dbt.adapters.factory import get_adapter
|
||||
from dbt.config import RuntimeConfig, Project
|
||||
from dbt.config.profile import read_profile
|
||||
from dbt.constants import DBT_PROJECT_FILE_NAME
|
||||
from dbt.contracts.graph.manifest import Manifest
|
||||
from dbt.contracts.results import (
|
||||
NodeStatus,
|
||||
@@ -23,9 +21,9 @@ from dbt.contracts.results import (
|
||||
RunningStatus,
|
||||
TimingInfo,
|
||||
)
|
||||
from dbt.events.contextvars import get_node_info
|
||||
from dbt.events.functions import fire_event
|
||||
from dbt.events.types import (
|
||||
from dbt.common.events.contextvars import get_node_info
|
||||
from dbt.common.events.functions import fire_event
|
||||
from dbt.common.events.types import (
|
||||
LogDbtProjectError,
|
||||
LogDbtProfileError,
|
||||
CatchableExceptionOnRun,
|
||||
@@ -47,7 +45,7 @@ from dbt.exceptions import (
|
||||
from dbt.flags import get_flags
|
||||
from dbt.graph import Graph
|
||||
from dbt.logger import log_manager
|
||||
from dbt.task.printer import print_run_result_error
|
||||
from .printer import print_run_result_error
|
||||
|
||||
|
||||
class NoneConfig:
|
||||
@@ -119,32 +117,35 @@ class BaseTask(metaclass=ABCMeta):
|
||||
return True
|
||||
|
||||
|
||||
def get_nearest_project_dir(project_dir: Optional[str]) -> Path:
|
||||
def get_nearest_project_dir(project_dir: Optional[str]) -> str:
|
||||
# If the user provides an explicit project directory, use that
|
||||
# but don't look at parent directories.
|
||||
if project_dir:
|
||||
cur_dir = Path(project_dir)
|
||||
project_file = Path(project_dir) / DBT_PROJECT_FILE_NAME
|
||||
if project_file.is_file():
|
||||
return cur_dir
|
||||
project_file = os.path.join(project_dir, "dbt_project.yml")
|
||||
if os.path.exists(project_file):
|
||||
return project_dir
|
||||
else:
|
||||
raise dbt.exceptions.DbtRuntimeError(
|
||||
"fatal: Invalid --project-dir flag. Not a dbt project. "
|
||||
"Missing dbt_project.yml file"
|
||||
)
|
||||
|
||||
cur_dir = Path.cwd()
|
||||
project_file = cur_dir / DBT_PROJECT_FILE_NAME
|
||||
if project_file.is_file():
|
||||
return cur_dir
|
||||
else:
|
||||
raise dbt.exceptions.DbtRuntimeError(
|
||||
"fatal: Not a dbt project (or any of the parent directories). "
|
||||
"Missing dbt_project.yml file"
|
||||
)
|
||||
root_path = os.path.abspath(os.sep)
|
||||
cwd = os.getcwd()
|
||||
|
||||
while cwd != root_path:
|
||||
project_file = os.path.join(cwd, "dbt_project.yml")
|
||||
if os.path.exists(project_file):
|
||||
return cwd
|
||||
cwd = os.path.dirname(cwd)
|
||||
|
||||
raise dbt.exceptions.DbtRuntimeError(
|
||||
"fatal: Not a dbt project (or any of the parent directories). "
|
||||
"Missing dbt_project.yml file"
|
||||
)
|
||||
|
||||
|
||||
def move_to_nearest_project_dir(project_dir: Optional[str]) -> Path:
|
||||
def move_to_nearest_project_dir(project_dir: Optional[str]) -> str:
|
||||
nearest_project_dir = get_nearest_project_dir(project_dir)
|
||||
os.chdir(nearest_project_dir)
|
||||
return nearest_project_dir
|
||||
|
||||
@@ -2,8 +2,8 @@ from pathlib import Path
|
||||
from shutil import rmtree
|
||||
|
||||
from dbt import deprecations
|
||||
from dbt.events.functions import fire_event
|
||||
from dbt.events.types import (
|
||||
from dbt.common.events.functions import fire_event
|
||||
from dbt.common.events.types import (
|
||||
CheckCleanPath,
|
||||
ConfirmCleanPath,
|
||||
FinishedCleanPaths,
|
||||
|
||||
@@ -5,7 +5,7 @@ from dbt.adapters.base import BaseRelation
|
||||
from dbt.clients.jinja import MacroGenerator
|
||||
from dbt.context.providers import generate_runtime_model_context
|
||||
from dbt.contracts.results import RunStatus, RunResult
|
||||
from dbt.dataclass_schema import dbtClassMixin
|
||||
from dbt.common.dataclass_schema import dbtClassMixin
|
||||
from dbt.exceptions import DbtInternalError, CompilationError
|
||||
from dbt.graph import ResourceTypeSelector
|
||||
from dbt.node_types import NodeType
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user