Compare commits

...

3 Commits

Author SHA1 Message Date
gwen windflower
a4786d1b33 WIP . selector working 2023-02-10 11:08:36 -06:00
gwen windflower
77eaf2a3d3 WIP ensuring metadata is there 2023-02-10 08:20:47 -06:00
gwen windflower
be68862bb4 WIP 2023-02-09 11:29:50 -06:00
7 changed files with 149 additions and 136 deletions

1
.gitignore vendored
View File

@@ -9,6 +9,7 @@ __pycache__/
# Distribution / packaging
.Python
env*/
.venv
dbt_env/
build/
!core/dbt/docs/build

View File

@@ -24,19 +24,19 @@ from dbt.contracts.graph.manifest import ManifestMetadata
from dbt.contracts.project import Configuration, UserConfig
from dbt.contracts.relation import ComponentName
from dbt.dataclass_schema import ValidationError
from dbt.events.functions import warn_or_error
from dbt.events.types import UnusedResourceConfigPath
from dbt.exceptions import (
ConfigContractBrokenError,
DbtProjectError,
NonUniquePackageNameError,
DbtRuntimeError,
NonUniquePackageNameError,
UninstalledPackagesFoundError,
)
from dbt.events.functions import warn_or_error
from dbt.events.types import UnusedResourceConfigPath
from dbt.helper_types import DictDefaultEmptyStr, FQNPath, PathSet
from .profile import Profile
from .project import Project, PartialProject
from .project import PartialProject, Project
from .renderer import DbtProjectYamlRenderer, ProfileRenderer
from .utils import parse_cli_vars
@@ -269,7 +269,11 @@ class RuntimeConfig(Project, Profile, AdapterRequiredConfig):
)
def get_metadata(self) -> ManifestMetadata:
return ManifestMetadata(project_id=self.hashed_name(), adapter_type=self.credentials.type)
return ManifestMetadata(
project_id=self.hashed_name(),
project_name=self.project_name,
adapter_type=self.credentials.type,
)
def _get_v2_config_paths(
self,

View File

@@ -1,57 +1,61 @@
import enum
from dataclasses import dataclass, field
from itertools import chain, islice
from mashumaro.mixins.msgpack import DataClassMessagePackMixin
from multiprocessing.synchronize import Lock
from typing import (
AbstractSet,
Any,
Callable,
ClassVar,
Dict,
Generic,
List,
Optional,
Union,
Mapping,
MutableMapping,
Any,
Optional,
Set,
Tuple,
TypeVar,
Callable,
Generic,
AbstractSet,
ClassVar,
Union,
)
from typing_extensions import Protocol
from uuid import UUID
import dbt.utils
from dbt import flags, tracking
from dbt.contracts.files import AnySourceFile, FileHash, SchemaSourceFile, SourceFile
from dbt.contracts.graph.nodes import (
Macro,
Documentation,
SourceDefinition,
GenericTestNode,
Exposure,
Metric,
UnpatchedSourceDefinition,
ManifestNode,
GraphMemberNode,
ResultNode,
BaseNode,
Documentation,
Exposure,
GenericTestNode,
GraphMemberNode,
Macro,
ManifestNode,
Metric,
ResultNode,
SourceDefinition,
UnpatchedSourceDefinition,
)
from dbt.contracts.graph.unparsed import SourcePatch
from dbt.contracts.files import SourceFile, SchemaSourceFile, FileHash, AnySourceFile
from dbt.contracts.util import BaseArtifactMetadata, SourceKey, ArtifactMixin, schema_version
from dbt.dataclass_schema import dbtClassMixin
from dbt.exceptions import (
CompilationError,
DuplicateResourceNameError,
DuplicateMacroInPackageError,
DuplicateMaterializationNameError,
from dbt.contracts.util import (
ArtifactMixin,
BaseArtifactMetadata,
SourceKey,
schema_version,
)
from dbt.helper_types import PathSet
from dbt.dataclass_schema import dbtClassMixin
from dbt.events.functions import fire_event
from dbt.events.types import MergedFromState
from dbt.exceptions import (
CompilationError,
DuplicateMacroInPackageError,
DuplicateMaterializationNameError,
DuplicateResourceNameError,
)
from dbt.helper_types import PathSet
from dbt.node_types import NodeType
from dbt import flags
from dbt import tracking
import dbt.utils
from mashumaro.mixins.msgpack import DataClassMessagePackMixin
from typing_extensions import Protocol
NodeEdgeMap = Dict[str, List[str]]
PackageName = str
@@ -278,6 +282,10 @@ class ManifestMetadata(BaseArtifactMetadata):
"description": "A unique identifier for the project",
},
)
project_name: Optional[str] = field(
default=None,
metadata={"description": "The human readable text name of the project"},
)
user_id: Optional[UUID] = field(
default=None,
metadata={

View File

@@ -1,29 +1,35 @@
import abc
from itertools import chain
from pathlib import Path
from typing import Set, List, Dict, Iterator, Tuple, Any, Union, Type, Optional, Callable
from dbt.dataclass_schema import StrEnum
from .graph import UniqueId
from typing import (
Any,
Callable,
Dict,
Iterator,
List,
Optional,
Set,
Tuple,
Type,
Union,
)
from dbt.contracts.graph.manifest import Manifest, WritableManifest
from dbt.contracts.graph.nodes import (
SingularTestNode,
Exposure,
Metric,
GenericTestNode,
SourceDefinition,
ResultNode,
ManifestNode,
Metric,
ResultNode,
SingularTestNode,
SourceDefinition,
)
from dbt.contracts.state import PreviousState
from dbt.exceptions import (
DbtInternalError,
DbtRuntimeError,
)
from dbt.dataclass_schema import StrEnum
from dbt.exceptions import DbtInternalError, DbtRuntimeError
from dbt.node_types import NodeType
from .graph import UniqueId
SELECTOR_GLOB = "*"
SELECTOR_DELIMITER = ":"
@@ -296,7 +302,10 @@ class PackageSelectorMethod(SelectorMethod):
def search(self, included_nodes: Set[UniqueId], selector: str) -> Iterator[UniqueId]:
"""Yields nodes from included that have the specified package"""
for node, real_node in self.all_nodes(included_nodes):
if real_node.package_name == selector:
if selector == ".":
if real_node.package_name == self.manifest.metadata.project_name:
yield node
elif real_node.package_name == selector:
yield node

View File

@@ -1,87 +1,84 @@
from copy import deepcopy
from dataclasses import dataclass
from dataclasses import field
from datetime import datetime
import os
import traceback
from typing import Dict, Optional, Mapping, Callable, Any, List, Type, Union, Tuple
from itertools import chain
import time
from dbt.events.base_types import EventLevel
import pprint
import time
import traceback
from copy import deepcopy
from dataclasses import dataclass, field
from datetime import datetime
from itertools import chain
from typing import Any, Callable, Dict, List, Mapping, Optional, Tuple, Type, Union
import dbt.exceptions
import dbt.tracking
import dbt.flags as flags
import dbt.tracking
from dbt.adapters.factory import (
get_adapter,
get_relation_class_by_name,
get_adapter_package_names,
get_relation_class_by_name,
)
from dbt.helper_types import PathSet
from dbt.events.functions import fire_event, get_invocation_id, warn_or_error
from dbt.events.types import (
PartialParsingErrorProcessingFile,
PartialParsingError,
PartialParsingSkipParsing,
UnableToPartialParse,
PartialParsingNotEnabled,
ParsedFileLoadFailed,
InvalidDisabledTargetInTestNode,
NodeNotFoundOrDisabled,
StateCheckVarsHash,
Note,
)
from dbt.logger import DbtProcessState
from dbt.node_types import NodeType
from dbt.clients.jinja import get_rendered, MacroStack
from dbt.clients.jinja import MacroStack, get_rendered
from dbt.clients.jinja_static import statically_extract_macro_calls
from dbt.clients.system import make_directory
from dbt.config import Project, RuntimeConfig
from dbt.context.configured import generate_macro_context
from dbt.context.docs import generate_runtime_docs_context
from dbt.context.macro_resolver import MacroResolver, TestMacroNamespace
from dbt.context.configured import generate_macro_context
from dbt.context.providers import ParseProvider
from dbt.contracts.files import FileHash, ParseFileType, SchemaSourceFile
from dbt.parser.read_files import read_files, load_source_file
from dbt.parser.partial import PartialParsing, special_override_macros
from dbt.contracts.graph.manifest import (
Manifest,
Disabled,
MacroManifest,
Manifest,
ManifestStateCheck,
ParsingInfo,
)
from dbt.contracts.graph.nodes import (
SourceDefinition,
Macro,
ColumnInfo,
Exposure,
Metric,
SeedNode,
Macro,
ManifestNode,
Metric,
ResultNode,
SeedNode,
SourceDefinition,
)
from dbt.contracts.util import Writable
from dbt.exceptions import TargetNotFoundError, AmbiguousAliasError
from dbt.parser.base import Parser
from dbt.dataclass_schema import StrEnum, dbtClassMixin
from dbt.events.base_types import EventLevel
from dbt.events.functions import fire_event, get_invocation_id, warn_or_error
from dbt.events.types import (
InvalidDisabledTargetInTestNode,
NodeNotFoundOrDisabled,
Note,
ParsedFileLoadFailed,
PartialParsingError,
PartialParsingErrorProcessingFile,
PartialParsingNotEnabled,
PartialParsingSkipParsing,
StateCheckVarsHash,
UnableToPartialParse,
)
from dbt.exceptions import AmbiguousAliasError, TargetNotFoundError
from dbt.helper_types import PathSet
from dbt.logger import DbtProcessState
from dbt.node_types import NodeType
from dbt.parser.analysis import AnalysisParser
from dbt.parser.generic_test import GenericTestParser
from dbt.parser.singular_test import SingularTestParser
from dbt.parser.base import Parser
from dbt.parser.docs import DocumentationParser
from dbt.parser.generic_test import GenericTestParser
from dbt.parser.hooks import HookParser
from dbt.parser.macros import MacroParser
from dbt.parser.models import ModelParser
from dbt.parser.partial import PartialParsing, special_override_macros
from dbt.parser.read_files import load_source_file, read_files
from dbt.parser.schemas import SchemaParser
from dbt.parser.search import FileBlock
from dbt.parser.seeds import SeedParser
from dbt.parser.singular_test import SingularTestParser
from dbt.parser.snapshots import SnapshotParser
from dbt.parser.sources import SourcePatcher
from dbt.version import __version__
from dbt.dataclass_schema import StrEnum, dbtClassMixin
PARTIAL_PARSE_FILE_NAME = "partial_parse.msgpack"
PARSING_STATE = DbtProcessState("parsing")

View File

@@ -1,25 +1,21 @@
from copy import deepcopy
from dbt.context.context_config import ContextConfig
from dbt.contracts.graph.nodes import ModelNode
from dbt.events.base_types import EventLevel
from dbt.events.types import Note
from dbt.events.functions import fire_event
import dbt.flags as flags
from dbt.node_types import NodeType, ModelLanguage
from dbt.parser.base import SimpleSQLParser
from dbt.parser.search import FileBlock
from dbt.clients.jinja import get_rendered
import dbt.tracking as tracking
from dbt import utils
from dbt_extractor import ExtractionError, py_extract_from_source # type: ignore
from functools import reduce
from itertools import chain
import random
from typing import Any, Dict, Iterator, List, Optional, Tuple, Union
# New for Python models :p
import ast
import random
from copy import deepcopy
from functools import reduce
from itertools import chain
from typing import Any, Dict, Iterator, List, Optional, Tuple, Union
import dbt.flags as flags
import dbt.tracking as tracking
from dbt import utils
from dbt.clients.jinja import get_rendered
from dbt.context.context_config import ContextConfig
from dbt.contracts.graph.nodes import ModelNode
from dbt.dataclass_schema import ValidationError
from dbt.events.base_types import EventLevel
from dbt.events.functions import fire_event
from dbt.events.types import Note
from dbt.exceptions import (
ModelConfigError,
ParsingError,
@@ -27,6 +23,10 @@ from dbt.exceptions import (
PythonParsingError,
UndefinedMacroError,
)
from dbt.node_types import ModelLanguage, NodeType
from dbt.parser.base import SimpleSQLParser
from dbt.parser.search import FileBlock
from dbt_extractor import ExtractionError, py_extract_from_source # type: ignore
dbt_function_key_words = set(["ref", "source", "config", "get"])
dbt_function_full_names = set(["dbt.ref", "dbt.source", "dbt.config", "dbt.config.get"])

View File

@@ -1,33 +1,27 @@
from typing import Optional
import os
import platform
import traceback
from dbt.clients.yaml_helper import ( # noqa:F401
yaml,
safe_load,
Loader,
Dumper,
)
from dbt.events.functions import fire_event, get_invocation_id
from dbt.events.types import (
DisableTracking,
SendingEvent,
SendEventFailure,
FlushEvents,
FlushEventsFailure,
TrackingInitializeFailure,
)
from dbt import version as dbt_version
from dbt import flags
from snowplow_tracker import Subject, Tracker, Emitter, logger as sp_logger
from snowplow_tracker import SelfDescribingJson
import uuid
from datetime import datetime
from typing import Optional
import logbook
import pytz
import platform
import uuid
import requests
import os
from dbt import flags
from dbt import version as dbt_version
from dbt.clients.yaml_helper import Dumper, Loader, safe_load, yaml # noqa:F401
from dbt.events.functions import fire_event, get_invocation_id
from dbt.events.types import (
DisableTracking,
FlushEvents,
FlushEventsFailure,
SendEventFailure,
SendingEvent,
TrackingInitializeFailure,
)
from snowplow_tracker import Emitter, SelfDescribingJson, Subject, Tracker
from snowplow_tracker import logger as sp_logger
sp_logger.setLevel(100)