Compare commits

...

14 Commits

Author SHA1 Message Date
Gerda Shank
47755b8d0a Merge branch 'main' into ct-1611-protobuf_nodes 2023-02-28 17:06:07 -05:00
Gerda Shank
0466cc26ee Move check of invalid groups earlier in parsing 2023-02-28 13:58:56 -05:00
Gerda Shank
a66075b55c Update test for existence of manifest.json 2023-02-28 12:33:54 -05:00
Gerda Shank
7bca265af1 Merge branch 'ct-1611-protobuf_nodes' of github.com:dbt-labs/dbt-core into ct-1611-protobuf_nodes 2023-02-28 12:00:30 -05:00
Gerda Shank
3088de3fb2 Add group to Metric node 2023-02-28 11:59:53 -05:00
Github Build Bot
dc0d229568 Add generated CLI API docs 2023-02-28 16:55:21 +00:00
Gerda Shank
26f8f983fa Merge branch 'main' into ct-1611-protobuf_nodes 2023-02-28 11:53:30 -05:00
Gerda Shank
316cab46ae Merge branch 'ct-1611-protobuf_nodes' of github.com:dbt-labs/dbt-core into ct-1611-protobuf_nodes 2023-02-27 18:15:12 -05:00
Gerda Shank
12997e483a Run new version of betterproto on events 2023-02-27 18:13:43 -05:00
Gerda Shank
896481818a Update nodes 2023-02-27 17:50:04 -05:00
Gerda Shank
729e85d743 Merge branch 'main' into ct-1611-protobuf_nodes 2023-02-27 17:35:04 -05:00
Github Build Bot
659ce584a9 Add generated CLI API docs 2023-02-27 22:22:28 +00:00
Gerda Shank
a1bc72c0b4 Merge branch 'main' into ct-1611-protobuf_nodes 2023-02-27 17:20:55 -05:00
Gerda Shank
7d5a50dcd5 Create protobuf message representations of graph nodes 2023-02-09 22:13:29 -05:00
23 changed files with 2677 additions and 440 deletions

View File

@@ -0,0 +1,6 @@
kind: Features
body: Crate protobuf representation of nodes
time: 2023-01-31T21:27:02.209758-05:00
custom:
Author: gshank
Issue: "6391"

View File

@@ -8,3 +8,5 @@ MAXIMUM_SEED_SIZE_NAME = "1MB"
PIN_PACKAGE_URL = (
"https://docs.getdbt.com/docs/package-management#section-specifying-package-versions"
)
MANIFEST_FILE_NAME = "manifest.json"

View File

@@ -0,0 +1,28 @@
# nodes.proto messages
For new fields in a node or node config to be included in the protobuf serialized output,
the messages in nodes.proto need to be updated.
Then proto.nodes need to be compiled: ```protoc --python_betterproto_out . nodes.proto```
In order to use optional fields (really necessary for nodes and configs) we had to use
a beta version of betterproto. This version has a bug in the way that it writes the
names of some generated classes, so we will have to update the name of the rpc node from
RpcNode to RPCNode in the generated file.
In addition, betterproto now always creates the generated python file as an __init__.py
file in a subdirectory. For now, I'm moving it up from proto_nodes/__init__.py to proto_nodes.py.
# updating nodes.py and model_config.py for nodes.proto changes
Protobuf python messages objects are created to "to_msg" methods. There is often a list
of attributes to set in a "msg_attributes" method, but this isn't entirely consistent.
If a class has a small number of additional attributes they are sometimes set directly.
Some attributes aren't handled well by the "get_msg_attribute_value" utility function,
in which case they are set directly. This is particularly true of lists or dictionaries
of objects, which need to be converted using a "to_msg" method.
The utility class "get_msg_attribute_value" does a couple of common conversions of
attribute values, such as getting the string value of an enum or converting dictionaries to
dictionaries of strings. A few common more elaborate conversions are also performed, such as
"columns".

View File

@@ -8,11 +8,12 @@ from dbt.dataclass_schema import (
register_pattern,
)
from dbt.contracts.graph.unparsed import AdditionalPropertiesAllowed, Docs
from dbt.contracts.graph.utils import validate_color
from dbt.contracts.graph.utils import validate_color, get_msg_attribute_value
from dbt.exceptions import DbtInternalError, CompilationError
from dbt.contracts.util import Replaceable, list_str
from dbt import hooks
from dbt.node_types import NodeType
from dbt.contracts.graph import proto_nodes
M = TypeVar("M", bound="Metadata")
@@ -195,6 +196,9 @@ class Hook(dbtClassMixin, Replaceable):
transaction: bool = True
index: Optional[int] = None
def to_msg(self):
return proto_nodes.Hook(sql=self.sql, transaction=self.transaction, index=self.index)
T = TypeVar("T", bound="BaseConfig")
@@ -409,9 +413,15 @@ class NodeAndTestConfig(BaseConfig):
metadata=CompareBehavior.Exclude.meta(),
)
@classmethod
def msg_attributes(self):
return ["enabled", "alias", "schema", "database", "tags", "meta"]
@dataclass
class NodeConfig(NodeAndTestConfig):
"""This config is used by ModelNode, AnalysisNode, RPCNode, SqlNode, HookNode"""
# Note: if any new fields are added with MergeBehavior, also update the
# 'mergebehavior' dictionary
materialized: str = "view"
@@ -453,6 +463,35 @@ class NodeConfig(NodeAndTestConfig):
)
contract: bool = False
@classmethod
def msg_attributes(self):
return [
"materialized",
"incremental_strategy",
"persist_docs",
"quoting",
"full_refresh",
"unique_key",
"on_schema_change",
"grants",
"packages",
"docs",
]
def to_msg(self):
# Get matching msg config class
config_name = type(self).__name__
msg_cls = getattr(proto_nodes, config_name)
msg_config = msg_cls()
for cls in [NodeAndTestConfig, NodeConfig]:
for attribute in cls.msg_attributes():
value = get_msg_attribute_value(self, attribute)
setattr(msg_config, attribute, value)
msg_config.post_hook = [hk.to_msg() for hk in self.post_hook]
msg_config.pre_hook = [hk.to_msg() for hk in self.pre_hook]
return msg_config
# we validate that node_color has a suitable value to prevent dbt-docs from crashing
def __post_init__(self):
if self.docs.node_color:
@@ -501,6 +540,11 @@ class SeedConfig(NodeConfig):
materialized: str = "seed"
quote_columns: Optional[bool] = None
def to_msg(self):
msg = super().to_msg()
msg.quote_columns = self.quote_columns
return msg
@classmethod
def validate(cls, data):
super().validate(data)
@@ -508,8 +552,11 @@ class SeedConfig(NodeConfig):
raise ValidationError("A seed must have a materialized value of 'seed'")
# This is used in both GenericTestNode and SingularTestNode, but some
# of these attributes seem specific to GenericTestNode.
@dataclass
class TestConfig(NodeAndTestConfig):
__test__ = False
# this is repeated because of a different default
schema: Optional[str] = field(
default="dbt_test__audit",
@@ -524,6 +571,27 @@ class TestConfig(NodeAndTestConfig):
warn_if: str = "!= 0"
error_if: str = "!= 0"
@classmethod
def msg_attributes(self):
return [
"materialized",
"severity",
"store_failures",
"where",
"limit",
"fail_calc",
"warn_if",
"error_if",
]
def to_msg(self):
msg_config = proto_nodes.TestConfig()
for cls in [NodeAndTestConfig, TestConfig]:
for attribute in cls.msg_attributes():
value = get_msg_attribute_value(self, attribute)
setattr(msg_config, attribute, value)
return msg_config
@classmethod
def same_contents(cls, unrendered: Dict[str, Any], other: Dict[str, Any]) -> bool:
"""This is like __eq__, except it explicitly checks certain fields."""
@@ -569,6 +637,27 @@ class SnapshotConfig(EmptySnapshotConfig):
# Not using Optional because of serialization issues with a Union of str and List[str]
check_cols: Union[str, List[str], None] = None
@classmethod
def msg_attributes(self):
return ["strategy", "unique_key", "target_schema", "target_database"]
def to_msg(self):
msg_config = super().to_msg() # Uses NodeConfig to_msg
for attribute in self.msg_attributes():
value = get_msg_attribute_value(self, attribute)
setattr(msg_config, attribute, value)
msg_config.check_cols = self.normalize_check_cols()
return msg_config
def normalize_check_cols(self):
"""Ensure that check_cols is always a list"""
if self.check_cols is None:
return []
elif isinstance(self.check_cols, str):
return [self.check_cols]
else:
return self.check_cols
@classmethod
def validate(cls, data):
super().validate(data)

View File

@@ -0,0 +1,607 @@
syntax = "proto3";
package proto_nodes;
message ListOfStrings {
repeated string value = 1;
}
message Hook {
string sql = 1;
bool transaction = 2;
optional int32 index = 3;
}
message Docs {
bool show = 1;
optional string node_color = 2;
}
message MacroDependsOn {
repeated string macros = 1;
}
message DependsOn {
repeated string macros = 1;
repeated string nodes = 2;
}
message MacroArgument {
string name = 1;
optional string type = 2;
string description = 3;
}
message Quoting {
optional string database = 1;
optional string schema = 2;
optional string identifier = 3;
optional string column = 4;
}
message Time {
optional int32 count = 1;
optional string period = 2;
}
message FreshnessThreshold {
optional Time warn_after = 1;
optional Time error_after = 2;
optional string filter = 3;
}
message ExposureOwner {
string email = 1;
optional string name = 2;
}
message MetricFilter {
string field = 1;
string operator = 2;
string value = 3;
}
message MetricTime {
optional int32 count = 1;
optional string period = 2;
}
message NodeConfig {
bool enabled = 1;
optional string alias = 2;
optional string schema = 3;
optional string database = 4;
repeated string tags = 5;
map<string, string> meta = 6;
string materialized = 7;
optional string incremental_strategy = 8;
map<string, string> persist_docs = 9;
Hook post_hook = 10;
Hook pre_hook = 11;
map<string, string> quoting = 12;
map<string, string> column_types = 13;
optional bool full_refresh = 14;
optional string unique_key = 15;
string on_schema_change = 16;
map<string, string> grants = 17;
repeated string packages = 18;
Docs docs = 19;
}
message SeedConfig {
bool enabled = 1;
optional string alias = 2;
optional string schema = 3;
optional string database = 4;
repeated string tags = 5;
map<string, string> meta = 6;
string materialized = 7;
optional string incremental_strategy = 8;
map<string, string> persist_docs = 9;
// post_hook = 10;
// pre_hook = 11;
map<string, string> quoting = 12;
map<string, string> column_types = 13;
optional bool full_refresh = 14;
optional string unique_key = 15;
string on_schema_change = 16;
map<string, string> grants = 17;
repeated string packages = 18;
Docs docs = 19;
optional bool quote_columns = 20;
}
message TestConfig {
bool enabled = 1;
optional string alias = 2;
optional string schema = 3;
optional string database = 4;
repeated string tags = 5;
map<string, string> meta = 6;
string materialized = 8;
string severity = 9;
optional bool store_failures = 10;
optional string where = 11;
optional int32 limit = 12;
string fail_calc = 13;
string warn_if = 14;
string error_if = 15;
}
// NodeConfig plus strategy, target_schema, target_database, check_cols
message SnapshotConfig {
bool enabled = 1;
optional string alias = 2;
optional string schema = 3;
optional string database = 4;
repeated string tags = 5;
map<string, string> meta = 6;
string materialized = 7;
optional string incremental_strategy = 8;
map<string, string> persist_docs = 9;
// post_hook = 10;
// pre_hook = 11;
map<string, string> quoting = 12;
map<string, string> column_types = 13;
optional bool full_refresh = 14;
optional string unique_key = 15;
string on_schema_change = 16;
map<string, string> grants = 17;
repeated string packages = 18;
Docs docs = 19;
optional string strategy = 20;
optional string target_schema = 21;
optional string target_database = 22;
repeated string check_cols = 23;
}
message SourceConfig {
bool enabled = 1;
}
message ExposureConfig {
bool enabled = 1;
}
message MetricConfig {
bool enabled = 1;
}
message ColumnInfo {
string name = 1;
string description = 2;
map <string, string> meta = 3;
optional string data_type = 4;
optional bool quote = 5;
repeated string tags = 6;
map <string, string> _extra = 7;
}
// There are three nodes that are exactly the same as this one (AnalysisNode, RPCNode, and
// SqlNOde), and one with only one additional attribute (HookNode). Making separate messages
// for now, but we can revisit later. If we clean up the config classes, some of those
// nodes might end up with different config classes, which would require separate node messages.
message ModelNode {
string name = 1;
string resource_type = 2;
string package_name = 3;
string path = 4;
string original_file_path = 5;
string unique_id = 6;
repeated string fqn = 7;
optional string database = 8;
string schema = 9;
string alias = 10;
NodeConfig config = 11;
repeated string tags = 12;
string description = 13;
map <string, ColumnInfo> columns = 14;
map<string, string> meta = 15;
Docs docs = 16;
string patch_path = 17;
bool deferred = 18;
map<string, string> unrendered_config = 19;
string relation_name = 20;
string raw_code = 21;
string language = 22;
repeated ListOfStrings refs = 23;
repeated ListOfStrings sources = 24;
repeated ListOfStrings metrics = 25;
DependsOn depends_on = 26;
bool compiled = 27;
string compiled_code = 28;
bool contract = 29;
string group = 30;
string access = 31;
}
// This should be exactly the same as ModelNode w/o access
message AnalysisNode {
string name = 1;
string resource_type = 2;
string package_name = 3;
string path = 4;
string original_file_path = 5;
string unique_id = 6;
repeated string fqn = 7;
optional string database = 8;
string schema = 9;
string alias = 10;
NodeConfig config = 11;
repeated string tags = 12;
string description = 13;
map <string, ColumnInfo> columns = 14;
map<string, string> meta = 15;
Docs docs = 16;
string patch_path = 17;
bool deferred = 18;
map<string, string> unrendered_config = 19;
string relation_name = 20;
string raw_code = 21;
string language = 22;
repeated ListOfStrings refs = 23;
repeated ListOfStrings sources = 24;
repeated ListOfStrings metrics = 25;
DependsOn depends_on = 26;
bool compiled = 27;
string compiled_code = 28;
bool contract = 29;
string group = 30;
}
// This should be exactly the same as ModelNode w/o access
message RPCNode {
string name = 1;
string resource_type = 2;
string package_name = 3;
string path = 4;
string original_file_path = 5;
string unique_id = 6;
repeated string fqn = 7;
optional string database = 8;
string schema = 9;
string alias = 10;
NodeConfig config = 11;
repeated string tags = 12;
string description = 13;
map <string, ColumnInfo> columns = 14;
map<string, string> meta = 15;
Docs docs = 16;
string patch_path = 17;
bool deferred = 18;
map<string, string> unrendered_config = 19;
string relation_name = 20;
string raw_code = 21;
string language = 22;
repeated ListOfStrings refs = 23;
repeated ListOfStrings sources = 24;
repeated ListOfStrings metrics = 25;
DependsOn depends_on = 26;
bool compiled = 27;
string compiled_code = 28;
bool contract = 29;
string group = 30;
}
// This should be exactly the same as ModelNode w/o access
message SqlNode {
string name = 1;
string resource_type = 2;
string package_name = 3;
string path = 4;
string original_file_path = 5;
string unique_id = 6;
repeated string fqn = 7;
optional string database = 8;
string schema = 9;
string alias = 10;
NodeConfig config = 11;
repeated string tags = 12;
string description = 13;
map <string, ColumnInfo> columns = 14;
map<string, string> meta = 15;
Docs docs = 16;
string patch_path = 17;
bool deferred = 18;
map<string, string> unrendered_config = 19;
string relation_name = 20;
string raw_code = 21;
string language = 22;
repeated ListOfStrings refs = 23;
repeated ListOfStrings sources = 24;
repeated ListOfStrings metrics = 25;
DependsOn depends_on = 26;
bool compiled = 27;
string compiled_code = 28;
bool contract = 29;
string group = 30;
}
// This should be the same as ModelNode (w/o access) plus additional "index" attribute
message HookNode {
string name = 1;
string resource_type = 2;
string package_name = 3;
string path = 4;
string original_file_path = 5;
string unique_id = 6;
repeated string fqn = 7;
optional string database = 8;
string schema = 9;
string alias = 10;
NodeConfig config = 11;
repeated string tags = 12;
string description = 13;
map <string, ColumnInfo> columns = 14;
map<string, string> meta = 15;
Docs docs = 16;
string patch_path = 17;
bool deferred = 18;
map<string, string> unrendered_config = 19;
string relation_name = 20;
string raw_code = 21;
string language = 22;
repeated ListOfStrings refs = 23;
repeated ListOfStrings sources = 24;
repeated ListOfStrings metrics = 25;
DependsOn depends_on = 26;
bool compiled = 27;
string compiled_code = 28;
bool contract = 29;
string group = 30;
optional int32 index = 31;
}
message SeedNode {
string name = 1;
string resource_type = 2;
string package_name = 3;
string path = 4;
string original_file_path = 5;
string unique_id = 6;
repeated string fqn = 7;
optional string database = 8;
string schema = 9;
string alias = 10;
SeedConfig config = 11;
repeated string tags = 12;
string description = 13;
map <string, ColumnInfo> columns = 14;
map<string, string> meta = 15;
Docs docs = 16;
string patch_path = 17;
bool deferred = 18;
map<string, string> unrendered_config = 19;
string relation_name = 20;
string raw_code = 21;
string root_path = 22;
string group = 23;
}
// Same as ModelNode w/o access except config is TestConfig
message SingularTestNode {
string name = 1;
string resource_type = 2;
string package_name = 3;
string path = 4;
string original_file_path = 5;
string unique_id = 6;
repeated string fqn = 7;
optional string database = 8;
string schema = 9;
string alias = 10;
TestConfig config = 11;
repeated string tags = 12;
string description = 13;
map <string, ColumnInfo> columns = 14;
map<string, string> meta = 15;
Docs docs = 16;
string patch_path = 17;
bool deferred = 18;
map<string, string> unrendered_config = 19;
string relation_name = 20;
string raw_code = 21;
string language = 22;
repeated ListOfStrings refs = 23;
repeated ListOfStrings sources = 24;
repeated ListOfStrings metrics = 25;
DependsOn depends_on = 26;
bool compiled = 27;
string compiled_code = 28;
bool contract = 29;
string group = 30;
}
message TestMetadata {
string name = 1;
map<string, string> kwargs = 2;
optional string namespace = 3;
}
// Same as ModelNode w/o access except config is TestConfig, and has test_metadata
// and column_name attributes.
message GenericTestNode {
string name = 1;
string resource_type = 2;
string package_name = 3;
string path = 4;
string original_file_path = 5;
string unique_id = 6;
repeated string fqn = 7;
optional string database = 8;
string schema = 9;
string alias = 10;
TestConfig config = 11;
repeated string tags = 12;
string description = 13;
map <string, ColumnInfo> columns = 14;
map<string, string> meta = 15;
Docs docs = 16;
string patch_path = 17;
bool deferred = 18;
map<string, string> unrendered_config = 19;
string relation_name = 20;
string raw_code = 21;
string language = 22;
repeated ListOfStrings refs = 23;
repeated ListOfStrings sources = 24;
repeated ListOfStrings metrics = 25;
DependsOn depends_on = 26;
bool compiled = 27;
string compiled_code = 28;
bool contract = 29;
string group = 30;
TestMetadata test_metadata = 31;
optional string column_name = 32;
}
// SnapshotNode - Sames as ModelNode w/o access except with SnapshotConfig
message SnapshotNode {
string name = 1;
string resource_type = 2;
string package_name = 3;
string path = 4;
string original_file_path = 5;
string unique_id = 6;
repeated string fqn = 7;
optional string database = 8;
string schema = 9;
string alias = 10;
SnapshotConfig config = 11;
repeated string tags = 12;
string description = 13;
map <string, ColumnInfo> columns = 14;
map<string, string> meta = 15;
Docs docs = 16;
string patch_path = 17;
bool deferred = 18;
map<string, string> unrendered_config = 19;
string relation_name = 20;
string raw_code = 21;
string language = 22;
repeated ListOfStrings refs = 23;
repeated ListOfStrings sources = 24;
repeated ListOfStrings metrics = 25;
DependsOn depends_on = 26;
bool compiled = 27;
string compiled_code = 28;
bool contract = 29;
string group = 30;
}
// Macro - BaseNode plus additional attributes
message Macro {
string name = 1;
string resource_type = 2;
string package_name = 3;
string path = 4;
string original_file_path = 5;
string unique_id = 6;
string macro_sql = 7;
MacroDependsOn depends_on = 8;
string description = 9;
map<string, string> meta = 10;
Docs docs = 11;
optional string patch_path = 12;
repeated MacroArgument arguments = 13;
repeated string supported_languages = 14;
}
// Documentation - BaseNode plus block_contents
message Documentation {
string name = 1;
string resource_type = 2;
string package_name = 3;
string path = 4;
string original_file_path = 5;
string unique_id = 6;
string block_contents = 7;
}
message SourceDefinition {
string name = 1;
string resource_type = 2;
string package_name = 3;
string path = 4;
string original_file_path = 5;
string unique_id = 6;
repeated string fqn = 7;
optional string database = 8;
string schema = 9;
string source_name = 10;
string source_description = 11;
string loader = 12;
string identifier = 13;
Quoting quoting = 14;
optional string loaded_at_field = 15;
optional FreshnessThreshold freshness = 16;
// optional ExternalTable external = 17;
string description = 18;
map<string, ColumnInfo> columns = 19;
map<string, string> meta = 20;
map<string, string> source_meta = 21;
repeated string tags = 22;
SourceConfig config = 23;
optional string patch_path = 24;
map<string, string> unrendered_config = 25;
optional string relation_name = 26;
}
message Exposure {
string name = 1;
string resource_type = 2;
string package_name = 3;
string path = 4;
string original_file_path = 5;
string unique_id = 6;
repeated string fqn = 7;
string type = 8;
ExposureOwner owner = 9;
string description = 10;
optional string label = 11;
optional string maturity = 12;
map<string, string> meta = 13;
repeated string tags = 14;
ExposureConfig config = 15;
map<string, string> unrendered_config = 16;
optional string url = 17;
DependsOn depends_on = 18;
repeated ListOfStrings refs = 24;
repeated ListOfStrings sources = 25;
repeated ListOfStrings metrics = 26;
}
message Metric {
string name = 1;
string resource_type = 2;
string package_name = 3;
string path = 4;
string original_file_path = 5;
string unique_id = 6;
repeated string fqn = 7;
string description = 8;
string label = 9;
string calculation_method = 10;
string expression = 11;
repeated MetricFilter filters = 12;
repeated string time_grains = 13;
repeated string dimensions = 14;
optional string timestamp = 15;
optional MetricTime window = 16;
optional string model = 17;
optional string model_unique_id = 18;
map<string, string> meta = 19;
repeated string tags = 20;
MetricConfig config = 21;
map<string, string> unrendered_config = 22;
repeated ListOfStrings refs = 24;
repeated ListOfStrings sources = 25;
repeated ListOfStrings metrics = 26;
DependsOn depends_on = 27;
string group = 28;
}

View File

@@ -35,6 +35,7 @@ from dbt.contracts.graph.unparsed import (
MetricTime,
)
from dbt.contracts.util import Replaceable, AdditionalPropertiesMixin
from dbt.contracts.graph.utils import get_msg_attribute_value
from dbt.events.proto_types import NodeInfo
from dbt.events.functions import warn_or_error
from dbt.exceptions import ParsingError, InvalidAccessTypeError
@@ -50,6 +51,7 @@ from dbt.flags import get_flags
from dbt.node_types import ModelLanguage, NodeType, AccessType
from dbt.utils import cast_dict_to_dict_of_strings
from dbt.contracts.graph import proto_nodes
from .model_config import (
NodeConfig,
@@ -97,6 +99,18 @@ class BaseNode(dbtClassMixin, Replaceable):
original_file_path: str
unique_id: str
@classmethod
def msg_attributes(self):
"""Attributes of this class that are included in protobuf definitions"""
return [
"name",
"resource_type",
"package_name",
"path",
"original_file_path",
"unique_id",
]
@property
def search_name(self):
return self.name
@@ -136,6 +150,10 @@ class GraphNode(BaseNode):
fqn: List[str]
@classmethod
def msg_attributes(self):
return ["fqn"]
def same_fqn(self, other) -> bool:
return self.fqn == other.fqn
@@ -154,6 +172,18 @@ class ColumnInfo(AdditionalPropertiesMixin, ExtensibleDbtClassMixin, Replaceable
tags: List[str] = field(default_factory=list)
_extra: Dict[str, Any] = field(default_factory=dict)
def to_msg(self):
column_info_msg = proto_nodes.ColumnInfo(
name=self.name,
description=self.description,
meta=cast_dict_to_dict_of_strings(self.meta),
data_type=self.data_type,
quote=self.quote,
tags=self.tags,
_extra=cast_dict_to_dict_of_strings(self._extra),
)
return column_info_msg
# Metrics, exposures,
@dataclass
@@ -183,6 +213,9 @@ class MacroDependsOn(dbtClassMixin, Replaceable):
if value not in self.macros:
self.macros.append(value)
def to_msg(self):
return proto_nodes.MacroDependsOn(macros=self.macros)
@dataclass
class DependsOn(MacroDependsOn):
@@ -192,13 +225,20 @@ class DependsOn(MacroDependsOn):
if value not in self.nodes:
self.nodes.append(value)
def to_msg(self):
return proto_nodes.DependsOn(nodes=self.nodes, macros=self.macros)
@dataclass
class ParsedNodeMandatory(GraphNode, HasRelationMetadata, Replaceable):
alias: str
checksum: FileHash
checksum: FileHash # not included in protobuf messages
config: NodeConfig = field(default_factory=NodeConfig)
@classmethod
def msg_attributes(self):
return ["fqn", "database", "schema", "alias", "config"]
@property
def identifier(self):
return self.alias
@@ -254,6 +294,23 @@ class ParsedNode(NodeInfoMixin, ParsedNodeMandatory, SerializableType):
relation_name: Optional[str] = None
raw_code: str = ""
@classmethod
def msg_attributes(self):
# Does not included created_at, config_call_dict, build_path
return [
"tags",
"description",
"columns",
"meta",
"docs",
"patch_path",
"deferred",
"unrendered_config",
"relation_name",
"raw_code",
"group",
]
def write_node(self, target_path: str, subdirectory: str, payload: str):
if os.path.basename(self.path) == os.path.basename(self.original_file_path):
# One-to-one relationship of nodes to files.
@@ -427,6 +484,19 @@ class CompiledNode(ParsedNode):
_pre_injected_sql: Optional[str] = None
contract: bool = False
@classmethod
def msg_attributes(self):
# Does not include extra_ctes_injected, extra_ctes, _pre_injected_sql, compiled_path
return [
"language",
"refs",
"sources",
"metrics",
"depends_on",
"compiled",
"compiled_code",
]
@property
def empty(self):
return not self.raw_code.strip()
@@ -466,6 +536,19 @@ class CompiledNode(ParsedNode):
def depends_on_macros(self):
return self.depends_on.macros
def to_msg(self):
# Get matching msg node class
node_name = type(self).__name__
msg_cls = getattr(proto_nodes, node_name)
msg_node = msg_cls()
for cls in [BaseNode, GraphNode, ParsedNodeMandatory, ParsedNode, CompiledNode]:
for attribute in cls.msg_attributes():
value = get_msg_attribute_value(self, attribute)
setattr(msg_node, attribute, value)
return msg_node
# ====================================
# CompiledNode subclasses
@@ -482,14 +565,24 @@ class HookNode(CompiledNode):
resource_type: NodeType = field(metadata={"restrict": [NodeType.Operation]})
index: Optional[int] = None
def to_msg(self):
msg = super().to_msg()
msg.index = self.index
return msg
@dataclass
class ModelNode(CompiledNode):
resource_type: NodeType = field(metadata={"restrict": [NodeType.Model]})
access: AccessType = AccessType.Protected
def to_msg(self):
msg = super().to_msg()
msg.access = self.access
return msg
# TODO: rm?
# TODO: this node type should probably be removed when the rpc server is no longer supported
@dataclass
class RPCNode(CompiledNode):
resource_type: NodeType = field(metadata={"restrict": [NodeType.RPCCall]})
@@ -514,6 +607,15 @@ class SeedNode(ParsedNode): # No SQLDefaults!
root_path: Optional[str] = None
depends_on: MacroDependsOn = field(default_factory=MacroDependsOn)
def to_msg(self):
seed_node = proto_nodes.SeedNode()
for cls in [BaseNode, ParsedNodeMandatory, ParsedNode]:
for attribute in cls.msg_attributes():
value = get_msg_attribute_value(self, attribute)
setattr(seed_node, attribute, value)
seed_node.root_path = self.root_path
return seed_node
def same_seeds(self, other: "SeedNode") -> bool:
# for seeds, we check the hashes. If the hashes are different types,
# no match. If the hashes are both the same 'path', log a warning and
@@ -637,6 +739,8 @@ class SingularTestNode(TestShouldStoreFailures, CompiledNode):
# refactor the various configs.
config: TestConfig = field(default_factory=TestConfig) # type: ignore
# no to_msg method because the CompiledNode default works fine
@property
def test_node_type(self):
return "singular"
@@ -649,6 +753,7 @@ class SingularTestNode(TestShouldStoreFailures, CompiledNode):
@dataclass
class TestMetadata(dbtClassMixin, Replaceable):
__test__ = False
name: str
# kwargs are the args that are left in the test builder after
# removing configs. They are set from the test builder when
@@ -668,12 +773,17 @@ class HasTestMetadata(dbtClassMixin):
class GenericTestNode(TestShouldStoreFailures, CompiledNode, HasTestMetadata):
resource_type: NodeType = field(metadata={"restrict": [NodeType.Test]})
column_name: Optional[str] = None
file_key_name: Optional[str] = None
file_key_name: Optional[str] = None # Not included in protobuf message
# Was not able to make mypy happy and keep the code working. We need to
# refactor the various configs.
config: TestConfig = field(default_factory=TestConfig) # type: ignore
attached_node: Optional[str] = None
def to_msg(self):
msg = super().to_msg()
msg.column_name = self.column_name
return msg
def same_contents(self, other) -> bool:
if other is None:
return False
@@ -727,6 +837,29 @@ class Macro(BaseNode):
created_at: float = field(default_factory=lambda: time.time())
supported_languages: Optional[List[ModelLanguage]] = None
@classmethod
def msg_attributes(self):
return ["macro_sql", "depends_on", "description", "meta", "docs", "patch_path"]
def to_msg(self):
msg = proto_nodes.Macro()
for cls in [BaseNode, Macro]:
for attribute in cls.msg_attributes():
value = getattr(self, attribute)
if value is None:
continue
setattr(msg, attribute, value)
arguments = [
proto_nodes.MacroArgument(name=ma.name, description=ma.description, type=ma.type)
for ma in self.arguments
]
msg.arguments = arguments
supported_languages = []
if isinstance(self.supported_languages, list):
supported_languages = [ml.value for ml in self.supported_languages]
msg.supported_languages = supported_languages
return msg
def patch(self, patch: "ParsedMacroPatch"):
self.patch_path: Optional[str] = patch.file_id
self.description = patch.description
@@ -757,6 +890,15 @@ class Documentation(BaseNode):
block_contents: str
resource_type: NodeType = field(metadata={"restrict": [NodeType.Documentation]})
def to_msg(self):
msg = proto_nodes.Documentation()
for cls in [BaseNode]:
for attribute in cls.msg_attributes():
value = getattr(self, attribute)
setattr(msg, attribute, value)
msg.block_contents = self.block_contents
return msg
@property
def search_name(self):
return self.name
@@ -851,6 +993,39 @@ class SourceDefinition(NodeInfoMixin, ParsedSourceMandatory):
relation_name: Optional[str] = None
created_at: float = field(default_factory=lambda: time.time())
@classmethod
def msg_attributes(self):
return [
"source_name",
"source_description",
"loader",
"identifier",
"resource_type",
"quoting",
"loaded_at_field",
"freshness",
"external",
"description",
"columns",
"meta",
"source_meta",
"tags",
"config",
"patch_path",
"unrendered_config",
"relation_name",
"database",
"schema",
]
def to_msg(self):
msg = proto_nodes.SourceDefinition()
for cls in [BaseNode, GraphNode, SourceDefinition]:
for attribute in cls.msg_attributes():
value = get_msg_attribute_value(self, attribute)
setattr(msg, attribute, value)
return msg
def __post_serialize__(self, dct):
if "_event_status" in dct:
del dct["_event_status"]
@@ -973,6 +1148,32 @@ class Exposure(GraphNode):
metrics: List[List[str]] = field(default_factory=list)
created_at: float = field(default_factory=lambda: time.time())
@classmethod
def msg_attributes(self):
return [
"type",
"description",
"label",
"maturity",
"meta",
"tags",
"config",
"unrendered_config",
"url",
"depends_on",
"refs",
"sources",
"metrics",
]
def to_msg(self):
msg = proto_nodes.Exposure()
for cls in [BaseNode, GraphNode, Exposure]:
for attribute in cls.msg_attributes():
value = get_msg_attribute_value(self, attribute)
setattr(msg, attribute, value)
return msg
@property
def depends_on_nodes(self):
return self.depends_on.nodes
@@ -1033,15 +1234,8 @@ class Exposure(GraphNode):
# ====================================
@dataclass
class MetricReference(dbtClassMixin, Replaceable):
sql: Optional[Union[str, int]]
unique_id: Optional[str]
@dataclass
class Metric(GraphNode):
name: str
description: str
label: str
calculation_method: str
@@ -1058,13 +1252,46 @@ class Metric(GraphNode):
tags: List[str] = field(default_factory=list)
config: MetricConfig = field(default_factory=MetricConfig)
unrendered_config: Dict[str, Any] = field(default_factory=dict)
sources: List[List[str]] = field(default_factory=list)
depends_on: DependsOn = field(default_factory=DependsOn)
refs: List[List[str]] = field(default_factory=list)
sources: List[List[str]] = field(default_factory=list)
metrics: List[List[str]] = field(default_factory=list)
depends_on: DependsOn = field(default_factory=DependsOn)
created_at: float = field(default_factory=lambda: time.time())
group: Optional[str] = None
@classmethod
def msg_attributes(self):
return [
"description",
"label",
"calculation_method",
"expression",
"time_grains",
"dimensions",
"timestamp",
"window",
"model",
"model_unique_id",
"meta",
"tags",
"config",
"unrendered_config",
"refs",
"sources",
"metrics",
"depends_on",
"group",
]
def to_msg(self):
msg = proto_nodes.Metric()
for cls in [BaseNode, GraphNode, Metric]:
for attribute in cls.msg_attributes():
value = get_msg_attribute_value(self, attribute)
setattr(msg, attribute, value)
msg.filters = [mf.to_msg() for mf in self.filters]
return msg
@property
def depends_on_nodes(self):
return self.depends_on.nodes

View File

@@ -0,0 +1,801 @@
# Generated by the protocol buffer compiler. DO NOT EDIT!
# sources: nodes.proto
# plugin: python-betterproto
from dataclasses import dataclass
from typing import (
Dict,
List,
Optional,
)
import betterproto
@dataclass(eq=False, repr=False)
class ListOfStrings(betterproto.Message):
value: List[str] = betterproto.string_field(1)
@dataclass(eq=False, repr=False)
class Hook(betterproto.Message):
sql: str = betterproto.string_field(1)
transaction: bool = betterproto.bool_field(2)
index: Optional[int] = betterproto.int32_field(3, optional=True, group="_index")
@dataclass(eq=False, repr=False)
class Docs(betterproto.Message):
show: bool = betterproto.bool_field(1)
node_color: Optional[str] = betterproto.string_field(2, optional=True, group="_node_color")
@dataclass(eq=False, repr=False)
class MacroDependsOn(betterproto.Message):
macros: List[str] = betterproto.string_field(1)
@dataclass(eq=False, repr=False)
class DependsOn(betterproto.Message):
macros: List[str] = betterproto.string_field(1)
nodes: List[str] = betterproto.string_field(2)
@dataclass(eq=False, repr=False)
class MacroArgument(betterproto.Message):
name: str = betterproto.string_field(1)
type: Optional[str] = betterproto.string_field(2, optional=True, group="_type")
description: str = betterproto.string_field(3)
@dataclass(eq=False, repr=False)
class Quoting(betterproto.Message):
database: Optional[str] = betterproto.string_field(1, optional=True, group="_database")
schema: Optional[str] = betterproto.string_field(2, optional=True, group="_schema")
identifier: Optional[str] = betterproto.string_field(3, optional=True, group="_identifier")
column: Optional[str] = betterproto.string_field(4, optional=True, group="_column")
@dataclass(eq=False, repr=False)
class Time(betterproto.Message):
count: Optional[int] = betterproto.int32_field(1, optional=True, group="_count")
period: Optional[str] = betterproto.string_field(2, optional=True, group="_period")
@dataclass(eq=False, repr=False)
class FreshnessThreshold(betterproto.Message):
warn_after: Optional["Time"] = betterproto.message_field(1, optional=True, group="_warn_after")
error_after: Optional["Time"] = betterproto.message_field(
2, optional=True, group="_error_after"
)
filter: Optional[str] = betterproto.string_field(3, optional=True, group="_filter")
@dataclass(eq=False, repr=False)
class ExposureOwner(betterproto.Message):
email: str = betterproto.string_field(1)
name: Optional[str] = betterproto.string_field(2, optional=True, group="_name")
@dataclass(eq=False, repr=False)
class MetricFilter(betterproto.Message):
field: str = betterproto.string_field(1)
operator: str = betterproto.string_field(2)
value: str = betterproto.string_field(3)
@dataclass(eq=False, repr=False)
class MetricTime(betterproto.Message):
count: Optional[int] = betterproto.int32_field(1, optional=True, group="_count")
period: Optional[str] = betterproto.string_field(2, optional=True, group="_period")
@dataclass(eq=False, repr=False)
class NodeConfig(betterproto.Message):
enabled: bool = betterproto.bool_field(1)
alias: Optional[str] = betterproto.string_field(2, optional=True, group="_alias")
schema: Optional[str] = betterproto.string_field(3, optional=True, group="_schema")
database: Optional[str] = betterproto.string_field(4, optional=True, group="_database")
tags: List[str] = betterproto.string_field(5)
meta: Dict[str, str] = betterproto.map_field(
6, betterproto.TYPE_STRING, betterproto.TYPE_STRING
)
materialized: str = betterproto.string_field(7)
incremental_strategy: Optional[str] = betterproto.string_field(
8, optional=True, group="_incremental_strategy"
)
persist_docs: Dict[str, str] = betterproto.map_field(
9, betterproto.TYPE_STRING, betterproto.TYPE_STRING
)
post_hook: "Hook" = betterproto.message_field(10)
pre_hook: "Hook" = betterproto.message_field(11)
quoting: Dict[str, str] = betterproto.map_field(
12, betterproto.TYPE_STRING, betterproto.TYPE_STRING
)
column_types: Dict[str, str] = betterproto.map_field(
13, betterproto.TYPE_STRING, betterproto.TYPE_STRING
)
full_refresh: Optional[bool] = betterproto.bool_field(14, optional=True, group="_full_refresh")
unique_key: Optional[str] = betterproto.string_field(15, optional=True, group="_unique_key")
on_schema_change: str = betterproto.string_field(16)
grants: Dict[str, str] = betterproto.map_field(
17, betterproto.TYPE_STRING, betterproto.TYPE_STRING
)
packages: List[str] = betterproto.string_field(18)
docs: "Docs" = betterproto.message_field(19)
@dataclass(eq=False, repr=False)
class SeedConfig(betterproto.Message):
enabled: bool = betterproto.bool_field(1)
alias: Optional[str] = betterproto.string_field(2, optional=True, group="_alias")
schema: Optional[str] = betterproto.string_field(3, optional=True, group="_schema")
database: Optional[str] = betterproto.string_field(4, optional=True, group="_database")
tags: List[str] = betterproto.string_field(5)
meta: Dict[str, str] = betterproto.map_field(
6, betterproto.TYPE_STRING, betterproto.TYPE_STRING
)
materialized: str = betterproto.string_field(7)
incremental_strategy: Optional[str] = betterproto.string_field(
8, optional=True, group="_incremental_strategy"
)
persist_docs: Dict[str, str] = betterproto.map_field(
9, betterproto.TYPE_STRING, betterproto.TYPE_STRING
)
quoting: Dict[str, str] = betterproto.map_field(
12, betterproto.TYPE_STRING, betterproto.TYPE_STRING
)
"""post_hook = 10; pre_hook = 11;"""
column_types: Dict[str, str] = betterproto.map_field(
13, betterproto.TYPE_STRING, betterproto.TYPE_STRING
)
full_refresh: Optional[bool] = betterproto.bool_field(14, optional=True, group="_full_refresh")
unique_key: Optional[str] = betterproto.string_field(15, optional=True, group="_unique_key")
on_schema_change: str = betterproto.string_field(16)
grants: Dict[str, str] = betterproto.map_field(
17, betterproto.TYPE_STRING, betterproto.TYPE_STRING
)
packages: List[str] = betterproto.string_field(18)
docs: "Docs" = betterproto.message_field(19)
quote_columns: Optional[bool] = betterproto.bool_field(
20, optional=True, group="_quote_columns"
)
@dataclass(eq=False, repr=False)
class TestConfig(betterproto.Message):
enabled: bool = betterproto.bool_field(1)
alias: Optional[str] = betterproto.string_field(2, optional=True, group="_alias")
schema: Optional[str] = betterproto.string_field(3, optional=True, group="_schema")
database: Optional[str] = betterproto.string_field(4, optional=True, group="_database")
tags: List[str] = betterproto.string_field(5)
meta: Dict[str, str] = betterproto.map_field(
6, betterproto.TYPE_STRING, betterproto.TYPE_STRING
)
materialized: str = betterproto.string_field(8)
severity: str = betterproto.string_field(9)
store_failures: Optional[bool] = betterproto.bool_field(
10, optional=True, group="_store_failures"
)
where: Optional[str] = betterproto.string_field(11, optional=True, group="_where")
limit: Optional[int] = betterproto.int32_field(12, optional=True, group="_limit")
fail_calc: str = betterproto.string_field(13)
warn_if: str = betterproto.string_field(14)
error_if: str = betterproto.string_field(15)
@dataclass(eq=False, repr=False)
class SnapshotConfig(betterproto.Message):
"""NodeConfig plus strategy, target_schema, target_database, check_cols"""
enabled: bool = betterproto.bool_field(1)
alias: Optional[str] = betterproto.string_field(2, optional=True, group="_alias")
schema: Optional[str] = betterproto.string_field(3, optional=True, group="_schema")
database: Optional[str] = betterproto.string_field(4, optional=True, group="_database")
tags: List[str] = betterproto.string_field(5)
meta: Dict[str, str] = betterproto.map_field(
6, betterproto.TYPE_STRING, betterproto.TYPE_STRING
)
materialized: str = betterproto.string_field(7)
incremental_strategy: Optional[str] = betterproto.string_field(
8, optional=True, group="_incremental_strategy"
)
persist_docs: Dict[str, str] = betterproto.map_field(
9, betterproto.TYPE_STRING, betterproto.TYPE_STRING
)
quoting: Dict[str, str] = betterproto.map_field(
12, betterproto.TYPE_STRING, betterproto.TYPE_STRING
)
"""post_hook = 10; pre_hook = 11;"""
column_types: Dict[str, str] = betterproto.map_field(
13, betterproto.TYPE_STRING, betterproto.TYPE_STRING
)
full_refresh: Optional[bool] = betterproto.bool_field(14, optional=True, group="_full_refresh")
unique_key: Optional[str] = betterproto.string_field(15, optional=True, group="_unique_key")
on_schema_change: str = betterproto.string_field(16)
grants: Dict[str, str] = betterproto.map_field(
17, betterproto.TYPE_STRING, betterproto.TYPE_STRING
)
packages: List[str] = betterproto.string_field(18)
docs: "Docs" = betterproto.message_field(19)
strategy: Optional[str] = betterproto.string_field(20, optional=True, group="_strategy")
target_schema: Optional[str] = betterproto.string_field(
21, optional=True, group="_target_schema"
)
target_database: Optional[str] = betterproto.string_field(
22, optional=True, group="_target_database"
)
check_cols: List[str] = betterproto.string_field(23)
@dataclass(eq=False, repr=False)
class SourceConfig(betterproto.Message):
enabled: bool = betterproto.bool_field(1)
@dataclass(eq=False, repr=False)
class ExposureConfig(betterproto.Message):
enabled: bool = betterproto.bool_field(1)
@dataclass(eq=False, repr=False)
class MetricConfig(betterproto.Message):
enabled: bool = betterproto.bool_field(1)
@dataclass(eq=False, repr=False)
class ColumnInfo(betterproto.Message):
name: str = betterproto.string_field(1)
description: str = betterproto.string_field(2)
meta: Dict[str, str] = betterproto.map_field(
3, betterproto.TYPE_STRING, betterproto.TYPE_STRING
)
data_type: Optional[str] = betterproto.string_field(4, optional=True, group="_data_type")
quote: Optional[bool] = betterproto.bool_field(5, optional=True, group="_quote")
tags: List[str] = betterproto.string_field(6)
extra: Dict[str, str] = betterproto.map_field(
7, betterproto.TYPE_STRING, betterproto.TYPE_STRING
)
@dataclass(eq=False, repr=False)
class ModelNode(betterproto.Message):
"""
There are three nodes that are exactly the same as this one (AnalysisNode,
RPCNode, and SqlNOde), and one with only one additional attribute
(HookNode). Making separate messages for now, but we can revisit later. If
we clean up the config classes, some of those nodes might end up with
different config classes, which would require separate node messages.
"""
name: str = betterproto.string_field(1)
resource_type: str = betterproto.string_field(2)
package_name: str = betterproto.string_field(3)
path: str = betterproto.string_field(4)
original_file_path: str = betterproto.string_field(5)
unique_id: str = betterproto.string_field(6)
fqn: List[str] = betterproto.string_field(7)
database: Optional[str] = betterproto.string_field(8, optional=True, group="_database")
schema: str = betterproto.string_field(9)
alias: str = betterproto.string_field(10)
config: "NodeConfig" = betterproto.message_field(11)
tags: List[str] = betterproto.string_field(12)
description: str = betterproto.string_field(13)
columns: Dict[str, "ColumnInfo"] = betterproto.map_field(
14, betterproto.TYPE_STRING, betterproto.TYPE_MESSAGE
)
meta: Dict[str, str] = betterproto.map_field(
15, betterproto.TYPE_STRING, betterproto.TYPE_STRING
)
docs: "Docs" = betterproto.message_field(16)
patch_path: str = betterproto.string_field(17)
deferred: bool = betterproto.bool_field(18)
unrendered_config: Dict[str, str] = betterproto.map_field(
19, betterproto.TYPE_STRING, betterproto.TYPE_STRING
)
relation_name: str = betterproto.string_field(20)
raw_code: str = betterproto.string_field(21)
language: str = betterproto.string_field(22)
refs: List["ListOfStrings"] = betterproto.message_field(23)
sources: List["ListOfStrings"] = betterproto.message_field(24)
metrics: List["ListOfStrings"] = betterproto.message_field(25)
depends_on: "DependsOn" = betterproto.message_field(26)
compiled: bool = betterproto.bool_field(27)
compiled_code: str = betterproto.string_field(28)
contract: bool = betterproto.bool_field(29)
group: str = betterproto.string_field(30)
access: str = betterproto.string_field(31)
@dataclass(eq=False, repr=False)
class AnalysisNode(betterproto.Message):
"""This should be exactly the same as ModelNode w/o access"""
name: str = betterproto.string_field(1)
resource_type: str = betterproto.string_field(2)
package_name: str = betterproto.string_field(3)
path: str = betterproto.string_field(4)
original_file_path: str = betterproto.string_field(5)
unique_id: str = betterproto.string_field(6)
fqn: List[str] = betterproto.string_field(7)
database: Optional[str] = betterproto.string_field(8, optional=True, group="_database")
schema: str = betterproto.string_field(9)
alias: str = betterproto.string_field(10)
config: "NodeConfig" = betterproto.message_field(11)
tags: List[str] = betterproto.string_field(12)
description: str = betterproto.string_field(13)
columns: Dict[str, "ColumnInfo"] = betterproto.map_field(
14, betterproto.TYPE_STRING, betterproto.TYPE_MESSAGE
)
meta: Dict[str, str] = betterproto.map_field(
15, betterproto.TYPE_STRING, betterproto.TYPE_STRING
)
docs: "Docs" = betterproto.message_field(16)
patch_path: str = betterproto.string_field(17)
deferred: bool = betterproto.bool_field(18)
unrendered_config: Dict[str, str] = betterproto.map_field(
19, betterproto.TYPE_STRING, betterproto.TYPE_STRING
)
relation_name: str = betterproto.string_field(20)
raw_code: str = betterproto.string_field(21)
language: str = betterproto.string_field(22)
refs: List["ListOfStrings"] = betterproto.message_field(23)
sources: List["ListOfStrings"] = betterproto.message_field(24)
metrics: List["ListOfStrings"] = betterproto.message_field(25)
depends_on: "DependsOn" = betterproto.message_field(26)
compiled: bool = betterproto.bool_field(27)
compiled_code: str = betterproto.string_field(28)
contract: bool = betterproto.bool_field(29)
group: str = betterproto.string_field(30)
@dataclass(eq=False, repr=False)
class RPCNode(betterproto.Message):
"""This should be exactly the same as ModelNode w/o access"""
name: str = betterproto.string_field(1)
resource_type: str = betterproto.string_field(2)
package_name: str = betterproto.string_field(3)
path: str = betterproto.string_field(4)
original_file_path: str = betterproto.string_field(5)
unique_id: str = betterproto.string_field(6)
fqn: List[str] = betterproto.string_field(7)
database: Optional[str] = betterproto.string_field(8, optional=True, group="_database")
schema: str = betterproto.string_field(9)
alias: str = betterproto.string_field(10)
config: "NodeConfig" = betterproto.message_field(11)
tags: List[str] = betterproto.string_field(12)
description: str = betterproto.string_field(13)
columns: Dict[str, "ColumnInfo"] = betterproto.map_field(
14, betterproto.TYPE_STRING, betterproto.TYPE_MESSAGE
)
meta: Dict[str, str] = betterproto.map_field(
15, betterproto.TYPE_STRING, betterproto.TYPE_STRING
)
docs: "Docs" = betterproto.message_field(16)
patch_path: str = betterproto.string_field(17)
deferred: bool = betterproto.bool_field(18)
unrendered_config: Dict[str, str] = betterproto.map_field(
19, betterproto.TYPE_STRING, betterproto.TYPE_STRING
)
relation_name: str = betterproto.string_field(20)
raw_code: str = betterproto.string_field(21)
language: str = betterproto.string_field(22)
refs: List["ListOfStrings"] = betterproto.message_field(23)
sources: List["ListOfStrings"] = betterproto.message_field(24)
metrics: List["ListOfStrings"] = betterproto.message_field(25)
depends_on: "DependsOn" = betterproto.message_field(26)
compiled: bool = betterproto.bool_field(27)
compiled_code: str = betterproto.string_field(28)
contract: bool = betterproto.bool_field(29)
group: str = betterproto.string_field(30)
@dataclass(eq=False, repr=False)
class SqlNode(betterproto.Message):
"""This should be exactly the same as ModelNode w/o access"""
name: str = betterproto.string_field(1)
resource_type: str = betterproto.string_field(2)
package_name: str = betterproto.string_field(3)
path: str = betterproto.string_field(4)
original_file_path: str = betterproto.string_field(5)
unique_id: str = betterproto.string_field(6)
fqn: List[str] = betterproto.string_field(7)
database: Optional[str] = betterproto.string_field(8, optional=True, group="_database")
schema: str = betterproto.string_field(9)
alias: str = betterproto.string_field(10)
config: "NodeConfig" = betterproto.message_field(11)
tags: List[str] = betterproto.string_field(12)
description: str = betterproto.string_field(13)
columns: Dict[str, "ColumnInfo"] = betterproto.map_field(
14, betterproto.TYPE_STRING, betterproto.TYPE_MESSAGE
)
meta: Dict[str, str] = betterproto.map_field(
15, betterproto.TYPE_STRING, betterproto.TYPE_STRING
)
docs: "Docs" = betterproto.message_field(16)
patch_path: str = betterproto.string_field(17)
deferred: bool = betterproto.bool_field(18)
unrendered_config: Dict[str, str] = betterproto.map_field(
19, betterproto.TYPE_STRING, betterproto.TYPE_STRING
)
relation_name: str = betterproto.string_field(20)
raw_code: str = betterproto.string_field(21)
language: str = betterproto.string_field(22)
refs: List["ListOfStrings"] = betterproto.message_field(23)
sources: List["ListOfStrings"] = betterproto.message_field(24)
metrics: List["ListOfStrings"] = betterproto.message_field(25)
depends_on: "DependsOn" = betterproto.message_field(26)
compiled: bool = betterproto.bool_field(27)
compiled_code: str = betterproto.string_field(28)
contract: bool = betterproto.bool_field(29)
group: str = betterproto.string_field(30)
@dataclass(eq=False, repr=False)
class HookNode(betterproto.Message):
"""
This should be the same as ModelNode (w/o access) plus additional "index"
attribute
"""
name: str = betterproto.string_field(1)
resource_type: str = betterproto.string_field(2)
package_name: str = betterproto.string_field(3)
path: str = betterproto.string_field(4)
original_file_path: str = betterproto.string_field(5)
unique_id: str = betterproto.string_field(6)
fqn: List[str] = betterproto.string_field(7)
database: Optional[str] = betterproto.string_field(8, optional=True, group="_database")
schema: str = betterproto.string_field(9)
alias: str = betterproto.string_field(10)
config: "NodeConfig" = betterproto.message_field(11)
tags: List[str] = betterproto.string_field(12)
description: str = betterproto.string_field(13)
columns: Dict[str, "ColumnInfo"] = betterproto.map_field(
14, betterproto.TYPE_STRING, betterproto.TYPE_MESSAGE
)
meta: Dict[str, str] = betterproto.map_field(
15, betterproto.TYPE_STRING, betterproto.TYPE_STRING
)
docs: "Docs" = betterproto.message_field(16)
patch_path: str = betterproto.string_field(17)
deferred: bool = betterproto.bool_field(18)
unrendered_config: Dict[str, str] = betterproto.map_field(
19, betterproto.TYPE_STRING, betterproto.TYPE_STRING
)
relation_name: str = betterproto.string_field(20)
raw_code: str = betterproto.string_field(21)
language: str = betterproto.string_field(22)
refs: List["ListOfStrings"] = betterproto.message_field(23)
sources: List["ListOfStrings"] = betterproto.message_field(24)
metrics: List["ListOfStrings"] = betterproto.message_field(25)
depends_on: "DependsOn" = betterproto.message_field(26)
compiled: bool = betterproto.bool_field(27)
compiled_code: str = betterproto.string_field(28)
contract: bool = betterproto.bool_field(29)
group: str = betterproto.string_field(30)
index: Optional[int] = betterproto.int32_field(31, optional=True, group="_index")
@dataclass(eq=False, repr=False)
class SeedNode(betterproto.Message):
name: str = betterproto.string_field(1)
resource_type: str = betterproto.string_field(2)
package_name: str = betterproto.string_field(3)
path: str = betterproto.string_field(4)
original_file_path: str = betterproto.string_field(5)
unique_id: str = betterproto.string_field(6)
fqn: List[str] = betterproto.string_field(7)
database: Optional[str] = betterproto.string_field(8, optional=True, group="_database")
schema: str = betterproto.string_field(9)
alias: str = betterproto.string_field(10)
config: "SeedConfig" = betterproto.message_field(11)
tags: List[str] = betterproto.string_field(12)
description: str = betterproto.string_field(13)
columns: Dict[str, "ColumnInfo"] = betterproto.map_field(
14, betterproto.TYPE_STRING, betterproto.TYPE_MESSAGE
)
meta: Dict[str, str] = betterproto.map_field(
15, betterproto.TYPE_STRING, betterproto.TYPE_STRING
)
docs: "Docs" = betterproto.message_field(16)
patch_path: str = betterproto.string_field(17)
deferred: bool = betterproto.bool_field(18)
unrendered_config: Dict[str, str] = betterproto.map_field(
19, betterproto.TYPE_STRING, betterproto.TYPE_STRING
)
relation_name: str = betterproto.string_field(20)
raw_code: str = betterproto.string_field(21)
root_path: str = betterproto.string_field(22)
group: str = betterproto.string_field(23)
@dataclass(eq=False, repr=False)
class SingularTestNode(betterproto.Message):
"""Same as ModelNode w/o access except config is TestConfig"""
name: str = betterproto.string_field(1)
resource_type: str = betterproto.string_field(2)
package_name: str = betterproto.string_field(3)
path: str = betterproto.string_field(4)
original_file_path: str = betterproto.string_field(5)
unique_id: str = betterproto.string_field(6)
fqn: List[str] = betterproto.string_field(7)
database: Optional[str] = betterproto.string_field(8, optional=True, group="_database")
schema: str = betterproto.string_field(9)
alias: str = betterproto.string_field(10)
config: "TestConfig" = betterproto.message_field(11)
tags: List[str] = betterproto.string_field(12)
description: str = betterproto.string_field(13)
columns: Dict[str, "ColumnInfo"] = betterproto.map_field(
14, betterproto.TYPE_STRING, betterproto.TYPE_MESSAGE
)
meta: Dict[str, str] = betterproto.map_field(
15, betterproto.TYPE_STRING, betterproto.TYPE_STRING
)
docs: "Docs" = betterproto.message_field(16)
patch_path: str = betterproto.string_field(17)
deferred: bool = betterproto.bool_field(18)
unrendered_config: Dict[str, str] = betterproto.map_field(
19, betterproto.TYPE_STRING, betterproto.TYPE_STRING
)
relation_name: str = betterproto.string_field(20)
raw_code: str = betterproto.string_field(21)
language: str = betterproto.string_field(22)
refs: List["ListOfStrings"] = betterproto.message_field(23)
sources: List["ListOfStrings"] = betterproto.message_field(24)
metrics: List["ListOfStrings"] = betterproto.message_field(25)
depends_on: "DependsOn" = betterproto.message_field(26)
compiled: bool = betterproto.bool_field(27)
compiled_code: str = betterproto.string_field(28)
contract: bool = betterproto.bool_field(29)
group: str = betterproto.string_field(30)
@dataclass(eq=False, repr=False)
class TestMetadata(betterproto.Message):
name: str = betterproto.string_field(1)
kwargs: Dict[str, str] = betterproto.map_field(
2, betterproto.TYPE_STRING, betterproto.TYPE_STRING
)
namespace: Optional[str] = betterproto.string_field(3, optional=True, group="_namespace")
@dataclass(eq=False, repr=False)
class GenericTestNode(betterproto.Message):
"""
Same as ModelNode w/o access except config is TestConfig, and has
test_metadata and column_name attributes.
"""
name: str = betterproto.string_field(1)
resource_type: str = betterproto.string_field(2)
package_name: str = betterproto.string_field(3)
path: str = betterproto.string_field(4)
original_file_path: str = betterproto.string_field(5)
unique_id: str = betterproto.string_field(6)
fqn: List[str] = betterproto.string_field(7)
database: Optional[str] = betterproto.string_field(8, optional=True, group="_database")
schema: str = betterproto.string_field(9)
alias: str = betterproto.string_field(10)
config: "TestConfig" = betterproto.message_field(11)
tags: List[str] = betterproto.string_field(12)
description: str = betterproto.string_field(13)
columns: Dict[str, "ColumnInfo"] = betterproto.map_field(
14, betterproto.TYPE_STRING, betterproto.TYPE_MESSAGE
)
meta: Dict[str, str] = betterproto.map_field(
15, betterproto.TYPE_STRING, betterproto.TYPE_STRING
)
docs: "Docs" = betterproto.message_field(16)
patch_path: str = betterproto.string_field(17)
deferred: bool = betterproto.bool_field(18)
unrendered_config: Dict[str, str] = betterproto.map_field(
19, betterproto.TYPE_STRING, betterproto.TYPE_STRING
)
relation_name: str = betterproto.string_field(20)
raw_code: str = betterproto.string_field(21)
language: str = betterproto.string_field(22)
refs: List["ListOfStrings"] = betterproto.message_field(23)
sources: List["ListOfStrings"] = betterproto.message_field(24)
metrics: List["ListOfStrings"] = betterproto.message_field(25)
depends_on: "DependsOn" = betterproto.message_field(26)
compiled: bool = betterproto.bool_field(27)
compiled_code: str = betterproto.string_field(28)
contract: bool = betterproto.bool_field(29)
group: str = betterproto.string_field(30)
test_metadata: "TestMetadata" = betterproto.message_field(31)
column_name: Optional[str] = betterproto.string_field(32, optional=True, group="_column_name")
@dataclass(eq=False, repr=False)
class SnapshotNode(betterproto.Message):
"""
SnapshotNode - Sames as ModelNode w/o access except with SnapshotConfig
"""
name: str = betterproto.string_field(1)
resource_type: str = betterproto.string_field(2)
package_name: str = betterproto.string_field(3)
path: str = betterproto.string_field(4)
original_file_path: str = betterproto.string_field(5)
unique_id: str = betterproto.string_field(6)
fqn: List[str] = betterproto.string_field(7)
database: Optional[str] = betterproto.string_field(8, optional=True, group="_database")
schema: str = betterproto.string_field(9)
alias: str = betterproto.string_field(10)
config: "SnapshotConfig" = betterproto.message_field(11)
tags: List[str] = betterproto.string_field(12)
description: str = betterproto.string_field(13)
columns: Dict[str, "ColumnInfo"] = betterproto.map_field(
14, betterproto.TYPE_STRING, betterproto.TYPE_MESSAGE
)
meta: Dict[str, str] = betterproto.map_field(
15, betterproto.TYPE_STRING, betterproto.TYPE_STRING
)
docs: "Docs" = betterproto.message_field(16)
patch_path: str = betterproto.string_field(17)
deferred: bool = betterproto.bool_field(18)
unrendered_config: Dict[str, str] = betterproto.map_field(
19, betterproto.TYPE_STRING, betterproto.TYPE_STRING
)
relation_name: str = betterproto.string_field(20)
raw_code: str = betterproto.string_field(21)
language: str = betterproto.string_field(22)
refs: List["ListOfStrings"] = betterproto.message_field(23)
sources: List["ListOfStrings"] = betterproto.message_field(24)
metrics: List["ListOfStrings"] = betterproto.message_field(25)
depends_on: "DependsOn" = betterproto.message_field(26)
compiled: bool = betterproto.bool_field(27)
compiled_code: str = betterproto.string_field(28)
contract: bool = betterproto.bool_field(29)
group: str = betterproto.string_field(30)
@dataclass(eq=False, repr=False)
class Macro(betterproto.Message):
"""Macro - BaseNode plus additional attributes"""
name: str = betterproto.string_field(1)
resource_type: str = betterproto.string_field(2)
package_name: str = betterproto.string_field(3)
path: str = betterproto.string_field(4)
original_file_path: str = betterproto.string_field(5)
unique_id: str = betterproto.string_field(6)
macro_sql: str = betterproto.string_field(7)
depends_on: "MacroDependsOn" = betterproto.message_field(8)
description: str = betterproto.string_field(9)
meta: Dict[str, str] = betterproto.map_field(
10, betterproto.TYPE_STRING, betterproto.TYPE_STRING
)
docs: "Docs" = betterproto.message_field(11)
patch_path: Optional[str] = betterproto.string_field(12, optional=True, group="_patch_path")
arguments: List["MacroArgument"] = betterproto.message_field(13)
supported_languages: List[str] = betterproto.string_field(14)
@dataclass(eq=False, repr=False)
class Documentation(betterproto.Message):
"""Documentation - BaseNode plus block_contents"""
name: str = betterproto.string_field(1)
resource_type: str = betterproto.string_field(2)
package_name: str = betterproto.string_field(3)
path: str = betterproto.string_field(4)
original_file_path: str = betterproto.string_field(5)
unique_id: str = betterproto.string_field(6)
block_contents: str = betterproto.string_field(7)
@dataclass(eq=False, repr=False)
class SourceDefinition(betterproto.Message):
name: str = betterproto.string_field(1)
resource_type: str = betterproto.string_field(2)
package_name: str = betterproto.string_field(3)
path: str = betterproto.string_field(4)
original_file_path: str = betterproto.string_field(5)
unique_id: str = betterproto.string_field(6)
fqn: List[str] = betterproto.string_field(7)
database: Optional[str] = betterproto.string_field(8, optional=True, group="_database")
schema: str = betterproto.string_field(9)
source_name: str = betterproto.string_field(10)
source_description: str = betterproto.string_field(11)
loader: str = betterproto.string_field(12)
identifier: str = betterproto.string_field(13)
quoting: "Quoting" = betterproto.message_field(14)
loaded_at_field: Optional[str] = betterproto.string_field(
15, optional=True, group="_loaded_at_field"
)
freshness: Optional["FreshnessThreshold"] = betterproto.message_field(
16, optional=True, group="_freshness"
)
description: str = betterproto.string_field(18)
"""optional ExternalTable external = 17;"""
columns: Dict[str, "ColumnInfo"] = betterproto.map_field(
19, betterproto.TYPE_STRING, betterproto.TYPE_MESSAGE
)
meta: Dict[str, str] = betterproto.map_field(
20, betterproto.TYPE_STRING, betterproto.TYPE_STRING
)
source_meta: Dict[str, str] = betterproto.map_field(
21, betterproto.TYPE_STRING, betterproto.TYPE_STRING
)
tags: List[str] = betterproto.string_field(22)
config: "SourceConfig" = betterproto.message_field(23)
patch_path: Optional[str] = betterproto.string_field(24, optional=True, group="_patch_path")
unrendered_config: Dict[str, str] = betterproto.map_field(
25, betterproto.TYPE_STRING, betterproto.TYPE_STRING
)
relation_name: Optional[str] = betterproto.string_field(
26, optional=True, group="_relation_name"
)
@dataclass(eq=False, repr=False)
class Exposure(betterproto.Message):
name: str = betterproto.string_field(1)
resource_type: str = betterproto.string_field(2)
package_name: str = betterproto.string_field(3)
path: str = betterproto.string_field(4)
original_file_path: str = betterproto.string_field(5)
unique_id: str = betterproto.string_field(6)
fqn: List[str] = betterproto.string_field(7)
type: str = betterproto.string_field(8)
owner: "ExposureOwner" = betterproto.message_field(9)
description: str = betterproto.string_field(10)
label: Optional[str] = betterproto.string_field(11, optional=True, group="_label")
maturity: Optional[str] = betterproto.string_field(12, optional=True, group="_maturity")
meta: Dict[str, str] = betterproto.map_field(
13, betterproto.TYPE_STRING, betterproto.TYPE_STRING
)
tags: List[str] = betterproto.string_field(14)
config: "ExposureConfig" = betterproto.message_field(15)
unrendered_config: Dict[str, str] = betterproto.map_field(
16, betterproto.TYPE_STRING, betterproto.TYPE_STRING
)
url: Optional[str] = betterproto.string_field(17, optional=True, group="_url")
depends_on: "DependsOn" = betterproto.message_field(18)
refs: List["ListOfStrings"] = betterproto.message_field(24)
sources: List["ListOfStrings"] = betterproto.message_field(25)
metrics: List["ListOfStrings"] = betterproto.message_field(26)
@dataclass(eq=False, repr=False)
class Metric(betterproto.Message):
name: str = betterproto.string_field(1)
resource_type: str = betterproto.string_field(2)
package_name: str = betterproto.string_field(3)
path: str = betterproto.string_field(4)
original_file_path: str = betterproto.string_field(5)
unique_id: str = betterproto.string_field(6)
fqn: List[str] = betterproto.string_field(7)
description: str = betterproto.string_field(8)
label: str = betterproto.string_field(9)
calculation_method: str = betterproto.string_field(10)
expression: str = betterproto.string_field(11)
filters: List["MetricFilter"] = betterproto.message_field(12)
time_grains: List[str] = betterproto.string_field(13)
dimensions: List[str] = betterproto.string_field(14)
timestamp: Optional[str] = betterproto.string_field(15, optional=True, group="_timestamp")
window: Optional["MetricTime"] = betterproto.message_field(16, optional=True, group="_window")
model: Optional[str] = betterproto.string_field(17, optional=True, group="_model")
model_unique_id: Optional[str] = betterproto.string_field(
18, optional=True, group="_model_unique_id"
)
meta: Dict[str, str] = betterproto.map_field(
19, betterproto.TYPE_STRING, betterproto.TYPE_STRING
)
tags: List[str] = betterproto.string_field(20)
config: "MetricConfig" = betterproto.message_field(21)
unrendered_config: Dict[str, str] = betterproto.map_field(
22, betterproto.TYPE_STRING, betterproto.TYPE_STRING
)
refs: List["ListOfStrings"] = betterproto.message_field(24)
sources: List["ListOfStrings"] = betterproto.message_field(25)
metrics: List["ListOfStrings"] = betterproto.message_field(26)
depends_on: "DependsOn" = betterproto.message_field(27)
group: str = betterproto.string_field(28)

View File

@@ -8,6 +8,7 @@ from dbt.contracts.util import (
Replaceable,
rename_metric_attr,
)
from dbt.contracts.graph import proto_nodes
# trigger the PathEncoder
import dbt.helper_types # noqa:F401
@@ -86,6 +87,12 @@ class Docs(dbtClassMixin, Replaceable):
show: bool = True
node_color: Optional[str] = None
def to_msg(self):
return proto_nodes.Docs(
show=self.show,
node_color=self.node_color,
)
@dataclass
class HasDocs(AdditionalPropertiesMixin, ExtensibleDbtClassMixin, Replaceable):
@@ -160,6 +167,11 @@ class MacroArgument(dbtClassMixin):
type: Optional[str] = None
description: str = ""
def to_msg(self):
return proto_nodes.MacroArgument(
name=self.name, type=self.type, description=self.description
)
@dataclass
class UnparsedMacroUpdate(HasConfig, HasDocs, HasYamlMetadata):
@@ -180,6 +192,12 @@ class Time(dbtClassMixin, Mergeable):
count: Optional[int] = None
period: Optional[TimePeriod] = None
def to_msg(self):
return proto_nodes.Time(
count=self.count,
period=(None if not self.period else self.period.value),
)
def exceeded(self, actual_age: float) -> bool:
if self.period is None or self.count is None:
return False
@@ -197,6 +215,14 @@ class FreshnessThreshold(dbtClassMixin, Mergeable):
error_after: Optional[Time] = field(default_factory=Time)
filter: Optional[str] = None
def to_msg(self):
msg = proto_nodes.FreshnessThreshold(filter=self.filter)
if self.warn_after:
msg.warn_after = self.warn_after.to_msg()
if self.error_after:
msg.error_after = self.error_after.to_msg()
return msg
def status(self, age: float) -> "dbt.contracts.results.FreshnessStatus":
from dbt.contracts.results import FreshnessStatus
@@ -247,6 +273,14 @@ class Quoting(dbtClassMixin, Mergeable):
identifier: Optional[bool] = None
column: Optional[bool] = None
def to_msg(self):
return proto_nodes.Quoting(
database=self.database,
schema=self.schema,
identifier=self.identifier,
column=self.column,
)
@dataclass
class UnparsedSourceTableDefinition(HasColumnTests, HasTests):
@@ -431,6 +465,12 @@ class Owner(AdditionalPropertiesAllowed, Replaceable):
email: Optional[str] = None
name: Optional[str] = None
def to_msg(self):
return proto_nodes.Owner(
email=self.email,
name=self.name,
)
@dataclass
class UnparsedExposure(dbtClassMixin, Replaceable):
@@ -465,6 +505,9 @@ class MetricFilter(dbtClassMixin, Replaceable):
# TODO : Can we make this Any?
value: str
def to_msg(self):
return proto_nodes.MetricFilter(field=self.field, operator=self.operator, value=self.value)
class MetricTimePeriod(StrEnum):
day = "day"
@@ -481,6 +524,11 @@ class MetricTime(dbtClassMixin, Mergeable):
count: Optional[int] = None
period: Optional[MetricTimePeriod] = None
def to_msg(self):
return proto_nodes.MetricTime(
count=self.count, period=(self.period.value if self.period else None)
)
def __bool__(self):
return self.count is not None and self.period is not None

View File

@@ -1,4 +1,23 @@
import re
import enum
from dbt.utils import cast_dict_to_dict_of_strings
def get_msg_attribute_value(obj, attribute):
orig_value = getattr(obj, attribute)
value = orig_value
if isinstance(orig_value, enum.Enum):
value = orig_value.value
elif hasattr(value, "to_msg"):
value = value.to_msg()
elif attribute == "columns":
value = {}
for k, v in orig_value:
value[k] = orig_value.to_msg()
elif isinstance(orig_value, dict):
value = cast_dict_to_dict_of_strings(value)
return value
HTML_COLORS = [
"aliceblue",

Binary file not shown.

View File

@@ -51,3 +51,10 @@ logger = AdapterLogger("<database name>")
## Compiling types.proto
After adding a new message in types.proto, in the core/dbt/events directory: ```protoc --python_betterproto_out . types.proto```
After switching to the 2.0.0b5 release of betterproto, there is now a bug where it generates incorrectly named python classes when the names have acronyms like SQL or YAML in them. For now, I'm renaming these. The bug has been fixed in the repo, so hopefully there will be a new release at some point. (SQL, YAML, GET get turned int Sql, Yaml, Get).
A search and replace s/Sql/SQL/g and s/Yaml/YAML/g, GetRequest/GETRequest, GetResponse/GETResponse should clean it up.
In addition it now changes "@dataclass" to "@dataclass(eq=False, repr=False)"
In addition, betterproto now puts the generated file in proto_types/__init__.py. I'm moving it to proto_types.py

File diff suppressed because it is too large Load Diff

View File

@@ -1533,10 +1533,8 @@ message LogFreshnessResultMsg {
LogFreshnessResult data = 2;
}
// Skipped Q019, Q020, Q021
// Q022
message LogCancelLine {
string conn_name = 1;

View File

@@ -1044,7 +1044,7 @@ class WrongResourceSchemaFile(WarnLevel, pt.WrongResourceSchemaFile):
@dataclass
class NoNodeForYamlKey(WarnLevel, pt.NoNodeForYamlKey):
class NoNodeForYAMLKey(WarnLevel, pt.NoNodeForYAMLKey):
def code(self):
return "I058"

View File

@@ -49,6 +49,7 @@ from dbt.context.providers import ParseProvider
from dbt.contracts.files import FileHash, ParseFileType, SchemaSourceFile
from dbt.parser.read_files import read_files, load_source_file
from dbt.parser.partial import PartialParsing, special_override_macros
from dbt.constants import MANIFEST_FILE_NAME
from dbt.contracts.graph.manifest import (
Manifest,
Disabled,
@@ -85,7 +86,6 @@ from dbt.version import __version__
from dbt.dataclass_schema import StrEnum, dbtClassMixin
MANIFEST_FILE_NAME = "manifest.json"
PARTIAL_PARSE_FILE_NAME = "partial_parse.msgpack"
PARSING_STATE = DbtProcessState("parsing")
PERF_INFO_FILE_NAME = "perf_info.json"
@@ -411,6 +411,9 @@ class ManifestLoader:
# write out the fully parsed manifest
self.write_manifest_for_partial_parse()
# write out manifest.json
parsed_manifest_path = os.path.join(self.root_project.target_path, MANIFEST_FILE_NAME)
self.manifest.write(parsed_manifest_path)
return self.manifest
@@ -586,7 +589,8 @@ class ManifestLoader:
)
fire_event(
Note(
msg=f"previous checksum: {self.manifest.state_check.vars_hash.checksum}, current checksum: {manifest.state_check.vars_hash.checksum}"
msg=f"previous checksum: {manifest.state_check.vars_hash.checksum}, "
f"current checksum: {self.manifest.state_check.vars_hash.checksum}"
),
level=EventLevel.DEBUG,
)

View File

@@ -71,7 +71,7 @@ from dbt.exceptions import (
YamlParseListError,
)
from dbt.events.functions import warn_or_error
from dbt.events.types import WrongResourceSchemaFile, NoNodeForYamlKey, MacroNotFoundForPatch
from dbt.events.types import WrongResourceSchemaFile, NoNodeForYAMLKey, MacroNotFoundForPatch
from dbt.node_types import NodeType
from dbt.parser.base import SimpleParser
from dbt.parser.search import FileBlock
@@ -935,7 +935,7 @@ class NodePatchParser(NonSourceParser[NodeTarget, ParsedNodePatch], Generic[Node
node.patch(patch)
else:
warn_or_error(
NoNodeForYamlKey(
NoNodeForYAMLKey(
patch_name=patch.name,
yaml_key=patch.yaml_key,
file_path=source_file.path.original_file_path,

View File

View File

@@ -678,3 +678,10 @@ def cast_dict_to_dict_of_strings(dct):
for k, v in dct.items():
new_dct[str(k)] = str(v)
return new_dct
def cast_to_bool(bool_value: Optional[bool]) -> bool:
if bool_value is None:
return False
else:
return True

View File

@@ -48,7 +48,7 @@ setup(
install_requires=[
"Jinja2==3.1.2",
"agate>=1.6,<1.7.1",
"betterproto==1.2.5",
"betterproto==2.0.0b5",
"click>=7.0,<9",
"colorama>=0.3.9,<0.4.7",
"hologram>=0.0.14,<=0.0.15",

View File

@@ -1,4 +1,4 @@
betterproto[compiler]==1.2.5
betterproto[compiler]==2.0.0b5
black==22.10.0
bumpversion
docutils

View File

@@ -1,9 +1,18 @@
import os
import pytest
from dbt.tests.util import run_dbt
from dbt.tests.util import run_dbt, get_manifest
class TestGenerate:
def test_generate_no_manifest_on_no_compile(self, project):
@pytest.fixture(scope="class")
def models(self):
return {"my_model.sql": "select 1 as fun"}
def test_manifest_not_compiled(self, project):
run_dbt(["docs", "generate", "--no-compile"])
assert not os.path.exists("./target/manifest.json")
# manifest.json is written out in parsing now, but it
# shouldn't be compiled because of the --no-compile flag
manifest = get_manifest(project.project_root)
model_id = "model.test.my_model"
assert model_id in manifest.nodes
assert manifest.nodes[model_id].compiled is False

View File

@@ -204,7 +204,7 @@ sample_values = [
types.WrongResourceSchemaFile(
patch_name="", resource_type="", file_path="", plural_resource_type=""
),
types.NoNodeForYamlKey(patch_name="", yaml_key="", file_path=""),
types.NoNodeForYAMLKey(patch_name="", yaml_key="", file_path=""),
types.MacroNotFoundForPatch(patch_name=""),
types.NodeNotFoundOrDisabled(
original_file_path="",

View File

@@ -0,0 +1,382 @@
from dbt.contracts.graph.nodes import (
ModelNode,
AnalysisNode,
RPCNode,
SqlNode,
HookNode,
SeedNode,
SingularTestNode,
GenericTestNode,
TestMetadata,
SnapshotNode,
Macro,
Documentation,
SourceDefinition,
Exposure,
Metric,
)
from dbt.contracts.graph.model_config import (
NodeConfig,
SeedConfig,
TestConfig,
SnapshotConfig,
SourceConfig,
ExposureConfig,
MetricConfig,
)
from dbt.contracts.graph.unparsed import Owner, MetricFilter
from dbt.node_types import NodeType, ModelLanguage
from dbt.contracts.files import FileHash
def test_nodes():
# Create NodeConfig for use in the next 5 nodes
node_config = NodeConfig(
enabled=True,
alias=None,
schema="my_schema",
database="my_database",
tags=["one", "two", "three"],
meta={"one": 1, "two": 2},
materialized="table",
incremental_strategy=None,
full_refresh=None,
on_schema_change="ignore",
packages=["one", "two", "three"],
)
# Create a dummy ModelNode
model_node = ModelNode(
database="testdb",
schema="testschema",
fqn=["my", "test"],
unique_id="test.model.my_node",
raw_code="select 1 from fun",
language="sql",
package_name="test",
path="my_node.sql",
original_file_path="models/my_node.sql",
name="my_node",
resource_type=NodeType.Model,
alias="my_node",
checksum=FileHash.from_contents("select 1 from fun"),
config=node_config,
group="analytics",
)
assert model_node
# Get a matching proto message
proto_model_msg = model_node.to_msg()
assert proto_model_msg
assert proto_model_msg.group == "analytics"
# Create a dummy AnalysisNode
analysis_node = AnalysisNode(
database="testdb",
schema="testschema",
fqn=["my", "test"],
unique_id="test.model.my_node",
raw_code="select 1 from fun",
language="sql",
package_name="test",
path="my_node.sql",
original_file_path="models/my_node.sql",
name="my_node",
resource_type=NodeType.Analysis,
alias="my_node",
checksum=FileHash.from_contents("select 1 from fun"),
config=node_config,
)
assert analysis_node
# Get a matching proto message
proto_analysis_msg = analysis_node.to_msg()
assert proto_analysis_msg
# Create a dummy RPCNode
rpc_node = RPCNode(
database="testdb",
schema="testschema",
fqn=["my", "test"],
unique_id="test.model.my_node",
raw_code="select 1 from fun",
language="sql",
package_name="test",
path="my_node.sql",
original_file_path="models/my_node.sql",
name="my_node",
resource_type=NodeType.RPCCall,
alias="my_node",
checksum=FileHash.from_contents("select 1 from fun"),
config=node_config,
)
assert rpc_node
# Get a matching proto message
proto_rpc_msg = rpc_node.to_msg()
assert proto_rpc_msg
# Create a dummy SqlNode
sql_node = SqlNode(
database="testdb",
schema="testschema",
fqn=["my", "test"],
unique_id="test.model.my_node",
raw_code="select 1 from fun",
language="sql",
package_name="test",
path="my_node.sql",
original_file_path="models/my_node.sql",
name="my_node",
resource_type=NodeType.SqlOperation,
alias="my_node",
checksum=FileHash.from_contents("select 1 from fun"),
config=node_config,
)
assert sql_node
# Get a matching proto message
proto_sql_msg = sql_node.to_msg()
assert proto_sql_msg
# Create a dummy HookNode
hook_node = HookNode(
database="testdb",
schema="testschema",
fqn=["my", "test"],
unique_id="hook.test.my_hook",
raw_code="select 1 from fun",
language="sql",
package_name="test",
path="my_node.sql",
original_file_path="models/my_node.sql",
name="my_hook",
resource_type=NodeType.Operation,
alias="my_hook",
checksum=FileHash.from_contents("select 1 from fun"),
config=node_config,
index=1,
)
assert hook_node
# Get a matching proto message
proto_hook_msg = hook_node.to_msg()
assert proto_hook_msg
assert proto_hook_msg.index
# Create a dummy SeedNode
seed_config = SeedConfig(
enabled=True,
alias=None,
schema="my_schema",
database="my_database",
tags=["one", "two", "three"],
meta={"one": 1, "two": 2},
)
seed_node = SeedNode(
database="testdb",
schema="testschema",
fqn=["my", "test"],
unique_id="test.seed.my_node",
raw_code="",
package_name="test",
path="seed.csv",
original_file_path="seeds/seed.csv",
name="seed",
resource_type=NodeType.Seed,
alias="seed",
checksum=FileHash.from_contents("test"),
root_path="some_path",
config=seed_config,
)
assert seed_node
# Get a matching proto message
proto_seed_msg = seed_node.to_msg()
assert proto_seed_msg
assert proto_seed_msg.root_path
# config for SingularTestNode and GenericTestNode
test_config = TestConfig(
enabled=True,
alias=None,
schema="my_schema",
database="my_database",
tags=["one", "two", "three"],
meta={"one": 1, "two": 2},
)
# Create a dummy SingularTestNode
singular_test_node = SingularTestNode(
database="testdb",
schema="testschema",
fqn=["my", "test"],
unique_id="test.model.my_node",
raw_code="select 1 from fun",
package_name="test",
path="my_node.sql",
original_file_path="models/my_node.sql",
name="my_node",
resource_type=NodeType.Test,
alias="my_node",
checksum=FileHash.from_contents("select 1 from fun"),
config=test_config,
)
assert singular_test_node
# Get a matching proto message
proto_singular_test_msg = singular_test_node.to_msg()
assert proto_singular_test_msg
# Create a dummy GenericTestNode
test_metadata = TestMetadata(
name="my_test",
kwargs={"one": 1, "two": "another"},
)
generic_test_node = GenericTestNode(
database="testdb",
schema="testschema",
fqn=["my", "test"],
unique_id="test.model.my_node",
raw_code="select 1 from fun",
package_name="test",
path="my_node.sql",
original_file_path="models/my_node.sql",
name="my_node",
resource_type=NodeType.Test,
alias="my_node",
checksum=FileHash.from_contents("select 1 from fun"),
config=test_config,
test_metadata=test_metadata,
column_name="some_column",
)
assert generic_test_node
# Get a matching proto message
proto_generic_test_msg = generic_test_node.to_msg()
assert proto_generic_test_msg
assert proto_generic_test_msg.column_name
# Create SnapshotConfig and SnapshotNode
snapshot_config = SnapshotConfig(
enabled=True,
alias=None,
schema="my_schema",
database="my_database",
tags=["one", "two", "three"],
meta={"one": 1, "two": 2},
materialized="table",
incremental_strategy=None,
full_refresh=None,
on_schema_change="ignore",
packages=["one", "two", "three"],
strategy="check",
target_schema="some_schema",
target_database="some_database",
check_cols="id",
)
snapshot_node = SnapshotNode(
database="testdb",
schema="testschema",
fqn=["my", "test"],
unique_id="test.model.my_test",
raw_code="select 1 from fun",
language="sql",
package_name="my_project",
path="my_node.sql",
original_file_path="models/my_node.sql",
name="my_test",
resource_type=NodeType.Snapshot,
alias="my_node",
checksum=FileHash.from_contents("select 1 from fun"),
config=snapshot_config,
)
assert snapshot_node
# Get a matching proto message
proto_snapshot_msg = snapshot_node.to_msg()
assert proto_snapshot_msg
assert proto_snapshot_msg.config.target_schema
# Create a dummy Macro
macro = Macro(
name="my_macro",
resource_type=NodeType.Macro,
package_name="my_project",
path="my_macro.sql",
original_file_path="macros/my_macro.sql",
unique_id="macro.my_project.my_macro",
macro_sql="{{ }}",
description="my macro",
supported_languages=[ModelLanguage.sql],
)
proto_macro_msg = macro.to_msg()
assert proto_macro_msg
assert proto_macro_msg.supported_languages == ["sql"]
# Create a dummy Documentation
doc = Documentation(
name="my_doc",
resource_type=NodeType.Macro,
package_name="my_project",
path="readme.md",
original_file_path="models/readme.md",
unique_id="doc.my_project.my_doc",
block_contents="this is my special doc",
)
proto_doc_msg = doc.to_msg()
assert proto_doc_msg
assert proto_doc_msg.block_contents
# Dummy SourceDefinition
source = SourceDefinition(
name="my_source",
resource_type=NodeType.Source,
package_name="my_project",
path="source.yml",
original_file_path="source.yml",
unique_id="source.my_project.my_source",
fqn=["sources", "my_source"],
database="my_database",
schema="my_schema",
source_name="my_source",
source_description="my source",
loader="loader",
identifier="my_source",
config=SourceConfig(enabled=True),
)
proto_source_msg = source.to_msg()
assert proto_source_msg
assert proto_source_msg.source_name
# Dummy Exposure
exposure = Exposure(
name="my_exposure",
resource_type=NodeType.Exposure,
package_name="my_project",
path="exposure.yml",
original_file_path="exposure.yml",
unique_id="exposure.my_project.my_exposure",
fqn=["my", "exposure"],
config=ExposureConfig(enabled=True),
type="dashboard",
owner=Owner(email="someone@somewhere"),
description="my exposure",
)
proto_exposure_msg = exposure.to_msg()
assert proto_exposure_msg
assert proto_exposure_msg.type
# Dummy Metric
metric = Metric(
name="my_metric",
resource_type=NodeType.Metric,
package_name="my_project",
path="metrics.yml",
original_file_path="metrics.yml",
unique_id="metric.my_project.my_metric",
fqn=["my", "metric"],
config=MetricConfig(enabled=True),
description="my metric",
label="my label",
calculation_method="*",
expression="select 1 as fun",
filters=[MetricFilter(field="sum", operator="sum", value="sum")],
time_grains=["day", "minute"],
dimensions=["day", "minute"],
group="analytics",
)
proto_metric_msg = metric.to_msg()
assert proto_metric_msg
assert proto_metric_msg.label
assert proto_metric_msg.group == "analytics"