Merge remote-tracking branch 'origin/releases/0.19.1rc2' into 0.19.latest

Fix compiled sql for ephemeral models

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.19.1b2
+current_version = 0.19.1rc2
 parse = (?P<major>\d+)
 	\.(?P<minor>\d+)
 	\.(?P<patch>\d+)

CHANGELOG.md

@@ -1,11 +1,29 @@
 ## dbt 0.19.1 (Release TBD)
 
+## dbt 0.19.1rc2 (March 25, 2021)
+
+
+### Fixes
+- Pass service-account scopes to gcloud-based oauth ([#3040](https://github.com/fishtown-analytics/dbt/issues/3040), [#3041](https://github.com/fishtown-analytics/dbt/pull/3041))
+
+Contributors:
+- [@yu-iskw](https://github.com/yu-iskw) ([#3041](https://github.com/fishtown-analytics/dbt/pull/3041))
+
+## dbt 0.19.1rc1 (March 15, 2021)
+
+### Under the hood
+- Update code to use Mashumaro 2.0 ([#3138](https://github.com/fishtown-analytics/dbt/pull/3138))
+- Pin `agate<1.6.2` to avoid installation errors relating to its new dependency `PyICU` ([#3160](https://github.com/fishtown-analytics/dbt/issues/3160), [#3161](https://github.com/fishtown-analytics/dbt/pull/3161))
+- Add an event to track resource counts ([#3050](https://github.com/fishtown-analytics/dbt/issues/3050), [#3157](https://github.com/fishtown-analytics/dbt/pull/3157))
+
+### Fixes
+
+- Fix compiled sql for ephemeral models ([#3139](https://github.com/fishtown-analytics/dbt/pull/3139), [#3056](https://github.com/fishtown-analytics/dbt/pull/3056))
+
 ## dbt 0.19.1b2 (February 15, 2021)
 
-
 ## dbt 0.19.1b1 (February 12, 2021)
 
-
 ### Fixes
 
 - On BigQuery, fix regressions for `insert_overwrite` incremental strategy with `int64` and `timestamp` partition columns ([#3063](https://github.com/fishtown-analytics/dbt/issues/3063), [#3095](https://github.com/fishtown-analytics/dbt/issues/3095), [#3098](https://github.com/fishtown-analytics/dbt/issues/3098))

core/dbt/adapters/base/relation.py

@@ -45,7 +45,7 @@ class BaseRelation(FakeAPIObject, Hashable):
     def __eq__(self, other):
         if not isinstance(other, self.__class__):
             return False
-        return self.to_dict() == other.to_dict()
+        return self.to_dict(omit_none=True) == other.to_dict(omit_none=True)
 
     @classmethod
     def get_default_quote_policy(cls) -> Policy:
@@ -185,10 +185,10 @@ class BaseRelation(FakeAPIObject, Hashable):
     def create_from_source(
         cls: Type[Self], source: ParsedSourceDefinition, **kwargs: Any
     ) -> Self:
-        source_quoting = source.quoting.to_dict()
+        source_quoting = source.quoting.to_dict(omit_none=True)
         source_quoting.pop('column', None)
         quote_policy = deep_merge(
-            cls.get_default_quote_policy().to_dict(),
+            cls.get_default_quote_policy().to_dict(omit_none=True),
             source_quoting,
             kwargs.get('quote_policy', {}),
         )

core/dbt/compilation.py

@@ -30,6 +30,7 @@ from dbt.graph import Graph
 from dbt.logger import GLOBAL_LOGGER as logger
 from dbt.node_types import NodeType
 from dbt.utils import pluralize
+import dbt.tracking
 
 graph_file_name = 'graph.gpickle'
 
@@ -58,6 +59,11 @@ def print_compile_stats(stats):
     results = {k: 0 for k in names.keys()}
     results.update(stats)
 
+    # create tracking event for resource_counts
+    if dbt.tracking.active_user is not None:
+        resource_counts = {k.pluralize(): v for k, v in results.items()}
+        dbt.tracking.track_resource_counts(resource_counts)
+
     stat_line = ", ".join([
         pluralize(ct, names.get(t)) for t, ct in results.items()
         if t in names
@@ -138,7 +144,7 @@ class Linker:
         """
         out_graph = self.graph.copy()
         for node_id in self.graph.nodes():
-            data = manifest.expect(node_id).to_dict()
+            data = manifest.expect(node_id).to_dict(omit_none=True)
             out_graph.add_node(node_id, **data)
         nx.write_gpickle(out_graph, outfile)
 
@@ -248,19 +254,19 @@ class Compiler:
     def _get_dbt_test_name(self) -> str:
         return 'dbt__CTE__INTERNAL_test'
 
-    # This method is called by the 'compile_node' method. Starting
-    # from the node that it is passed in, it will recursively call
-    # itself using the 'extra_ctes'.  The 'ephemeral' models do
-    # not produce SQL that is executed directly, instead they
-    # are rolled up into the models that refer to them by
-    # inserting CTEs into the SQL.
     def _recursively_prepend_ctes(
         self,
         model: NonSourceCompiledNode,
         manifest: Manifest,
         extra_context: Optional[Dict[str, Any]],
     ) -> Tuple[NonSourceCompiledNode, List[InjectedCTE]]:
-
+        """This method is called by the 'compile_node' method. Starting
+        from the node that it is passed in, it will recursively call
+        itself using the 'extra_ctes'.  The 'ephemeral' models do
+        not produce SQL that is executed directly, instead they
+        are rolled up into the models that refer to them by
+        inserting CTEs into the SQL.
+        """
         if model.compiled_sql is None:
             raise RuntimeException(
                 'Cannot inject ctes into an unparsed node', model
@@ -324,22 +330,22 @@ class Compiler:
                 _extend_prepended_ctes(prepended_ctes, new_prepended_ctes)
 
                 new_cte_name = self.add_ephemeral_prefix(cte_model.name)
-                sql = f' {new_cte_name} as (\n{cte_model.compiled_sql}\n)'
+                rendered_sql = (
+                    cte_model._pre_injected_sql or cte_model.compiled_sql
+                )
+                sql = f' {new_cte_name} as (\n{rendered_sql}\n)'
 
             _add_prepended_cte(prepended_ctes, InjectedCTE(id=cte.id, sql=sql))
 
-        # We don't save injected_sql into compiled sql for ephemeral models
-        # because it will cause problems with processing of subsequent models.
-        # Ephemeral models do not produce executable SQL of their own.
-        if not model.is_ephemeral_model:
-            injected_sql = self._inject_ctes_into_sql(
-                model.compiled_sql,
-                prepended_ctes,
-            )
-            model.compiled_sql = injected_sql
+        injected_sql = self._inject_ctes_into_sql(
+            model.compiled_sql,
+            prepended_ctes,
+        )
+        model._pre_injected_sql = model.compiled_sql
+        model.compiled_sql = injected_sql
         model.extra_ctes_injected = True
         model.extra_ctes = prepended_ctes
-        model.validate(model.to_dict())
+        model.validate(model.to_dict(omit_none=True))
 
         manifest.update_node(model)
 
@@ -388,7 +394,7 @@ class Compiler:
 
         logger.debug("Compiling {}".format(node.unique_id))
 
-        data = node.to_dict()
+        data = node.to_dict(omit_none=True)
         data.update({
             'compiled': False,
             'compiled_sql': None,
@@ -487,11 +493,6 @@ class Compiler:
             )
         return node
 
-    # This is the main entry point into this code. It's called by
-    # CompileRunner.compile, GenericRPCRunner.compile, and
-    # RunTask.get_hook_sql. It calls '_compile_node' to convert
-    # the node into a compiled node, and then calls the
-    # recursive method to "prepend" the ctes.
     def compile_node(
         self,
         node: ManifestNode,
@@ -499,6 +500,12 @@ class Compiler:
         extra_context: Optional[Dict[str, Any]] = None,
         write: bool = True,
     ) -> NonSourceCompiledNode:
+        """This is the main entry point into this code. It's called by
+        CompileRunner.compile, GenericRPCRunner.compile, and
+        RunTask.get_hook_sql. It calls '_compile_node' to convert
+        the node into a compiled node, and then calls the
+        recursive method to "prepend" the ctes.
+        """
         node = self._compile_node(node, manifest, extra_context)
 
         node, _ = self._recursively_prepend_ctes(

core/dbt/config/profile.py

@@ -111,8 +111,8 @@ class Profile(HasCredentials):
             'credentials': self.credentials,
         }
         if serialize_credentials:
-            result['config'] = self.config.to_dict()
-            result['credentials'] = self.credentials.to_dict()
+            result['config'] = self.config.to_dict(omit_none=True)
+            result['credentials'] = self.credentials.to_dict(omit_none=True)
         return result
 
     def to_target_dict(self) -> Dict[str, Any]:
@@ -125,7 +125,7 @@ class Profile(HasCredentials):
             'name': self.target_name,
             'target_name': self.target_name,
             'profile_name': self.profile_name,
-            'config': self.config.to_dict(),
+            'config': self.config.to_dict(omit_none=True),
         })
         return target
 
@@ -138,7 +138,7 @@ class Profile(HasCredentials):
     def validate(self):
         try:
             if self.credentials:
-                dct = self.credentials.to_dict()
+                dct = self.credentials.to_dict(omit_none=True)
                 self.credentials.validate(dct)
             dct = self.to_profile_info(serialize_credentials=True)
             ProfileConfig.validate(dct)

core/dbt/config/project.py

@@ -347,7 +347,7 @@ class PartialProject(RenderComponents):
         # break many things
         quoting: Dict[str, Any] = {}
         if cfg.quoting is not None:
-            quoting = cfg.quoting.to_dict()
+            quoting = cfg.quoting.to_dict(omit_none=True)
 
         models: Dict[str, Any]
         seeds: Dict[str, Any]
@@ -578,10 +578,11 @@ class Project:
             'config-version': self.config_version,
         })
         if self.query_comment:
-            result['query-comment'] = self.query_comment.to_dict()
+            result['query-comment'] = \
+                self.query_comment.to_dict(omit_none=True)
 
         if with_packages:
-            result.update(self.packages.to_dict())
+            result.update(self.packages.to_dict(omit_none=True))
 
         return result
 

core/dbt/config/runtime.py

@@ -78,7 +78,7 @@ class RuntimeConfig(Project, Profile, AdapterRequiredConfig):
             get_relation_class_by_name(profile.credentials.type)
             .get_default_quote_policy()
             .replace_dict(_project_quoting_dict(project, profile))
-        ).to_dict()
+        ).to_dict(omit_none=True)
 
         cli_vars: Dict[str, Any] = parse_cli_vars(getattr(args, 'vars', '{}'))
 
@@ -391,7 +391,7 @@ class UnsetConfig(UserConfig):
                 f"'UnsetConfig' object has no attribute {name}"
             )
 
-    def __post_serialize__(self, dct, options=None):
+    def __post_serialize__(self, dct):
         return {}
 
 

core/dbt/context/base.py

@@ -538,4 +538,5 @@ class BaseContext(metaclass=ContextMeta):
 
 def generate_base_context(cli_vars: Dict[str, Any]) -> Dict[str, Any]:
     ctx = BaseContext(cli_vars)
+    # This is not a Mashumaro to_dict call
     return ctx.to_dict()

core/dbt/context/context_config.py

@@ -196,7 +196,7 @@ class ContextConfigGenerator(BaseContextConfigGenerator[C]):
             base=base,
         )
         finalized = config.finalize_and_validate()
-        return finalized.to_dict()
+        return finalized.to_dict(omit_none=True)
 
 
 class UnrenderedConfigGenerator(BaseContextConfigGenerator[Dict[str, Any]]):

core/dbt/context/docs.py

@@ -77,4 +77,5 @@ def generate_runtime_docs(
     current_project: str,
 ) -> Dict[str, Any]:
     ctx = DocsRuntimeContext(config, target, manifest, current_project)
+    # This is not a Mashumaro to_dict call
     return ctx.to_dict()

core/dbt/context/providers.py

@@ -1115,7 +1115,7 @@ class ProviderContext(ManifestContext):
 
     @contextproperty('model')
     def ctx_model(self) -> Dict[str, Any]:
-        return self.model.to_dict()
+        return self.model.to_dict(omit_none=True)
 
     @contextproperty
     def pre_hooks(self) -> Optional[List[Dict[str, Any]]]:
@@ -1231,7 +1231,7 @@ class ModelContext(ProviderContext):
         if isinstance(self.model, ParsedSourceDefinition):
             return []
         return [
-            h.to_dict() for h in self.model.config.pre_hook
+            h.to_dict(omit_none=True) for h in self.model.config.pre_hook
         ]
 
     @contextproperty
@@ -1239,7 +1239,7 @@ class ModelContext(ProviderContext):
         if isinstance(self.model, ParsedSourceDefinition):
             return []
         return [
-            h.to_dict() for h in self.model.config.post_hook
+            h.to_dict(omit_none=True) for h in self.model.config.post_hook
         ]
 
     @contextproperty

core/dbt/contracts/connection.py

@@ -132,7 +132,7 @@ class Credentials(
     ) -> Iterable[Tuple[str, Any]]:
         """Return an ordered iterator of key/value pairs for pretty-printing.
         """
-        as_dict = self.to_dict(options={'keep_none': True})
+        as_dict = self.to_dict(omit_none=False)
         connection_keys = set(self._connection_keys())
         aliases: List[str] = []
         if with_aliases:
@@ -148,8 +148,8 @@ class Credentials(
         raise NotImplementedError
 
     @classmethod
-    def __pre_deserialize__(cls, data, options=None):
-        data = super().__pre_deserialize__(data, options=options)
+    def __pre_deserialize__(cls, data):
+        data = super().__pre_deserialize__(data)
         data = cls.translate_aliases(data)
         return data
 
@@ -159,7 +159,7 @@ class Credentials(
     ) -> Dict[str, Any]:
         return translate_aliases(kwargs, cls._ALIASES, recurse)
 
-    def __post_serialize__(self, dct, options=None):
+    def __post_serialize__(self, dct):
         # no super() -- do we need it?
         if self._ALIASES:
             dct.update({

core/dbt/contracts/graph/compiled.py

@@ -43,6 +43,7 @@ class CompiledNode(ParsedNode, CompiledNodeMixin):
     extra_ctes_injected: bool = False
     extra_ctes: List[InjectedCTE] = field(default_factory=list)
     relation_name: Optional[str] = None
+    _pre_injected_sql: Optional[str] = None
 
     def set_cte(self, cte_id: str, sql: str):
         """This is the equivalent of what self.extra_ctes[cte_id] = sql would
@@ -55,6 +56,12 @@ class CompiledNode(ParsedNode, CompiledNodeMixin):
         else:
             self.extra_ctes.append(InjectedCTE(id=cte_id, sql=sql))
 
+    def __post_serialize__(self, dct):
+        dct = super().__post_serialize__(dct)
+        if '_pre_injected_sql' in dct:
+            del dct['_pre_injected_sql']
+        return dct
+
 
 @dataclass
 class CompiledAnalysisNode(CompiledNode):
@@ -178,7 +185,7 @@ def parsed_instance_for(compiled: CompiledNode) -> ParsedResource:
         raise ValueError('invalid resource_type: {}'
                          .format(compiled.resource_type))
 
-    return cls.from_dict(compiled.to_dict())
+    return cls.from_dict(compiled.to_dict(omit_none=True))
 
 
 NonSourceCompiledNode = Union[

core/dbt/contracts/graph/manifest.py

@@ -240,7 +240,7 @@ def build_edges(nodes: List[ManifestNode]):
 
 
 def _deepcopy(value):
-    return value.from_dict(value.to_dict())
+    return value.from_dict(value.to_dict(omit_none=True))
 
 
 class Locality(enum.IntEnum):
@@ -564,11 +564,11 @@ class Manifest(MacroMethods):
         """
         self.flat_graph = {
             'nodes': {
-                k: v.to_dict(options={'keep_none': True})
+                k: v.to_dict(omit_none=False)
                 for k, v in self.nodes.items()
             },
             'sources': {
-                k: v.to_dict(options={'keep_none': True})
+                k: v.to_dict(omit_none=False)
                 for k, v in self.sources.items()
             }
         }
@@ -755,7 +755,7 @@ class Manifest(MacroMethods):
 
     # When 'to_dict' is called on the Manifest, it substitues a
     # WritableManifest
-    def __pre_serialize__(self, options=None):
+    def __pre_serialize__(self):
         return self.writable_manifest()
 
     def write(self, path):

core/dbt/contracts/graph/model_config.py

@@ -307,7 +307,7 @@ class BaseConfig(
         """
         # sadly, this is a circular import
         from dbt.adapters.factory import get_config_class_by_name
-        dct = self.to_dict(options={'keep_none': True})
+        dct = self.to_dict(omit_none=False)
 
         adapter_config_cls = get_config_class_by_name(adapter_type)
 
@@ -326,12 +326,12 @@ class BaseConfig(
         return self.from_dict(dct)
 
     def finalize_and_validate(self: T) -> T:
-        dct = self.to_dict(options={'keep_none': True})
+        dct = self.to_dict(omit_none=False)
         self.validate(dct)
         return self.from_dict(dct)
 
     def replace(self, **kwargs):
-        dct = self.to_dict()
+        dct = self.to_dict(omit_none=True)
 
         mapping = self.field_mapping()
         for key, value in kwargs.items():
@@ -396,8 +396,8 @@ class NodeConfig(BaseConfig):
     full_refresh: Optional[bool] = None
 
     @classmethod
-    def __pre_deserialize__(cls, data, options=None):
-        data = super().__pre_deserialize__(data, options=options)
+    def __pre_deserialize__(cls, data):
+        data = super().__pre_deserialize__(data)
         field_map = {'post-hook': 'post_hook', 'pre-hook': 'pre_hook'}
         # create a new dict because otherwise it gets overwritten in
         # tests
@@ -414,8 +414,8 @@ class NodeConfig(BaseConfig):
                 data[new_name] = data.pop(field_name)
         return data
 
-    def __post_serialize__(self, dct, options=None):
-        dct = super().__post_serialize__(dct, options=options)
+    def __post_serialize__(self, dct):
+        dct = super().__post_serialize__(dct)
         field_map = {'post_hook': 'post-hook', 'pre_hook': 'pre-hook'}
         for field_name in field_map:
             if field_name in dct:
@@ -480,7 +480,7 @@ class SnapshotConfig(EmptySnapshotConfig):
         # formerly supported with GenericSnapshotConfig
 
     def finalize_and_validate(self):
-        data = self.to_dict()
+        data = self.to_dict(omit_none=True)
         self.validate(data)
         return self.from_dict(data)
 

core/dbt/contracts/graph/parsed.py

@@ -99,8 +99,8 @@ class HasRelationMetadata(dbtClassMixin, Replaceable):
     # because it messes up the subclasses and default parameters
     # so hack it here
     @classmethod
-    def __pre_deserialize__(cls, data, options=None):
-        data = super().__pre_deserialize__(data, options=options)
+    def __pre_deserialize__(cls, data):
+        data = super().__pre_deserialize__(data)
         if 'database' not in data:
             data['database'] = None
         return data
@@ -141,7 +141,7 @@ class ParsedNodeMixins(dbtClassMixin):
             # Maybe there should be validation or restrictions
             # elsewhere?
             assert isinstance(self, dbtClassMixin)
-            dct = self.to_dict(options={'keep_none': True})
+            dct = self.to_dict(omit_none=False)
             self.validate(dct)
 
     def get_materialization(self):
@@ -454,7 +454,7 @@ class ParsedMacro(UnparsedBaseNode, HasUniqueID):
         if flags.STRICT_MODE:
             # What does this actually validate?
             assert isinstance(self, dbtClassMixin)
-            dct = self.to_dict(options={'keep_none': True})
+            dct = self.to_dict(omit_none=False)
             self.validate(dct)
 
     def same_contents(self, other: Optional['ParsedMacro']) -> bool:

core/dbt/contracts/graph/unparsed.py

@@ -231,12 +231,9 @@ class UnparsedSourceTableDefinition(HasColumnTests, HasTests):
     external: Optional[ExternalTable] = None
     tags: List[str] = field(default_factory=list)
 
-    def __post_serialize__(self, dct, options=None):
+    def __post_serialize__(self, dct):
         dct = super().__post_serialize__(dct)
-        keep_none = False
-        if options and 'keep_none' in options and options['keep_none']:
-            keep_none = True
-        if not keep_none and self.freshness is None:
+        if 'freshness' not in dct and self.freshness is None:
             dct['freshness'] = None
         return dct
 
@@ -261,12 +258,9 @@ class UnparsedSourceDefinition(dbtClassMixin, Replaceable):
     def yaml_key(self) -> 'str':
         return 'sources'
 
-    def __post_serialize__(self, dct, options=None):
+    def __post_serialize__(self, dct):
         dct = super().__post_serialize__(dct)
-        keep_none = False
-        if options and 'keep_none' in options and options['keep_none']:
-            keep_none = True
-        if not keep_none and self.freshness is None:
+        if 'freshness' not in dct and self.freshness is None:
             dct['freshness'] = None
         return dct
 
@@ -290,7 +284,7 @@ class SourceTablePatch(dbtClassMixin):
     columns: Optional[Sequence[UnparsedColumn]] = None
 
     def to_patch_dict(self) -> Dict[str, Any]:
-        dct = self.to_dict()
+        dct = self.to_dict(omit_none=True)
         remove_keys = ('name')
         for key in remove_keys:
             if key in dct:
@@ -327,7 +321,7 @@ class SourcePatch(dbtClassMixin, Replaceable):
     tags: Optional[List[str]] = None
 
     def to_patch_dict(self) -> Dict[str, Any]:
-        dct = self.to_dict()
+        dct = self.to_dict(omit_none=True)
         remove_keys = ('name', 'overrides', 'tables', 'path')
         for key in remove_keys:
             if key in dct:

core/dbt/contracts/relation.py

@@ -52,7 +52,7 @@ class FakeAPIObject(dbtClassMixin, Replaceable, Mapping):
         return len(fields(self.__class__))
 
     def incorporate(self, **kwargs):
-        value = self.to_dict()
+        value = self.to_dict(omit_none=True)
         value = deep_merge(value, kwargs)
         return self.from_dict(value)
 

core/dbt/contracts/results.py

@@ -97,8 +97,8 @@ class BaseResult(dbtClassMixin):
     message: Optional[Union[str, int]]
 
     @classmethod
-    def __pre_deserialize__(cls, data, options=None):
-        data = super().__pre_deserialize__(data, options=options)
+    def __pre_deserialize__(cls, data):
+        data = super().__pre_deserialize__(data)
         if 'message' not in data:
             data['message'] = None
         return data
@@ -206,7 +206,7 @@ class RunResultsArtifact(ExecutionResult, ArtifactMixin):
         )
 
     def write(self, path: str):
-        write_json(path, self.to_dict(options={'keep_none': True}))
+        write_json(path, self.to_dict(omit_none=False))
 
 
 @dataclass
@@ -448,8 +448,8 @@ class CatalogResults(dbtClassMixin):
     errors: Optional[List[str]] = None
     _compile_results: Optional[Any] = None
 
-    def __post_serialize__(self, dct, options=None):
-        dct = super().__post_serialize__(dct, options=options)
+    def __post_serialize__(self, dct):
+        dct = super().__post_serialize__(dct)
         if '_compile_results' in dct:
             del dct['_compile_results']
         return dct

core/dbt/contracts/rpc.py

@@ -38,8 +38,8 @@ class RPCParameters(dbtClassMixin):
     timeout: Optional[float]
 
     @classmethod
-    def __pre_deserialize__(cls, data, options=None):
-        data = super().__pre_deserialize__(data, options=options)
+    def __pre_deserialize__(cls, data, omit_none=True):
+        data = super().__pre_deserialize__(data)
         if 'timeout' not in data:
             data['timeout'] = None
         if 'task_tags' not in data:
@@ -428,8 +428,8 @@ class TaskTiming(dbtClassMixin):
     # These ought to be defaults but superclass order doesn't
     # allow that to work
     @classmethod
-    def __pre_deserialize__(cls, data, options=None):
-        data = super().__pre_deserialize__(data, options=options)
+    def __pre_deserialize__(cls, data):
+        data = super().__pre_deserialize__(data)
         for field_name in ('start', 'end', 'elapsed'):
             if field_name not in data:
                 data[field_name] = None
@@ -496,8 +496,8 @@ class PollResult(RemoteResult, TaskTiming):
     # These ought to be defaults but superclass order doesn't
     # allow that to work
     @classmethod
-    def __pre_deserialize__(cls, data, options=None):
-        data = super().__pre_deserialize__(data, options=options)
+    def __pre_deserialize__(cls, data):
+        data = super().__pre_deserialize__(data)
         for field_name in ('start', 'end', 'elapsed'):
             if field_name not in data:
                 data[field_name] = None

core/dbt/contracts/util.py

@@ -58,7 +58,7 @@ class Mergeable(Replaceable):
 class Writable:
     def write(self, path: str):
         write_json(
-            path, self.to_dict(options={'keep_none': True})  # type: ignore
+            path, self.to_dict(omit_none=False)  # type: ignore
         )
 
 
@@ -74,7 +74,7 @@ class AdditionalPropertiesMixin:
     # not in the class definitions and puts them in an
     # _extra dict in the class
     @classmethod
-    def __pre_deserialize__(cls, data, options=None):
+    def __pre_deserialize__(cls, data):
         # dir() did not work because fields with
         # metadata settings are not found
         # The original version of this would create the
@@ -93,18 +93,18 @@ class AdditionalPropertiesMixin:
             else:
                 new_dict[key] = value
         data = new_dict
-        data = super().__pre_deserialize__(data, options=options)
+        data = super().__pre_deserialize__(data)
         return data
 
-    def __post_serialize__(self, dct, options=None):
-        data = super().__post_serialize__(dct, options=options)
+    def __post_serialize__(self, dct):
+        data = super().__post_serialize__(dct)
         data.update(self.extra)
         if '_extra' in data:
             del data['_extra']
         return data
 
     def replace(self, **kwargs):
-        dct = self.to_dict(options={'keep_none': True})
+        dct = self.to_dict(omit_none=False)
         dct.update(kwargs)
         return self.from_dict(dct)
 

core/dbt/dataclass_schema.py

@@ -1,5 +1,5 @@
 from typing import (
-    Type, ClassVar, Dict, cast, TypeVar
+    Type, ClassVar, cast,
 )
 import re
 from dataclasses import fields
@@ -9,29 +9,28 @@ from dateutil.parser import parse
 
 from hologram import JsonSchemaMixin, FieldEncoder, ValidationError
 
+# type: ignore
 from mashumaro import DataClassDictMixin
-from mashumaro.types import SerializableEncoder, SerializableType
+from mashumaro.config import (
+    TO_DICT_ADD_OMIT_NONE_FLAG, BaseConfig as MashBaseConfig
+)
+from mashumaro.types import SerializableType, SerializationStrategy
 
 
-class DateTimeSerializableEncoder(SerializableEncoder[datetime]):
-    @classmethod
-    def _serialize(cls, value: datetime) -> str:
+class DateTimeSerialization(SerializationStrategy):
+    def serialize(self, value):
         out = value.isoformat()
         # Assume UTC if timezone is missing
         if value.tzinfo is None:
             out = out + "Z"
         return out
 
-    @classmethod
-    def _deserialize(cls, value: str) -> datetime:
+    def deserialize(self, value):
         return (
             value if isinstance(value, datetime) else parse(cast(str, value))
         )
 
 
-TV = TypeVar("TV")
-
-
 # This class pulls in both JsonSchemaMixin from Hologram and
 # DataClassDictMixin from our fork of Mashumaro. The 'to_dict'
 # and 'from_dict' methods come from Mashumaro. Building
@@ -43,23 +42,21 @@ class dbtClassMixin(DataClassDictMixin, JsonSchemaMixin):
        against the schema
     """
 
-    _serializable_encoders: ClassVar[Dict[str, SerializableEncoder]] = {
-        'datetime.datetime': DateTimeSerializableEncoder(),
-    }
+    class Config(MashBaseConfig):
+        code_generation_options = [
+            TO_DICT_ADD_OMIT_NONE_FLAG,
+        ]
+        serialization_strategy = {
+            datetime: DateTimeSerialization(),
+        }
+
     _hyphenated: ClassVar[bool] = False
     ADDITIONAL_PROPERTIES: ClassVar[bool] = False
 
     # This is called by the mashumaro to_dict in order to handle
     # nested classes.
     # Munges the dict that's returned.
-    def __post_serialize__(self, dct, options=None):
-        keep_none = False
-        if options and 'keep_none' in options and options['keep_none']:
-            keep_none = True
-        if not keep_none:  # remove attributes that are None
-            new_dict = {k: v for k, v in dct.items() if v is not None}
-            dct = new_dict
-
+    def __post_serialize__(self, dct):
         if self._hyphenated:
             new_dict = {}
             for key in dct:
@@ -75,7 +72,7 @@ class dbtClassMixin(DataClassDictMixin, JsonSchemaMixin):
     # This is called by the mashumaro _from_dict method, before
     # performing the conversion to a dict
     @classmethod
-    def __pre_deserialize__(cls, data, options=None):
+    def __pre_deserialize__(cls, data):
         if cls._hyphenated:
             new_dict = {}
             for key in data:
@@ -92,8 +89,8 @@ class dbtClassMixin(DataClassDictMixin, JsonSchemaMixin):
     # hologram and in mashumaro.
     def _local_to_dict(self, **kwargs):
         args = {}
-        if 'omit_none' in kwargs and kwargs['omit_none'] is False:
-            args['options'] = {'keep_none': True}
+        if 'omit_none' in kwargs:
+            args['omit_none'] = kwargs['omit_none']
         return self.to_dict(**args)
 
 

core/dbt/logger.py

@@ -95,7 +95,8 @@ class JsonFormatter(LogMessageFormatter):
         # utils imports exceptions which imports logger...
         import dbt.utils
         log_message = super().__call__(record, handler)
-        return json.dumps(log_message.to_dict(), cls=dbt.utils.JSONEncoder)
+        dct = log_message.to_dict(omit_none=True)
+        return json.dumps(dct, cls=dbt.utils.JSONEncoder)
 
 
 class FormatterMixin:
@@ -127,6 +128,7 @@ class OutputHandler(logbook.StreamHandler, FormatterMixin):
     The `format_string` parameter only changes the default text output, not
       debug mode or json.
     """
+
     def __init__(
         self,
         stream,
@@ -220,7 +222,8 @@ class TimingProcessor(logbook.Processor):
 
     def process(self, record):
         if self.timing_info is not None:
-            record.extra['timing_info'] = self.timing_info.to_dict()
+            record.extra['timing_info'] = self.timing_info.to_dict(
+                omit_none=True)
 
 
 class DbtProcessState(logbook.Processor):
@@ -349,6 +352,7 @@ def make_log_dir_if_missing(log_dir):
 class DebugWarnings(logbook.compat.redirected_warnings):
     """Log warnings, except send them to 'debug' instead of 'warning' level.
     """
+
     def make_record(self, message, exception, filename, lineno):
         rv = super().make_record(message, exception, filename, lineno)
         rv.level = logbook.DEBUG

core/dbt/parser/base.py

@@ -252,7 +252,7 @@ class ConfiguredParser(
             'raw_sql': block.contents,
             'unique_id': self.generate_unique_id(name),
             'config': self.config_dict(config),
-            'checksum': block.file.checksum.to_dict(),
+            'checksum': block.file.checksum.to_dict(omit_none=True),
         }
         dct.update(kwargs)
         try:
@@ -301,7 +301,7 @@ class ConfiguredParser(
         self, parsed_node: IntermediateNode, config_dict: Dict[str, Any]
     ) -> None:
         # Overwrite node config
-        final_config_dict = parsed_node.config.to_dict()
+        final_config_dict = parsed_node.config.to_dict(omit_none=True)
         final_config_dict.update(config_dict)
         # re-mangle hooks, in case we got new ones
         self._mangle_hooks(final_config_dict)

core/dbt/parser/schemas.py

@@ -385,7 +385,7 @@ class SchemaParser(SimpleParser[SchemaTestBlock, ParsedSchemaTestNode]):
             'config': self.config_dict(config),
             'test_metadata': test_metadata,
             'column_name': column_name,
-            'checksum': FileHash.empty().to_dict(),
+            'checksum': FileHash.empty().to_dict(omit_none=True),
         }
         try:
             ParsedSchemaTestNode.validate(dct)

core/dbt/parser/snapshots.py

@@ -68,7 +68,8 @@ class SnapshotParser(
 
     def transform(self, node: IntermediateSnapshotNode) -> ParsedSnapshotNode:
         try:
-            parsed_node = ParsedSnapshotNode.from_dict(node.to_dict())
+            dct = node.to_dict(omit_none=True)
+            parsed_node = ParsedSnapshotNode.from_dict(dct)
             self.set_snapshot_attributes(parsed_node)
             return parsed_node
         except ValidationError as exc:

core/dbt/parser/sources.py

@@ -49,8 +49,8 @@ class SourcePatcher:
         if patch is None:
             return unpatched
 
-        source_dct = unpatched.source.to_dict()
-        table_dct = unpatched.table.to_dict()
+        source_dct = unpatched.source.to_dict(omit_none=True)
+        table_dct = unpatched.table.to_dict(omit_none=True)
         patch_path: Optional[Path] = None
 
         source_table_patch: Optional[SourceTablePatch] = None

core/dbt/rpc/builtins.py

@@ -177,7 +177,7 @@ def poll_complete(
 
 
 def _dict_logs(logs: List[LogMessage]) -> List[Dict[str, Any]]:
-    return [log.to_dict() for log in logs]
+    return [log.to_dict(omit_none=True) for log in logs]
 
 
 class Poll(RemoteBuiltinMethod[PollParameters, PollResult]):

core/dbt/rpc/response_manager.py

@@ -97,7 +97,7 @@ class ResponseManager(JSONRPCResponseManager):
                     # Note: errors in to_dict do not show up anywhere in
                     # the output and all you get is a generic 500 error
                     output.result = \
-                        output.result.to_dict(options={'keep_none': True})
+                        output.result.to_dict(omit_none=False)
             yield output
 
     @classmethod

core/dbt/rpc/task_handler.py

@@ -391,7 +391,7 @@ class RequestTaskHandler(threading.Thread, TaskHandlerProtocol):
             except RPCException as exc:
                 # RPC Exceptions come already preserialized for the jsonrpc
                 # framework
-                exc.logs = [log.to_dict() for log in self.logs]
+                exc.logs = [log.to_dict(omit_none=True) for log in self.logs]
                 exc.tags = self.tags
                 raise
 

core/dbt/task/generate.py

@@ -114,8 +114,8 @@ class Catalog(Dict[CatalogKey, CatalogTable]):
                 if unique_id in sources:
                     dbt.exceptions.raise_ambiguous_catalog_match(
                         unique_id,
-                        sources[unique_id].to_dict(),
-                        table.to_dict(),
+                        sources[unique_id].to_dict(omit_none=True),
+                        table.to_dict(omit_none=True),
                     )
                 else:
                     sources[unique_id] = table.replace(unique_id=unique_id)

core/dbt/task/list.py

@@ -110,7 +110,7 @@ class ListTask(GraphRunnableTask):
         for node in self._iterate_selected_nodes():
             yield json.dumps({
                 k: v
-                for k, v in node.to_dict(options={'keep_none': True}).items()
+                for k, v in node.to_dict(omit_none=False).items()
                 if k in self.ALLOWED_KEYS
             })
 

core/dbt/task/printer.py

@@ -169,7 +169,7 @@ def print_snapshot_result_line(
 
     info, status, logger_fn = get_printable_result(
         result, 'snapshotted', 'snapshotting')
-    cfg = model.config.to_dict()
+    cfg = model.config.to_dict(omit_none=True)
 
     msg = "{info} {description}".format(
         info=info, description=description, **cfg)

core/dbt/task/run.py

@@ -117,7 +117,7 @@ def track_model_run(index, num_nodes, run_model_result):
         "hashed_contents": utils.get_hashed_contents(
             run_model_result.node
         ),
-        "timing": [t.to_dict() for t in run_model_result.timing],
+        "timing": [t.to_dict(omit_none=True) for t in run_model_result.timing],
     })
 
 
@@ -193,7 +193,7 @@ class ModelRunner(CompileRunner):
         result = context['load_result']('main')
         adapter_response = {}
         if isinstance(result.response, dbtClassMixin):
-            adapter_response = result.response.to_dict()
+            adapter_response = result.response.to_dict(omit_none=True)
         return RunResult(
             node=model,
             status=RunStatus.Success,

core/dbt/tracking.py

@@ -29,6 +29,7 @@ PACKAGE_INSTALL_SPEC = 'iglu:com.dbt/package_install/jsonschema/1-0-0'
 RPC_REQUEST_SPEC = 'iglu:com.dbt/rpc_request/jsonschema/1-0-1'
 DEPRECATION_WARN_SPEC = 'iglu:com.dbt/deprecation_warn/jsonschema/1-0-0'
 LOAD_ALL_TIMING_SPEC = 'iglu:com.dbt/load_all_timing/jsonschema/1-0-0'
+RESOURCE_COUNTS = 'iglu:com.dbt/resource_counts/jsonschema/1-0-0'
 
 DBT_INVOCATION_ENV = 'DBT_INVOCATION_ENV'
 
@@ -289,6 +290,20 @@ def track_project_load(options):
     )
 
 
+def track_resource_counts(resource_counts):
+    context = [SelfDescribingJson(RESOURCE_COUNTS, resource_counts)]
+    assert active_user is not None, \
+        'Cannot track resource counts when active user is None'
+
+    track(
+        active_user,
+        category='dbt',
+        action='resource_counts',
+        label=active_user.invocation_id,
+        context=context
+    )
+
+
 def track_model_run(options):
     context = [SelfDescribingJson(RUN_MODEL_SPEC, options)]
     assert active_user is not None, \

core/dbt/utils.py

@@ -320,7 +320,7 @@ class JSONEncoder(json.JSONEncoder):
         if hasattr(obj, 'to_dict'):
             # if we have a to_dict we should try to serialize the result of
             # that!
-            return obj.to_dict()
+            return obj.to_dict(omit_none=True)
         return super().default(obj)
 
 

core/dbt/version.py

@@ -96,5 +96,5 @@ def _get_dbt_plugins_info():
         yield plugin_name, mod.version
 
 
-__version__ = '0.19.1b2'
+__version__ = '0.19.1rc2'
 installed = get_installed_version()

core/mashumaro/LICENSE

@@ -1,201 +0,0 @@
-                                 Apache License
-                           Version 2.0, January 2004
-                        http://www.apache.org/licenses/
-
-   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
-   1. Definitions.
-
-      "License" shall mean the terms and conditions for use, reproduction,
-      and distribution as defined by Sections 1 through 9 of this document.
-
-      "Licensor" shall mean the copyright owner or entity authorized by
-      the copyright owner that is granting the License.
-
-      "Legal Entity" shall mean the union of the acting entity and all
-      other entities that control, are controlled by, or are under common
-      control with that entity. For the purposes of this definition,
-      "control" means (i) the power, direct or indirect, to cause the
-      direction or management of such entity, whether by contract or
-      otherwise, or (ii) ownership of fifty percent (50%) or more of the
-      outstanding shares, or (iii) beneficial ownership of such entity.
-
-      "You" (or "Your") shall mean an individual or Legal Entity
-      exercising permissions granted by this License.
-
-      "Source" form shall mean the preferred form for making modifications,
-      including but not limited to software source code, documentation
-      source, and configuration files.
-
-      "Object" form shall mean any form resulting from mechanical
-      transformation or translation of a Source form, including but
-      not limited to compiled object code, generated documentation,
-      and conversions to other media types.
-
-      "Work" shall mean the work of authorship, whether in Source or
-      Object form, made available under the License, as indicated by a
-      copyright notice that is included in or attached to the work
-      (an example is provided in the Appendix below).
-
-      "Derivative Works" shall mean any work, whether in Source or Object
-      form, that is based on (or derived from) the Work and for which the
-      editorial revisions, annotations, elaborations, or other modifications
-      represent, as a whole, an original work of authorship. For the purposes
-      of this License, Derivative Works shall not include works that remain
-      separable from, or merely link (or bind by name) to the interfaces of,
-      the Work and Derivative Works thereof.
-
-      "Contribution" shall mean any work of authorship, including
-      the original version of the Work and any modifications or additions
-      to that Work or Derivative Works thereof, that is intentionally
-      submitted to Licensor for inclusion in the Work by the copyright owner
-      or by an individual or Legal Entity authorized to submit on behalf of
-      the copyright owner. For the purposes of this definition, "submitted"
-      means any form of electronic, verbal, or written communication sent
-      to the Licensor or its representatives, including but not limited to
-      communication on electronic mailing lists, source code control systems,
-      and issue tracking systems that are managed by, or on behalf of, the
-      Licensor for the purpose of discussing and improving the Work, but
-      excluding communication that is conspicuously marked or otherwise
-      designated in writing by the copyright owner as "Not a Contribution."
-
-      "Contributor" shall mean Licensor and any individual or Legal Entity
-      on behalf of whom a Contribution has been received by Licensor and
-      subsequently incorporated within the Work.
-
-   2. Grant of Copyright License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      copyright license to reproduce, prepare Derivative Works of,
-      publicly display, publicly perform, sublicense, and distribute the
-      Work and such Derivative Works in Source or Object form.
-
-   3. Grant of Patent License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      (except as stated in this section) patent license to make, have made,
-      use, offer to sell, sell, import, and otherwise transfer the Work,
-      where such license applies only to those patent claims licensable
-      by such Contributor that are necessarily infringed by their
-      Contribution(s) alone or by combination of their Contribution(s)
-      with the Work to which such Contribution(s) was submitted. If You
-      institute patent litigation against any entity (including a
-      cross-claim or counterclaim in a lawsuit) alleging that the Work
-      or a Contribution incorporated within the Work constitutes direct
-      or contributory patent infringement, then any patent licenses
-      granted to You under this License for that Work shall terminate
-      as of the date such litigation is filed.
-
-   4. Redistribution. You may reproduce and distribute copies of the
-      Work or Derivative Works thereof in any medium, with or without
-      modifications, and in Source or Object form, provided that You
-      meet the following conditions:
-
-      (a) You must give any other recipients of the Work or
-          Derivative Works a copy of this License; and
-
-      (b) You must cause any modified files to carry prominent notices
-          stating that You changed the files; and
-
-      (c) You must retain, in the Source form of any Derivative Works
-          that You distribute, all copyright, patent, trademark, and
-          attribution notices from the Source form of the Work,
-          excluding those notices that do not pertain to any part of
-          the Derivative Works; and
-
-      (d) If the Work includes a "NOTICE" text file as part of its
-          distribution, then any Derivative Works that You distribute must
-          include a readable copy of the attribution notices contained
-          within such NOTICE file, excluding those notices that do not
-          pertain to any part of the Derivative Works, in at least one
-          of the following places: within a NOTICE text file distributed
-          as part of the Derivative Works; within the Source form or
-          documentation, if provided along with the Derivative Works; or,
-          within a display generated by the Derivative Works, if and
-          wherever such third-party notices normally appear. The contents
-          of the NOTICE file are for informational purposes only and
-          do not modify the License. You may add Your own attribution
-          notices within Derivative Works that You distribute, alongside
-          or as an addendum to the NOTICE text from the Work, provided
-          that such additional attribution notices cannot be construed
-          as modifying the License.
-
-      You may add Your own copyright statement to Your modifications and
-      may provide additional or different license terms and conditions
-      for use, reproduction, or distribution of Your modifications, or
-      for any such Derivative Works as a whole, provided Your use,
-      reproduction, and distribution of the Work otherwise complies with
-      the conditions stated in this License.
-
-   5. Submission of Contributions. Unless You explicitly state otherwise,
-      any Contribution intentionally submitted for inclusion in the Work
-      by You to the Licensor shall be under the terms and conditions of
-      this License, without any additional terms or conditions.
-      Notwithstanding the above, nothing herein shall supersede or modify
-      the terms of any separate license agreement you may have executed
-      with Licensor regarding such Contributions.
-
-   6. Trademarks. This License does not grant permission to use the trade
-      names, trademarks, service marks, or product names of the Licensor,
-      except as required for reasonable and customary use in describing the
-      origin of the Work and reproducing the content of the NOTICE file.
-
-   7. Disclaimer of Warranty. Unless required by applicable law or
-      agreed to in writing, Licensor provides the Work (and each
-      Contributor provides its Contributions) on an "AS IS" BASIS,
-      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-      implied, including, without limitation, any warranties or conditions
-      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
-      PARTICULAR PURPOSE. You are solely responsible for determining the
-      appropriateness of using or redistributing the Work and assume any
-      risks associated with Your exercise of permissions under this License.
-
-   8. Limitation of Liability. In no event and under no legal theory,
-      whether in tort (including negligence), contract, or otherwise,
-      unless required by applicable law (such as deliberate and grossly
-      negligent acts) or agreed to in writing, shall any Contributor be
-      liable to You for damages, including any direct, indirect, special,
-      incidental, or consequential damages of any character arising as a
-      result of this License or out of the use or inability to use the
-      Work (including but not limited to damages for loss of goodwill,
-      work stoppage, computer failure or malfunction, or any and all
-      other commercial damages or losses), even if such Contributor
-      has been advised of the possibility of such damages.
-
-   9. Accepting Warranty or Additional Liability. While redistributing
-      the Work or Derivative Works thereof, You may choose to offer,
-      and charge a fee for, acceptance of support, warranty, indemnity,
-      or other liability obligations and/or rights consistent with this
-      License. However, in accepting such obligations, You may act only
-      on Your own behalf and on Your sole responsibility, not on behalf
-      of any other Contributor, and only if You agree to indemnify,
-      defend, and hold each Contributor harmless for any liability
-      incurred by, or claims asserted against, such Contributor by reason
-      of your accepting any such warranty or additional liability.
-
-   END OF TERMS AND CONDITIONS
-
-   APPENDIX: How to apply the Apache License to your work.
-
-      To apply the Apache License to your work, attach the following
-      boilerplate notice, with the fields enclosed by brackets "{}"
-      replaced with your own identifying information. (Don't include
-      the brackets!)  The text should be enclosed in the appropriate
-      comment syntax for the file format. We also recommend that a
-      file or class name and description of purpose be included on the
-      same "printed page" as the copyright notice for easier
-      identification within third-party archives.
-
-   Copyright 2017 Alexander Tikhonov
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.

core/mashumaro/__init__.py

@@ -1,15 +0,0 @@
-from mashumaro.exceptions import MissingField
-from mashumaro.helper import field_options
-from mashumaro.serializer.base.dict import DataClassDictMixin
-from mashumaro.serializer.json import DataClassJSONMixin
-from mashumaro.serializer.msgpack import DataClassMessagePackMixin
-from mashumaro.serializer.yaml import DataClassYAMLMixin
-
-__all__ = [
-    "MissingField",
-    "DataClassDictMixin",
-    "DataClassJSONMixin",
-    "DataClassMessagePackMixin",
-    "DataClassYAMLMixin",
-    "field_options",
-]

core/mashumaro/exceptions.py

@@ -1,84 +0,0 @@
-from mashumaro.meta.helpers import type_name
-
-
-class MissingField(LookupError):
-    def __init__(self, field_name, field_type, holder_class):
-        self.field_name = field_name
-        self.field_type = field_type
-        self.holder_class = holder_class
-
-    @property
-    def field_type_name(self):
-        return type_name(self.field_type)
-
-    @property
-    def holder_class_name(self):
-        return type_name(self.holder_class)
-
-    def __str__(self):
-        return (
-            f'Field "{self.field_name}" of type {self.field_type_name}'
-            f" is missing in {self.holder_class_name} instance"
-        )
-
-
-class UnserializableDataError(TypeError):
-    pass
-
-
-class UnserializableField(UnserializableDataError):
-    def __init__(self, field_name, field_type, holder_class, msg=None):
-        self.field_name = field_name
-        self.field_type = field_type
-        self.holder_class = holder_class
-        self.msg = msg
-
-    @property
-    def field_type_name(self):
-        return type_name(self.field_type)
-
-    @property
-    def holder_class_name(self):
-        return type_name(self.holder_class)
-
-    def __str__(self):
-        s = (
-            f'Field "{self.field_name}" of type {self.field_type_name} '
-            f"in {self.holder_class_name} is not serializable"
-        )
-        if self.msg:
-            s += f": {self.msg}"
-        return s
-
-
-class InvalidFieldValue(ValueError):
-    def __init__(
-        self, field_name, field_type, field_value, holder_class, msg=None
-    ):
-        self.field_name = field_name
-        self.field_type = field_type
-        self.field_value = field_value
-        self.holder_class = holder_class
-        self.msg = msg
-
-    @property
-    def field_type_name(self):
-        return type_name(self.field_type)
-
-    @property
-    def holder_class_name(self):
-        return type_name(self.holder_class)
-
-    def __str__(self):
-        s = (
-            f'Field "{self.field_name}" of type {self.field_type_name} '
-            f"in {self.holder_class_name} has invalid value "
-            f"{repr(self.field_value)}"
-        )
-        if self.msg:
-            s += f": {self.msg}"
-        return s
-
-
-class BadHookSignature(TypeError):
-    pass

core/mashumaro/helper.py

@@ -1,11 +0,0 @@
-from typing import Any, Callable, Optional, Union
-
-
-def field_options(
-    serialize: Optional[Callable[[Any], Any]] = None,
-    deserialize: Optional[Union[str, Callable[[Any], Any]]] = None,
-):
-    return {"serialize": serialize, "deserialize": deserialize}
-
-
-__all__ = ["field_options"]

core/mashumaro/meta/helpers.py

@@ -1,105 +0,0 @@
-import dataclasses
-import types
-import typing
-
-from .macros import PY_36, PY_37, PY_38, PY_39
-
-
-def get_imported_module_names():
-    # noinspection PyUnresolvedReferences
-    return {
-        value.__name__
-        for value in globals().values()
-        if isinstance(value, types.ModuleType)
-    }
-
-
-def get_type_origin(t):
-    try:
-        if PY_36:
-            return t.__extra__
-        elif PY_37 or PY_38 or PY_39:
-            return t.__origin__
-    except AttributeError:
-        return t
-
-
-def type_name(t):
-    if is_generic(t):
-        return str(t)
-    else:
-        try:
-            return f"{t.__module__}.{t.__name__}"
-        except AttributeError:
-            return str(t)
-
-
-def is_special_typing_primitive(t):
-    try:
-        issubclass(t, object)
-        return False
-    except TypeError:
-        return True
-
-
-def is_generic(t):
-    if PY_37 or PY_38 or PY_39:
-        # noinspection PyProtectedMember
-        # noinspection PyUnresolvedReferences
-        return t.__class__ is typing._GenericAlias
-    elif PY_36:
-        # noinspection PyUnresolvedReferences
-        return issubclass(t.__class__, typing.GenericMeta)
-    else:
-        raise NotImplementedError
-
-
-def is_union(t):
-    try:
-        return t.__origin__ is typing.Union
-    except AttributeError:
-        return False
-
-
-def is_type_var(t):
-    return hasattr(t, "__constraints__")
-
-
-def is_class_var(t):
-    if PY_36:
-        return (
-            is_special_typing_primitive(t) and type(t).__name__ == "_ClassVar"
-        )
-    if PY_37 or PY_38 or PY_39:
-        return get_type_origin(t) is typing.ClassVar
-    else:
-        raise NotImplementedError
-
-
-def is_init_var(t):
-    if PY_36 or PY_37:
-        return get_type_origin(t) is dataclasses.InitVar
-    elif PY_38 or PY_39:
-        return isinstance(t, dataclasses.InitVar)
-    else:
-        raise NotImplementedError
-
-
-def get_class_that_define_method(method_name, cls):
-    for cls in cls.__mro__:
-        if method_name in cls.__dict__:
-            return cls
-
-
-__all__ = [
-    "get_imported_module_names",
-    "get_type_origin",
-    "type_name",
-    "is_special_typing_primitive",
-    "is_generic",
-    "is_union",
-    "is_type_var",
-    "is_class_var",
-    "is_init_var",
-    "get_class_that_define_method",
-]

core/mashumaro/meta/macros.py

@@ -1,9 +0,0 @@
-import sys
-
-PY_36 = sys.version_info.major == 3 and sys.version_info.minor == 6
-PY_37 = sys.version_info.major == 3 and sys.version_info.minor == 7
-PY_38 = sys.version_info.major == 3 and sys.version_info.minor == 8
-PY_39 = sys.version_info.major == 3 and sys.version_info.minor == 9
-
-
-__all__ = ["PY_36", "PY_37", "PY_38", "PY_39"]

core/mashumaro/meta/patch.py

@@ -1,13 +0,0 @@
-from .macros import PY_36
-
-
-def patch_fromisoformat():
-    if PY_36:
-        # noinspection PyPackageRequirements
-        # noinspection PyUnresolvedReferences
-        from backports.datetime_fromisoformat import MonkeyPatch
-
-        MonkeyPatch.patch_fromisoformat()
-
-
-__all__ = ["patch_fromisoformat"]

core/mashumaro/serializer/base/__init__.py

@@ -1,3 +0,0 @@
-from .dict import DataClassDictMixin
-
-__all__ = ["DataClassDictMixin"]

core/mashumaro/serializer/base/dict.py

@@ -1,61 +0,0 @@
-# this file has been modified for use with dbt
-from typing import Any, Dict, Mapping, Type, TypeVar, Optional, ClassVar
-
-from mashumaro.serializer.base.metaprogramming import CodeBuilder
-from mashumaro.types import SerializableEncoder
-
-T = TypeVar("T", bound="DataClassDictMixin")
-
-
-class DataClassDictMixin:
-    _serializable_encoders: ClassVar[Dict[Type, SerializableEncoder]] = {}
-
-    def __init_subclass__(cls: Type[T], **kwargs):
-        builder = CodeBuilder(cls)
-        exc = None
-        try:
-            builder.add_from_dict()
-        except Exception as e:
-            exc = e
-        try:
-            builder.add_to_dict()
-        except Exception as e:
-            exc = e
-        if exc:
-            raise exc
-
-    def to_dict(
-        self: T,
-        use_bytes: bool = False,
-        use_enum: bool = False,
-        use_datetime: bool = False,
-        options: Optional[Dict[str, Any]] = None,
-    ) -> dict:
-        ...
-
-    @classmethod
-    def from_dict(
-        cls: Type[T],
-        d: Mapping,
-        use_bytes: bool = False,
-        use_enum: bool = False,
-        use_datetime: bool = False,
-        options: Optional[Dict[str, Any]] = None,
-    ) -> T:
-        ...
-
-    @classmethod
-    def __pre_deserialize__(cls: Type[T], d: Dict[Any, Any], options: Optional[Dict[str, Any]]) -> Dict[Any, Any]:
-        ...
-
-    @classmethod
-    def __post_deserialize__(cls: Type[T], obj: T, options: Optional[Dict[str, Any]]) -> T:
-        ...
-
-    def __pre_serialize__(self: T, options: Optional[Dict[str, Any]]) -> T:
-        ...
-
-    def __post_serialize__(self: T, d: Dict[Any, Any], options: Optional[Dict[str, Any]]) -> Dict[Any, Any]:
-        ...
-
-__all__ = ["DataClassDictMixin"]

core/mashumaro/serializer/base/helpers.py

@@ -1,26 +0,0 @@
-import datetime
-import re
-
-
-def parse_timezone(s: str):
-    regexp = re.compile(r"^UTC(([+-][0-2][0-9]):([0-5][0-9]))?$")
-    match = regexp.match(s)
-    if not match:
-        raise ValueError(
-            f"Time zone {s} must be either UTC " f"or in format UTC[+-]hh:mm"
-        )
-    if match.group(1):
-        hours = int(match.group(2))
-        minutes = int(match.group(3))
-        return datetime.timezone(
-            datetime.timedelta(
-                hours=hours, minutes=minutes if hours >= 0 else -minutes
-            )
-        )
-    else:
-        return datetime.timezone.utc
-
-
-__all__ = [
-    "parse_timezone",
-]

core/mashumaro/serializer/base/metaprogramming.py

@@ -1,830 +0,0 @@
-# this file as been modified for use with dbt
-# noinspection PyUnresolvedReferences
-import builtins  # noqa
-import collections
-import collections.abc
-import datetime
-import enum
-import ipaddress
-import os
-import pathlib
-import typing
-import uuid
-
-# noinspection PyUnresolvedReferences
-from base64 import decodebytes, encodebytes  # noqa
-from contextlib import contextmanager, suppress
-
-# noinspection PyProtectedMember
-from dataclasses import _FIELDS, MISSING, Field, is_dataclass
-from decimal import Decimal
-from fractions import Fraction
-
-# noinspection PyUnresolvedReferences
-from mashumaro.exceptions import (  # noqa
-    BadHookSignature,
-    InvalidFieldValue,
-    MissingField,
-    UnserializableDataError,
-    UnserializableField,
-)
-from mashumaro.meta.helpers import (
-    get_class_that_define_method,
-    get_imported_module_names,
-    get_type_origin,
-    is_class_var,
-    is_generic,
-    is_init_var,
-    is_special_typing_primitive,
-    is_type_var,
-    is_union,
-    type_name,
-)
-from mashumaro.meta.patch import patch_fromisoformat
-from mashumaro.serializer.base.helpers import *  # noqa
-from mashumaro.types import SerializableType, SerializationStrategy
-
-patch_fromisoformat()
-
-
-NoneType = type(None)
-INITIAL_MODULES = get_imported_module_names()
-
-
-__PRE_SERIALIZE__ = "__pre_serialize__"
-__PRE_DESERIALIZE__ = "__pre_deserialize__"
-__POST_SERIALIZE__ = "__post_serialize__"
-__POST_DESERIALIZE__ = "__post_deserialize__"
-
-DataClassDictMixinPath = "mashumaro.serializer.base.dict.DataClassDictMixin"
-
-
-class CodeBuilder:
-    def __init__(self, cls):
-        self.cls = cls
-        self.lines: typing.List[str] = []
-        self.modules: typing.Set[str] = set()
-        self.globals: typing.Set[str] = set()
-        self._current_indent: str = ""
-
-    def reset(self) -> None:
-        self.lines = []
-        self.modules = INITIAL_MODULES.copy()
-        self.globals = set()
-        self._current_indent = ""
-
-    @property
-    def namespace(self) -> typing.Dict[typing.Any, typing.Any]:
-        return self.cls.__dict__
-
-    @property
-    def annotations(self) -> typing.Dict[str, typing.Any]:
-        return self.namespace.get("__annotations__", {})
-
-    def __get_field_types(
-        self, recursive=True
-    ) -> typing.Dict[str, typing.Any]:
-        fields = {}
-        for fname, ftype in typing.get_type_hints(self.cls).items():
-            if is_class_var(ftype) or is_init_var(ftype):
-                continue
-            if recursive or fname in self.annotations:
-                fields[fname] = ftype
-        return fields
-
-    @property
-    def field_types(self) -> typing.Dict[str, typing.Any]:
-        return self.__get_field_types()
-
-    @property
-    def defaults(self) -> typing.Dict[str, typing.Any]:
-        d = {}
-        for ancestor in self.cls.__mro__[-1:0:-1]:
-            if is_dataclass(ancestor):
-                for field in getattr(ancestor, _FIELDS).values():
-                    if field.default is not MISSING:
-                        d[field.name] = field.default
-                    else:
-                        d[field.name] = field.default_factory
-        for name in self.__get_field_types(recursive=False):
-            field = self.namespace.get(name, MISSING)
-            if isinstance(field, Field):
-                if field.default is not MISSING:
-                    d[name] = field.default
-                else:
-                    # https://github.com/python/mypy/issues/6910
-                    d[name] = field.default_factory  # type: ignore
-            else:
-                d[name] = field
-        return d
-
-    @property
-    def metadatas(self) -> typing.Dict[str, typing.Mapping[str, typing.Any]]:
-        d = {}
-        for ancestor in self.cls.__mro__[-1:0:-1]:
-            if is_dataclass(ancestor):
-                for field in getattr(ancestor, _FIELDS).values():
-                    d[field.name] = field.metadata
-        for name in self.__get_field_types(recursive=False):
-            field = self.namespace.get(name, MISSING)
-            if isinstance(field, Field):
-                d[name] = field.metadata
-        return d
-
-    def _add_type_modules(self, *types_) -> None:
-        for t in types_:
-            module = getattr(t, "__module__", None)
-            if not module:
-                return
-            self.ensure_module_imported(module)
-            args = getattr(t, "__args__", ())
-            if args:
-                self._add_type_modules(*args)
-            constraints = getattr(t, "__constraints__", ())
-            if constraints:
-                self._add_type_modules(*constraints)
-
-    def ensure_module_imported(self, module: str) -> None:
-        if module not in self.modules:
-            self.modules.add(module)
-            self.add_line(f"if '{module}' not in globals():")
-            with self.indent():
-                self.add_line(f"import {module}")
-            root_module = module.split(".")[0]
-            if root_module not in self.globals:
-                self.globals.add(root_module)
-                self.add_line("else:")
-                with self.indent():
-                    self.add_line(f"global {root_module}")
-
-    def add_line(self, line) -> None:
-        self.lines.append(f"{self._current_indent}{line}")
-
-    @contextmanager
-    def indent(self) -> typing.Generator[None, None, None]:
-        self._current_indent += " " * 4
-        try:
-            yield
-        finally:
-            self._current_indent = self._current_indent[:-4]
-
-    def compile(self) -> None:
-        exec("\n".join(self.lines), globals(), self.__dict__)
-
-    def get_declared_hook(self, method_name: str):
-        if not hasattr(self.cls, method_name):
-            return
-        cls = get_class_that_define_method(method_name, self.cls)
-        if type_name(cls) != DataClassDictMixinPath:
-            return cls.__dict__[method_name]
-
-    def add_from_dict(self) -> None:
-
-        self.reset()
-        self.add_line("@classmethod")
-        self.add_line(
-            "def from_dict(cls, d, use_bytes=False, use_enum=False, "
-            "use_datetime=False, options=None):"
-        )
-        with self.indent():
-            pre_deserialize = self.get_declared_hook(__PRE_DESERIALIZE__)
-            if pre_deserialize:
-                if not isinstance(pre_deserialize, classmethod):
-                    raise BadHookSignature(
-                        f"`{__PRE_DESERIALIZE__}` must be a class method with "
-                        f"Callable[[Dict[Any, Any]], Dict[Any, Any]] signature"
-                    )
-                else:
-                    self.add_line(f"d = cls.{__PRE_DESERIALIZE__}(d, options=options)")
-            self.add_line("try:")
-            with self.indent():
-                self.add_line("kwargs = {}")
-                for fname, ftype in self.field_types.items():
-                    metadata = self.metadatas.get(fname)
-                    self._add_type_modules(ftype)
-                    self.add_line(f"value = d.get('{fname}', MISSING)")
-                    self.add_line("if value is None:")
-                    with self.indent():
-                        self.add_line(f"kwargs['{fname}'] = None")
-                    self.add_line("else:")
-                    with self.indent():
-                        if self.defaults[fname] is MISSING:
-                            self.add_line("if value is MISSING:")
-                            with self.indent():
-                                if isinstance(ftype, SerializationStrategy):
-                                    self.add_line(
-                                        f"raise MissingField('{fname}',"
-                                        f"{type_name(ftype.__class__)},cls)"
-                                    )
-                                else:
-                                    self.add_line(
-                                        f"raise MissingField('{fname}',"
-                                        f"{type_name(ftype)},cls)"
-                                    )
-                            self.add_line("else:")
-                            with self.indent():
-                                unpacked_value = self._unpack_field_value(
-                                    fname=fname,
-                                    ftype=ftype,
-                                    parent=self.cls,
-                                    metadata=metadata,
-                                )
-                                self.add_line("try:")
-                                with self.indent():
-                                    self.add_line(
-                                        f"kwargs['{fname}'] = {unpacked_value}"
-                                    )
-                                self.add_line("except Exception as e:")
-                                with self.indent():
-                                    if isinstance(
-                                        ftype, SerializationStrategy
-                                    ):
-                                        field_type = type_name(ftype.__class__)
-                                    else:
-                                        field_type = type_name(ftype)
-                                    self.add_line(
-                                        f"raise InvalidFieldValue('{fname}',"
-                                        f"{field_type},value,cls)"
-                                    )
-                        else:
-                            self.add_line("if value is not MISSING:")
-                            with self.indent():
-                                unpacked_value = self._unpack_field_value(
-                                    fname=fname,
-                                    ftype=ftype,
-                                    parent=self.cls,
-                                    metadata=metadata,
-                                )
-                                self.add_line("try:")
-                                with self.indent():
-                                    self.add_line(
-                                        f"kwargs['{fname}'] = {unpacked_value}"
-                                    )
-                                self.add_line("except Exception as e:")
-                                with self.indent():
-                                    if isinstance(
-                                        ftype, SerializationStrategy
-                                    ):
-                                        field_type = type_name(ftype.__class__)
-                                    else:
-                                        field_type = type_name(ftype)
-                                    self.add_line(
-                                        f"raise InvalidFieldValue('{fname}',"
-                                        f"{field_type},value,cls)"
-                                    )
-            self.add_line("except AttributeError:")
-            with self.indent():
-                self.add_line("if not isinstance(d, dict):")
-                with self.indent():
-                    self.add_line(
-                        f"raise ValueError('Argument for "
-                        f"{type_name(self.cls)}.from_dict method "
-                        f"should be a dict instance') from None"
-                    )
-                self.add_line("else:")
-                with self.indent():
-                    self.add_line("raise")
-            post_deserialize = self.get_declared_hook(__POST_DESERIALIZE__)
-            if post_deserialize:
-                if not isinstance(post_deserialize, classmethod):
-                    raise BadHookSignature(
-                        f"`{__POST_DESERIALIZE__}` must be a class method "
-                        f"with Callable[[{type_name(self.cls)}], "
-                        f"{type_name(self.cls)}] signature"
-                    )
-                else:
-                    self.add_line(
-                        f"return cls.{__POST_DESERIALIZE__}(cls(**kwargs), options=options)"
-                    )
-            else:
-                self.add_line("return cls(**kwargs)")
-        self.add_line("setattr(cls, 'from_dict', from_dict)")
-        self.compile()
-
-    def add_to_dict(self) -> None:
-
-        self.reset()
-        self.add_line(
-            "def to_dict(self, use_bytes=False, use_enum=False, "
-            "use_datetime=False, options=None):"
-        )
-        with self.indent():
-            pre_serialize = self.get_declared_hook(__PRE_SERIALIZE__)
-            if pre_serialize:
-                self.add_line(f"self = self.{__PRE_SERIALIZE__}(options=options)")
-            self.add_line("kwargs = {}")
-            for fname, ftype in self.field_types.items():
-                metadata = self.metadatas.get(fname)
-                self.add_line(f"value = getattr(self, '{fname}')")
-                self.add_line("if value is None:")
-                with self.indent():
-                    self.add_line(f"kwargs['{fname}'] = None")
-                self.add_line("else:")
-                with self.indent():
-                    packed_value = self._pack_value(
-                        fname=fname,
-                        ftype=ftype,
-                        parent=self.cls,
-                        metadata=metadata,
-                    )
-                    self.add_line(f"kwargs['{fname}'] = {packed_value}")
-            post_serialize = self.get_declared_hook(__POST_SERIALIZE__)
-            if post_serialize:
-                self.add_line(f"return self.{__POST_SERIALIZE__}(kwargs, options=options)")
-            else:
-                self.add_line("return kwargs")
-        self.add_line("setattr(cls, 'to_dict', to_dict)")
-        self.compile()
-
-    def add_pack_union(self, fname, ftype, parent, variant_types, value_name, metadata):
-        self._add_type_modules(*variant_types)
-        self.add_line(f'def resolve_union(value):')
-        with self.indent():
-            for variant in variant_types:
-                if is_generic(variant):
-                    variant_name = get_type_origin(variant).__name__
-                else:
-                    variant_name = type_name(variant)
-                self.add_line(f'if isinstance(value, {variant_name}):')
-                with self.indent():
-                    self.add_line('try:')
-                    with self.indent():
-                        packed = self._pack_value(fname, variant, parent, value_name, metadata=metadata)
-                        self.add_line(f'return {packed}')
-                    self.add_line('except (TypeError, AttributeError, ValueError, LookupError) as e:')
-                    with self.indent():
-                        self.add_line('pass')
-            else:
-                variant_type_names = ", ".join([type_name(v) for v in variant_types])
-                self.add_line(f"raise ValueError('Union value could not be "
-                              f"encoded using types ({variant_type_names})')")
-            return 'resolve_union(value)'
-
-    def _pack_value(
-        self, fname, ftype, parent, value_name="value", metadata=None
-    ):
-
-        overridden: typing.Optional[str] = None
-        if metadata is not None:
-            serialize_option = metadata.get("serialize")
-            if callable(serialize_option):
-                setattr(
-                    self.cls,
-                    f"__{fname}_serialize",
-                    staticmethod(serialize_option),
-                )
-                return f"self.__{fname}_serialize(self.{fname})"
-
-        if is_dataclass(ftype):
-            return f"{value_name}.to_dict(use_bytes, use_enum, use_datetime, options)"
-
-        with suppress(TypeError):
-            if issubclass(ftype, SerializableType):
-                return f"{value_name}._serialize()"
-        if isinstance(ftype, SerializationStrategy):
-            return (
-                f"self.__dataclass_fields__['{fname}'].type"
-                f"._serialize({value_name})"
-            )
-        if type_name(ftype) in self.cls._serializable_encoders:
-            return f"self._serializable_encoders['{type_name(ftype)}']._serialize({value_name})"
-
-        origin_type = get_type_origin(ftype)
-        if is_special_typing_primitive(origin_type):
-            if origin_type is typing.Any:
-                return overridden or value_name
-            elif is_union(ftype):
-                args = getattr(ftype, "__args__", ())
-                if len(args) == 2 and args[1] == NoneType:  # it is Optional
-                    return self._pack_value(fname, args[0], parent, metadata=metadata)
-                else:
-                    return self.add_pack_union(fname, ftype, parent, args, value_name, metadata)
-            elif origin_type is typing.AnyStr:
-                raise UnserializableDataError(
-                    "AnyStr is not supported by mashumaro"
-                )
-            elif is_type_var(ftype):
-                raise UnserializableDataError(
-                    "TypeVars are not supported by mashumaro"
-                )
-            else:
-                raise UnserializableDataError(
-                    f"{ftype} as a field type is not supported by mashumaro"
-                )
-        elif origin_type in (bool, NoneType):
-            return overridden or value_name
-        elif issubclass(origin_type, typing.Collection):
-            args = getattr(ftype, "__args__", ())
-
-            def inner_expr(arg_num=0, v_name="value"):
-                return self._pack_value(fname, args[arg_num], parent, v_name, metadata=metadata)
-
-            if issubclass(
-                origin_type,
-                (typing.List, typing.Deque, typing.Tuple, typing.AbstractSet),
-            ):
-                if is_generic(ftype):
-                    return (
-                        overridden
-                        or f"[{inner_expr()} for value in {value_name}]"
-                    )
-                elif ftype is list:
-                    raise UnserializableField(
-                        fname, ftype, parent, "Use typing.List[T] instead"
-                    )
-                elif ftype is collections.deque:
-                    raise UnserializableField(
-                        fname, ftype, parent, "Use typing.Deque[T] instead"
-                    )
-                elif ftype is tuple:
-                    raise UnserializableField(
-                        fname, ftype, parent, "Use typing.Tuple[T] instead"
-                    )
-                elif ftype is set:
-                    raise UnserializableField(
-                        fname, ftype, parent, "Use typing.Set[T] instead"
-                    )
-                elif ftype is frozenset:
-                    raise UnserializableField(
-                        fname, ftype, parent, "Use typing.FrozenSet[T] instead"
-                    )
-            elif issubclass(origin_type, typing.ChainMap):
-                if ftype is collections.ChainMap:
-                    raise UnserializableField(
-                        fname,
-                        ftype,
-                        parent,
-                        "Use typing.ChainMap[KT,VT] instead",
-                    )
-                elif is_generic(ftype):
-                    if is_dataclass(args[0]):
-                        raise UnserializableDataError(
-                            "ChainMaps with dataclasses as keys "
-                            "are not supported by mashumaro"
-                        )
-                    else:
-                        return (
-                            overridden
-                            or f'[{{{inner_expr(0,"key")}:{inner_expr(1)} '
-                            f"for key,value in m.items()}} "
-                            f"for m in value.maps]"
-                        )
-            elif issubclass(origin_type, str):
-                return overridden or value_name
-            elif issubclass(origin_type, typing.Mapping):
-                if ftype is dict:
-                    raise UnserializableField(
-                        fname,
-                        ftype,
-                        parent,
-                        "Use typing.Dict[KT,VT] or Mapping[KT,VT] instead",
-                    )
-                elif is_generic(ftype):
-                    if is_dataclass(args[0]):
-                        raise UnserializableDataError(
-                            "Mappings with dataclasses as keys "
-                            "are not supported by mashumaro"
-                        )
-                    else:
-                        return (
-                            overridden
-                            or f'{{{inner_expr(0,"key")}: {inner_expr(1)} '
-                            f"for key, value in {value_name}.items()}}"
-                        )
-            elif issubclass(origin_type, typing.ByteString):
-                specific = f"encodebytes({value_name}).decode()"
-                return (
-                    f"{value_name} if use_bytes else {overridden or specific}"
-                )
-            elif issubclass(origin_type, typing.Sequence):
-                if is_generic(ftype):
-                    return (
-                        overridden
-                        or f"[{inner_expr()} for value in {value_name}]"
-                    )
-        elif issubclass(origin_type, os.PathLike):
-            return overridden or f"{value_name}.__fspath__()"
-        elif issubclass(origin_type, enum.Enum):
-            specific = f"{value_name}.value"
-            return f"{value_name} if use_enum else {overridden or specific}"
-        elif origin_type is int:
-            return overridden or f"int({value_name})"
-        elif origin_type is float:
-            return overridden or f"float({value_name})"
-        elif origin_type in (datetime.datetime, datetime.date, datetime.time):
-            if overridden:
-                return f"{value_name} if use_datetime else {overridden}"
-            return (
-                f"{value_name} if use_datetime else {value_name}.isoformat()"
-            )
-        elif origin_type is datetime.timedelta:
-            return overridden or f"{value_name}.total_seconds()"
-        elif origin_type is datetime.timezone:
-            return overridden or f"{value_name}.tzname(None)"
-        elif origin_type is uuid.UUID:
-            return overridden or f"str({value_name})"
-        elif origin_type in [
-            ipaddress.IPv4Address,
-            ipaddress.IPv6Address,
-            ipaddress.IPv4Network,
-            ipaddress.IPv6Network,
-            ipaddress.IPv4Interface,
-            ipaddress.IPv6Interface,
-        ]:
-            return overridden or f"str({value_name})"
-        elif origin_type is Decimal:
-            return overridden or f"str({value_name})"
-        elif origin_type is Fraction:
-            return overridden or f"str({value_name})"
-
-        raise UnserializableDataError(f"_unpack_value: fname: {fname}, ftype: {ftype}, parent: {parent}, origin_type: {origin_type}")
-
-    def add_unpack_union(self, fname, ftype, parent, variant_types, value_name, metadata):
-        self.add_line(f'def resolve_union(value):')
-        with self.indent():
-            for variant in variant_types:
-                if is_generic(variant):
-                    variant_name = get_type_origin(variant).__name__
-                else:
-                    variant_name = type_name(variant)
-                self.add_line('try:')
-                with self.indent():
-                    packed = self._unpack_field_value(fname, variant, parent, value_name)
-                    self.add_line(f'packed = {packed}')
-                    self.add_line(f'if isinstance(packed, {variant_name}):')
-                    with self.indent():
-                        self.add_line(f'return packed')
-                self.add_line('except (TypeError, AttributeError, ValueError, LookupError) as e:')
-                with self.indent():
-                    self.add_line('pass')
-            else:
-                variant_type_names = ", ".join([type_name(v) for v in variant_types])
-                self.add_line(f"raise ValueError('Union value could not be "
-                              f"decoded using types ({variant_type_names})')")
-            return 'resolve_union(value)'
-
-    def _unpack_field_value(
-        self, fname, ftype, parent, value_name="value", metadata=None
-    ):
-
-        deserialize_option: typing.Optional[typing.Any] = None
-        overridden: typing.Optional[str] = None
-        if metadata is not None:
-            deserialize_option = metadata.get("deserialize")
-            if callable(deserialize_option):
-                setattr(self.cls, f"__{fname}_deserialize", deserialize_option)
-                return f"cls.__{fname}_deserialize({value_name})"
-
-        if is_dataclass(ftype):
-            return (
-                f"{type_name(ftype)}.from_dict({value_name}, "
-                f"use_bytes, use_enum, use_datetime, options)"
-            )
-
-        with suppress(TypeError):
-            if issubclass(ftype, SerializableType):
-                return f"{type_name(ftype)}._deserialize({value_name})"
-        if isinstance(ftype, SerializationStrategy):
-            return (
-                f"cls.__dataclass_fields__['{fname}'].type"
-                f"._deserialize({value_name})"
-            )
-        if type_name(ftype) in self.cls._serializable_encoders:
-            return f"cls._serializable_encoders['{type_name(ftype)}']._deserialize({value_name})"
-
-        origin_type = get_type_origin(ftype)
-        if is_special_typing_primitive(origin_type):
-            if origin_type is typing.Any:
-                return overridden or value_name
-            elif is_union(ftype):
-                args = getattr(ftype, "__args__", ())
-                if len(args) == 2 and args[1] == NoneType:  # it is Optional
-                    return self._unpack_field_value(
-                        fname, args[0], parent, metadata=metadata
-                    )
-                else:
-                    return self.add_unpack_union(fname, ftype, parent, args, value_name, metadata)
-            elif origin_type is typing.AnyStr:
-                raise UnserializableDataError(
-                    "AnyStr is not supported by mashumaro"
-                )
-            elif is_type_var(ftype):
-                raise UnserializableDataError(
-                    "TypeVars are not supported by mashumaro"
-                )
-            else:
-                raise UnserializableDataError(
-                    f"{ftype} as a field type is not supported by mashumaro"
-                )
-        elif origin_type in (bool, NoneType):
-            return overridden or value_name
-        elif issubclass(origin_type, typing.Collection):
-            args = getattr(ftype, "__args__", ())
-
-            def inner_expr(arg_num=0, v_name="value"):
-                return self._unpack_field_value(
-                    fname, args[arg_num], parent, v_name
-                )
-
-            if issubclass(origin_type, typing.List):
-                if is_generic(ftype):
-                    return (
-                        overridden
-                        or f"[{inner_expr()} for value in {value_name}]"
-                    )
-                elif ftype is list:
-                    raise UnserializableField(
-                        fname, ftype, parent, "Use typing.List[T] instead"
-                    )
-            elif issubclass(origin_type, typing.Deque):
-                if is_generic(ftype):
-                    return (
-                        overridden
-                        or f"collections.deque([{inner_expr()} "
-                        f"for value in {value_name}])"
-                    )
-                elif ftype is collections.deque:
-                    raise UnserializableField(
-                        fname, ftype, parent, "Use typing.Deque[T] instead"
-                    )
-            elif issubclass(origin_type, typing.Tuple):
-                if is_generic(ftype):
-                    return (
-                        overridden
-                        or f"tuple([{inner_expr()} for value in {value_name}])"
-                    )
-                elif ftype is tuple:
-                    raise UnserializableField(
-                        fname, ftype, parent, "Use typing.Tuple[T] instead"
-                    )
-            elif issubclass(origin_type, typing.FrozenSet):
-                if is_generic(ftype):
-                    return (
-                        overridden
-                        or f"frozenset([{inner_expr()} "
-                        f"for value in {value_name}])"
-                    )
-                elif ftype is frozenset:
-                    raise UnserializableField(
-                        fname, ftype, parent, "Use typing.FrozenSet[T] instead"
-                    )
-            elif issubclass(origin_type, typing.AbstractSet):
-                if is_generic(ftype):
-                    return (
-                        overridden
-                        or f"set([{inner_expr()} for value in {value_name}])"
-                    )
-                elif ftype is set:
-                    raise UnserializableField(
-                        fname, ftype, parent, "Use typing.Set[T] instead"
-                    )
-            elif issubclass(origin_type, typing.ChainMap):
-                if ftype is collections.ChainMap:
-                    raise UnserializableField(
-                        fname,
-                        ftype,
-                        parent,
-                        "Use typing.ChainMap[KT,VT] instead",
-                    )
-                elif is_generic(ftype):
-                    if is_dataclass(args[0]):
-                        raise UnserializableDataError(
-                            "ChainMaps with dataclasses as keys "
-                            "are not supported by mashumaro"
-                        )
-                    else:
-                        return (
-                            overridden
-                            or f"collections.ChainMap("
-                            f'*[{{{inner_expr(0,"key")}:{inner_expr(1)} '
-                            f"for key, value in m.items()}} "
-                            f"for m in {value_name}])"
-                        )
-            elif issubclass(origin_type, str):
-                return overridden or value_name
-            elif issubclass(origin_type, typing.Mapping):
-                if ftype is dict:
-                    raise UnserializableField(
-                        fname,
-                        ftype,
-                        parent,
-                        "Use typing.Dict[KT,VT] or Mapping[KT,VT] instead",
-                    )
-                elif is_generic(ftype):
-                    if is_dataclass(args[0]):
-                        raise UnserializableDataError(
-                            "Mappings with dataclasses as keys "
-                            "are not supported by mashumaro"
-                        )
-                    else:
-                        return (
-                            overridden
-                            or f'{{{inner_expr(0,"key")}: {inner_expr(1)} '
-                            f"for key, value in {value_name}.items()}}"
-                        )
-            elif issubclass(origin_type, typing.ByteString):
-                if origin_type is bytes:
-                    specific = f"decodebytes({value_name}.encode())"
-                    return (
-                        f"{value_name} if use_bytes else "
-                        f"{overridden or specific}"
-                    )
-                elif origin_type is bytearray:
-                    if overridden:
-                        overridden = (
-                            f"bytearray({value_name}) if use_bytes else "
-                            f"{overridden}"
-                        )
-                    specific = (
-                        f"bytearray({value_name} if use_bytes else "
-                        f"decodebytes({value_name}.encode()))"
-                    )
-                    return overridden or specific
-            elif issubclass(origin_type, typing.Sequence):
-                if is_generic(ftype):
-                    return (
-                        overridden
-                        or f"[{inner_expr()} for value in {value_name}]"
-                    )
-        elif issubclass(origin_type, os.PathLike):
-            if overridden:
-                return overridden
-            elif issubclass(origin_type, pathlib.PosixPath):
-                return f"pathlib.PosixPath({value_name})"
-            elif issubclass(origin_type, pathlib.WindowsPath):
-                return f"pathlib.WindowsPath({value_name})"
-            elif issubclass(origin_type, pathlib.Path):
-                return f"pathlib.Path({value_name})"
-            elif issubclass(origin_type, pathlib.PurePosixPath):
-                return f"pathlib.PurePosixPath({value_name})"
-            elif issubclass(origin_type, pathlib.PureWindowsPath):
-                return f"pathlib.PureWindowsPath({value_name})"
-            elif issubclass(origin_type, pathlib.PurePath):
-                return f"pathlib.PurePath({value_name})"
-            elif origin_type is os.PathLike:
-                return f"pathlib.PurePath({value_name})"
-            else:
-                return f"{type_name(origin_type)}({value_name})"
-        elif issubclass(origin_type, enum.Enum):
-            specific = f"{type_name(origin_type)}({value_name})"
-            return f"{value_name} if use_enum else {overridden or specific}"
-        elif origin_type is int:
-            return overridden or f"int({value_name})"
-        elif origin_type is float:
-            return overridden or f"float({value_name})"
-        elif origin_type in (datetime.datetime, datetime.date, datetime.time):
-            if overridden:
-                return f"{value_name} if use_datetime else {overridden}"
-            elif deserialize_option is not None:
-                if deserialize_option == "ciso8601":
-                    self.ensure_module_imported("ciso8601")
-                    datetime_parser = "ciso8601.parse_datetime"
-                elif deserialize_option == "pendulum":
-                    self.ensure_module_imported("pendulum")
-                    datetime_parser = "pendulum.parse"
-                else:
-                    raise UnserializableField(
-                        fname,
-                        ftype,
-                        parent,
-                        f"Unsupported deserialization engine "
-                        f'"{deserialize_option}"',
-                    )
-                suffix = ""
-                if origin_type is datetime.date:
-                    suffix = ".date()"
-                elif origin_type is datetime.time:
-                    suffix = ".time()"
-                return (
-                    f"{value_name} if use_datetime else "
-                    f"{datetime_parser}({value_name}){suffix}"
-                )
-            return (
-                f"{value_name} if use_datetime else "
-                f"datetime.{origin_type.__name__}."
-                f"fromisoformat({value_name})"
-            )
-        elif origin_type is datetime.timedelta:
-            return overridden or f"datetime.timedelta(seconds={value_name})"
-        elif origin_type is datetime.timezone:
-            return overridden or f"parse_timezone({value_name})"
-        elif origin_type is uuid.UUID:
-            return overridden or f"uuid.UUID({value_name})"
-        elif origin_type is ipaddress.IPv4Address:
-            return overridden or f"ipaddress.IPv4Address({value_name})"
-        elif origin_type is ipaddress.IPv6Address:
-            return overridden or f"ipaddress.IPv6Address({value_name})"
-        elif origin_type is ipaddress.IPv4Network:
-            return overridden or f"ipaddress.IPv4Network({value_name})"
-        elif origin_type is ipaddress.IPv6Network:
-            return overridden or f"ipaddress.IPv6Network({value_name})"
-        elif origin_type is ipaddress.IPv4Interface:
-            return overridden or f"ipaddress.IPv4Interface({value_name})"
-        elif origin_type is ipaddress.IPv6Interface:
-            return overridden or f"ipaddress.IPv6Interface({value_name})"
-        elif origin_type is Decimal:
-            return overridden or f"Decimal({value_name})"
-        elif origin_type is Fraction:
-            return overridden or f"Fraction({value_name})"
-
-        raise UnserializableField(fname, ftype, parent)

core/mashumaro/serializer/json.py

@@ -1,53 +0,0 @@
-import json
-from types import MappingProxyType
-from typing import Any, Dict, Mapping, Type, TypeVar, Union
-
-from typing_extensions import Protocol
-
-from mashumaro.serializer.base import DataClassDictMixin
-
-DEFAULT_DICT_PARAMS = {
-    "use_bytes": False,
-    "use_enum": False,
-    "use_datetime": False,
-}
-EncodedData = Union[str, bytes, bytearray]
-T = TypeVar("T", bound="DataClassJSONMixin")
-
-
-class Encoder(Protocol):  # pragma no cover
-    def __call__(self, obj, **kwargs) -> EncodedData:
-        ...
-
-
-class Decoder(Protocol):  # pragma no cover
-    def __call__(self, s: EncodedData, **kwargs) -> Dict[Any, Any]:
-        ...
-
-
-class DataClassJSONMixin(DataClassDictMixin):
-    def to_json(
-        self: T,
-        encoder: Encoder = json.dumps,
-        dict_params: Mapping = MappingProxyType({}),
-        **encoder_kwargs,
-    ) -> EncodedData:
-
-        return encoder(
-            self.to_dict(**dict(DEFAULT_DICT_PARAMS, **dict_params)),
-            **encoder_kwargs,
-        )
-
-    @classmethod
-    def from_json(
-        cls: Type[T],
-        data: EncodedData,
-        decoder: Decoder = json.loads,
-        dict_params: Mapping = MappingProxyType({}),
-        **decoder_kwargs,
-    ) -> T:
-
-        return cls.from_dict(
-            decoder(data, **decoder_kwargs),
-            **dict(DEFAULT_DICT_PARAMS, **dict_params),
-        )

core/mashumaro/serializer/msgpack.py

@@ -1,53 +0,0 @@
-from functools import partial
-from types import MappingProxyType
-from typing import Any, Dict, Mapping, Type, TypeVar, Union
-
-import msgpack
-from typing_extensions import Protocol
-
-from mashumaro.serializer.base import DataClassDictMixin
-
-DEFAULT_DICT_PARAMS = {
-    "use_bytes": True,
-    "use_enum": False,
-    "use_datetime": False,
-}
-EncodedData = Union[str, bytes, bytearray]
-T = TypeVar("T", bound="DataClassMessagePackMixin")
-
-
-class Encoder(Protocol):  # pragma no cover
-    def __call__(self, o, **kwargs) -> EncodedData:
-        ...
-
-
-class Decoder(Protocol):  # pragma no cover
-    def __call__(self, packed: EncodedData, **kwargs) -> Dict[Any, Any]:
-        ...
-
-
-class DataClassMessagePackMixin(DataClassDictMixin):
-    def to_msgpack(
-        self: T,
-        encoder: Encoder = partial(msgpack.packb, use_bin_type=True),
-        dict_params: Mapping = MappingProxyType({}),
-        **encoder_kwargs,
-    ) -> EncodedData:
-
-        return encoder(
-            self.to_dict(**dict(DEFAULT_DICT_PARAMS, **dict_params)),
-            **encoder_kwargs,
-        )
-
-    @classmethod
-    def from_msgpack(
-        cls: Type[T],
-        data: EncodedData,
-        decoder: Decoder = partial(msgpack.unpackb, raw=False),
-        dict_params: Mapping = MappingProxyType({}),
-        **decoder_kwargs,
-    ) -> T:
-        return cls.from_dict(
-            decoder(data, **decoder_kwargs),
-            **dict(DEFAULT_DICT_PARAMS, **dict_params),
-        )

core/mashumaro/serializer/yaml.py

@@ -1,43 +0,0 @@
-from types import MappingProxyType
-from typing import Callable, Dict, Mapping, Type, TypeVar, Union
-
-import yaml
-
-from mashumaro.serializer.base import DataClassDictMixin
-
-DEFAULT_DICT_PARAMS = {
-    "use_bytes": False,
-    "use_enum": False,
-    "use_datetime": False,
-}
-EncodedData = Union[str, bytes]
-Encoder = Callable[[Dict], EncodedData]
-Decoder = Callable[[EncodedData], Dict]
-T = TypeVar("T", bound="DataClassYAMLMixin")
-
-
-class DataClassYAMLMixin(DataClassDictMixin):
-    def to_yaml(
-        self: T,
-        encoder: Encoder = yaml.dump,
-        dict_params: Mapping = MappingProxyType({}),
-        **encoder_kwargs,
-    ) -> EncodedData:
-
-        return encoder(
-            self.to_dict(**dict(DEFAULT_DICT_PARAMS, **dict_params)),
-            **encoder_kwargs,
-        )
-
-    @classmethod
-    def from_yaml(
-        cls: Type[T],
-        data: EncodedData,
-        decoder: Decoder = yaml.safe_load,
-        dict_params: Mapping = MappingProxyType({}),
-        **decoder_kwargs,
-    ) -> T:
-        return cls.from_dict(
-            decoder(data, **decoder_kwargs),
-            **dict(DEFAULT_DICT_PARAMS, **dict_params),
-        )

core/mashumaro/types.py

@@ -1,58 +0,0 @@
-# this file has been modified for use with dbt
-import decimal
-from typing import TypeVar, Generic
-
-TV = TypeVar("TV")
-
-class SerializableEncoder(Generic[TV]):
-    @classmethod
-    def _serialize(cls, value):
-        raise NotImplementedError
-
-    @classmethod
-    def _deserialize(cls, value):
-        raise NotImplementedError
-
-class SerializableType:
-    def _serialize(self):
-        raise NotImplementedError
-
-    @classmethod
-    def _deserialize(cls, value):
-        raise NotImplementedError
-
-
-class SerializationStrategy:
-    def _serialize(self, value):
-        raise NotImplementedError
-
-    def _deserialize(self, value):
-        raise NotImplementedError
-
-
-class RoundedDecimal(SerializationStrategy):
-    def __init__(self, places=None, rounding=None):
-        if places is not None:
-            self.exp = decimal.Decimal((0, (1,), -places))
-        else:
-            self.exp = None
-        self.rounding = rounding
-
-    def _serialize(self, value) -> str:
-        if self.exp:
-            if self.rounding:
-                return str(value.quantize(self.exp, rounding=self.rounding))
-            else:
-                return str(value.quantize(self.exp))
-        else:
-            return str(value)
-
-    def _deserialize(self, value: str) -> decimal.Decimal:
-        return decimal.Decimal(str(value))
-
-
-__all__ = [
-    "SerializableType",
-    "SerializationStrategy",
-    "RoundedDecimal",
-]

core/setup.py

@@ -24,7 +24,7 @@ def read(fname):
 
 
 package_name = "dbt-core"
-package_version = "0.19.1b2"
+package_version = "0.19.1rc2"
 description = """dbt (data build tool) is a command line tool that helps \
 analysts and engineers transform data in their warehouse more effectively"""
 
@@ -37,12 +37,7 @@ setup(
     author="Fishtown Analytics",
     author_email="info@fishtownanalytics.com",
     url="https://github.com/fishtown-analytics/dbt",
-    packages=find_namespace_packages(include=[
-        'dbt',
-        'dbt.*',
-        'mashumaro',
-        'mashumaro.*'
-    ]),
+    packages=find_namespace_packages(include=['dbt', 'dbt.*']),
     package_data={
         'dbt': [
             'include/index.html',
@@ -70,21 +65,19 @@ setup(
         'networkx>=2.3,<3',
         'minimal-snowplow-tracker==0.0.2',
         'colorama>=0.3.9,<0.4.4',
-        'agate>=1.6,<2',
+        'agate>=1.6,<1.6.2',
         'isodate>=0.6,<0.7',
         'json-rpc>=1.12,<2',
         'werkzeug>=0.15,<2.0',
         'dataclasses==0.6;python_version<"3.7"',
         'hologram==0.0.13',
         'logbook>=1.5,<1.6',
+        'mashumaro==2.0',
         'typing-extensions>=3.7.4,<3.8',
         # the following are all to match snowflake-connector-python
         'requests>=2.18.0,<2.24.0',
         'idna<2.10',
         'cffi>=1.9,<1.15',
-        # the following are pulled in from mashumaro
-        "backports-datetime-fromisoformat;python_version=='3.6'",
-        "msgpack>=0.5.6",
     ],
     zip_safe=False,
     classifiers=[

dev_requirements.txt

@@ -13,4 +13,3 @@ mypy==0.782
 wheel
 twine
 pytest-logbook>=1.2.0,<1.3
-jsonschema

docker/requirements/requirements.0.19.1b1.txt

@@ -0,0 +1,73 @@
+agate==1.6.1
+asn1crypto==1.4.0
+attrs==20.3.0
+azure-common==1.1.26
+azure-core==1.11.0
+azure-storage-blob==12.7.1
+Babel==2.9.0
+boto3==1.15.18
+botocore==1.18.18
+cachetools==4.2.1
+certifi==2020.12.5
+cffi==1.14.5
+chardet==3.0.4
+colorama==0.4.3
+cryptography==3.4.4
+decorator==4.4.2
+google-api-core==1.23.0
+google-auth==1.26.1
+google-cloud-bigquery==2.3.1
+google-cloud-core==1.4.4
+google-crc32c==1.1.2
+google-resumable-media==1.2.0
+googleapis-common-protos==1.52.0
+grpcio==1.35.0
+hologram==0.0.13
+idna==2.9
+importlib-metadata==3.4.0
+isodate==0.6.0
+jeepney==0.6.0
+Jinja2==2.11.2
+jmespath==0.10.0
+json-rpc==1.13.0
+jsonschema==3.1.1
+keyring==21.8.0
+leather==0.3.3
+Logbook==1.5.3
+MarkupSafe==1.1.1
+mashumaro @ https://github.com/fishtown-analytics/dbt-mashumaro/archive/c25ed077e7982e2bfe8aee7df80bfeac63d3927f.tar.gz
+minimal-snowplow-tracker==0.0.2
+msgpack==1.0.2
+msrest==0.6.21
+networkx==2.5
+oauthlib==3.1.0
+oscrypto==1.2.1
+parsedatetime==2.6
+proto-plus==1.13.0
+protobuf==3.14.0
+psycopg2-binary==2.8.6
+pyasn1==0.4.8
+pyasn1-modules==0.2.8
+pycparser==2.20
+pycryptodomex==3.10.1
+PyJWT==1.7.1
+pyOpenSSL==20.0.1
+pyrsistent==0.17.3
+python-dateutil==2.8.1
+python-slugify==4.0.1
+pytimeparse==1.1.8
+pytz==2020.5
+PyYAML==5.4.1
+requests==2.23.0
+requests-oauthlib==1.3.0
+rsa==4.7
+s3transfer==0.3.4
+SecretStorage==3.3.1
+six==1.15.0
+snowflake-connector-python==2.3.6
+sqlparse==0.3.1
+text-unidecode==1.3
+typing-extensions==3.7.4.3
+urllib3==1.25.11
+Werkzeug==1.0.1
+zipp==3.4.0

docker/requirements/requirements.0.19.1rc1.txt

@@ -0,0 +1,73 @@
+agate==1.6.1
+asn1crypto==1.4.0
+attrs==20.3.0
+azure-common==1.1.26
+azure-core==1.12.0
+azure-storage-blob==12.8.0
+Babel==2.9.0
+boto3==1.15.18
+botocore==1.18.18
+cachetools==4.2.1
+certifi==2020.12.5
+cffi==1.14.5
+chardet==3.0.4
+colorama==0.4.3
+cryptography==3.4.6
+decorator==4.4.2
+google-api-core==1.23.0
+google-auth==1.27.1
+google-cloud-bigquery==2.3.1
+google-cloud-core==1.4.4
+google-crc32c==1.1.2
+google-resumable-media==1.2.0
+googleapis-common-protos==1.52.0
+grpcio==1.36.1
+hologram==0.0.13
+idna==2.9
+importlib-metadata==3.7.3
+isodate==0.6.0
+jeepney==0.6.0
+Jinja2==2.11.2
+jmespath==0.10.0
+json-rpc==1.13.0
+jsonschema==3.1.1
+keyring==21.8.0
+leather==0.3.3
+Logbook==1.5.3
+MarkupSafe==1.1.1
+mashumaro==2.0
+minimal-snowplow-tracker==0.0.2
+msgpack==1.0.2
+msrest==0.6.21
+networkx==2.5
+oauthlib==3.1.0
+oscrypto==1.2.1
+parsedatetime==2.6
+proto-plus==1.17.0
+protobuf==3.15.6
+psycopg2-binary==2.8.6
+pyasn1==0.4.8
+pyasn1-modules==0.2.8
+pycparser==2.20
+pycryptodomex==3.10.1
+PyJWT==1.7.1
+pyOpenSSL==20.0.1
+pyrsistent==0.17.3
+python-dateutil==2.8.1
+python-slugify==4.0.1
+pytimeparse==1.1.8
+pytz==2020.5
+PyYAML==5.4.1
+requests==2.23.0
+requests-oauthlib==1.3.0
+rsa==4.7.2
+s3transfer==0.3.4
+SecretStorage==3.3.1
+six==1.15.0
+snowflake-connector-python==2.3.6
+sqlparse==0.3.1
+text-unidecode==1.3
+typing-extensions==3.7.4.3
+urllib3==1.25.11
+Werkzeug==1.0.1
+zipp==3.4.1

docker/requirements/requirements.0.19.1rc2.txt

@@ -0,0 +1,73 @@
+agate==1.6.1
+asn1crypto==1.4.0
+attrs==20.3.0
+azure-common==1.1.26
+azure-core==1.12.0
+azure-storage-blob==12.8.0
+Babel==2.9.0
+boto3==1.15.18
+botocore==1.18.18
+cachetools==4.2.1
+certifi==2020.12.5
+cffi==1.14.5
+chardet==3.0.4
+colorama==0.4.3
+cryptography==3.4.6
+decorator==4.4.2
+google-api-core==1.23.0
+google-auth==1.28.0
+google-cloud-bigquery==2.3.1
+google-cloud-core==1.4.4
+google-crc32c==1.1.2
+google-resumable-media==1.2.0
+googleapis-common-protos==1.52.0
+grpcio==1.36.1
+hologram==0.0.13
+idna==2.9
+importlib-metadata==3.7.3
+isodate==0.6.0
+jeepney==0.6.0
+Jinja2==2.11.2
+jmespath==0.10.0
+json-rpc==1.13.0
+jsonschema==3.1.1
+keyring==21.8.0
+leather==0.3.3
+Logbook==1.5.3
+MarkupSafe==1.1.1
+mashumaro==2.0
+minimal-snowplow-tracker==0.0.2
+msgpack==1.0.2
+msrest==0.6.21
+networkx==2.5
+oauthlib==3.1.0
+oscrypto==1.2.1
+parsedatetime==2.6
+proto-plus==1.18.1
+protobuf==3.15.6
+psycopg2-binary==2.8.6
+pyasn1==0.4.8
+pyasn1-modules==0.2.8
+pycparser==2.20
+pycryptodomex==3.10.1
+PyJWT==1.7.1
+pyOpenSSL==20.0.1
+pyrsistent==0.17.3
+python-dateutil==2.8.1
+python-slugify==4.0.1
+pytimeparse==1.1.8
+pytz==2020.5
+PyYAML==5.4.1
+requests==2.23.0
+requests-oauthlib==1.3.0
+rsa==4.7.2
+s3transfer==0.3.6
+SecretStorage==3.3.1
+six==1.15.0
+snowflake-connector-python==2.3.6
+sqlparse==0.3.1
+text-unidecode==1.3
+typing-extensions==3.7.4.3
+urllib3==1.25.11
+Werkzeug==1.0.1
+zipp==3.4.1

plugins/bigquery/dbt/adapters/bigquery/__version__.py

@@ -1 +1 @@
-version = '0.19.1b2'
+version = '0.19.1rc2'

plugins/bigquery/dbt/adapters/bigquery/connections.py

@@ -48,14 +48,14 @@ RETRYABLE_ERRORS = (
 
 
 @lru_cache()
-def get_bigquery_defaults() -> Tuple[Any, Optional[str]]:
+def get_bigquery_defaults(scopes=None) -> Tuple[Any, Optional[str]]:
     """
     Returns (credentials, project_id)
 
     project_id is returned available from the environment; otherwise None
     """
     # Cached, because the underlying implementation shells out, taking ~1s
-    return google.auth.default()
+    return google.auth.default(scopes=scopes)
 
 
 class Priority(StrEnum):
@@ -201,7 +201,7 @@ class BigQueryConnectionManager(BaseConnectionManager):
         creds = GoogleServiceAccountCredentials.Credentials
 
         if method == BigQueryConnectionMethod.OAUTH:
-            credentials, _ = get_bigquery_defaults()
+            credentials, _ = get_bigquery_defaults(scopes=cls.SCOPE)
             return credentials
 
         elif method == BigQueryConnectionMethod.SERVICE_ACCOUNT:

plugins/bigquery/setup.py

@@ -20,7 +20,7 @@ except ImportError:
 
 
 package_name = "dbt-bigquery"
-package_version = "0.19.1b2"
+package_version = "0.19.1rc2"
 description = """The bigquery adapter plugin for dbt (data build tool)"""
 
 this_directory = os.path.abspath(os.path.dirname(__file__))

plugins/postgres/dbt/adapters/postgres/__version__.py

@@ -1 +1 @@
-version = '0.19.1b2'
+version = '0.19.1rc2'

plugins/postgres/setup.py

@@ -41,7 +41,7 @@ def _dbt_psycopg2_name():
 
 
 package_name = "dbt-postgres"
-package_version = "0.19.1b2"
+package_version = "0.19.1rc2"
 description = """The postgres adpter plugin for dbt (data build tool)"""
 
 this_directory = os.path.abspath(os.path.dirname(__file__))

plugins/redshift/dbt/adapters/redshift/__version__.py

@@ -1 +1 @@
-version = '0.19.1b2'
+version = '0.19.1rc2'

plugins/redshift/setup.py

@@ -20,7 +20,7 @@ except ImportError:
 
 
 package_name = "dbt-redshift"
-package_version = "0.19.1b2"
+package_version = "0.19.1rc2"
 description = """The redshift adapter plugin for dbt (data build tool)"""
 
 this_directory = os.path.abspath(os.path.dirname(__file__))

plugins/snowflake/dbt/adapters/snowflake/__version__.py

@@ -1 +1 @@
-version = '0.19.1b2'
+version = '0.19.1rc2'

plugins/snowflake/setup.py

@@ -20,7 +20,7 @@ except ImportError:
 
 
 package_name = "dbt-snowflake"
-package_version = "0.19.1b2"
+package_version = "0.19.1rc2"
 description = """The snowflake adapter plugin for dbt (data build tool)"""
 
 this_directory = os.path.abspath(os.path.dirname(__file__))

setup.py

@@ -24,7 +24,7 @@ with open(os.path.join(this_directory, 'README.md')) as f:
 
 
 package_name = "dbt"
-package_version = "0.19.1b2"
+package_version = "0.19.1rc2"
 description = """With dbt, data analysts and engineers can build analytics \
 the way engineers build applications."""
 

test/integration/033_event_tracking_test/test_events.py

@@ -105,6 +105,24 @@ class TestEventTracking(DBTIntegrationTest):
             }]
         return populate
 
+    def resource_counts_context(self):
+       return [
+          {
+              'schema': 'iglu:com.dbt/resource_counts/jsonschema/1-0-0',
+              'data': {
+                  'models': ANY,
+                  'tests': ANY,
+                  'snapshots': ANY,
+                  'analyses': ANY,
+                  'macros': ANY,
+                  'operations': ANY,
+                  'seeds': ANY,
+                  'sources': ANY,
+                  'exposures': ANY,
+              }
+          }
+       ]
+
     def build_context(
         self,
         command,
@@ -231,6 +249,12 @@ class TestEventTrackingSuccess(TestEventTracking):
                 label=ANY,
                 context=ANY
             ),
+            call(
+                category='dbt',
+                action='resource_counts',
+                label=ANY,
+                context=ANY
+            ),
             call(
                 category='dbt',
                 action='invocation',
@@ -242,6 +266,7 @@ class TestEventTrackingSuccess(TestEventTracking):
         expected_contexts = [
             self.build_context('compile', 'start'),
             self.load_context(),
+            self.resource_counts_context(),
             self.build_context('compile', 'end', result_type='ok')
         ]
 
@@ -345,6 +370,12 @@ class TestEventTrackingSuccess(TestEventTracking):
                 label=ANY,
                 context=ANY
             ),
+            call(
+                category='dbt',
+                action='resource_counts',
+                label=ANY,
+                context=ANY
+            ),
             call(
                 category='dbt',
                 action='run_model',
@@ -362,6 +393,7 @@ class TestEventTrackingSuccess(TestEventTracking):
         expected_contexts = [
             self.build_context('seed', 'start'),
             self.load_context(),
+            self.resource_counts_context(),
             seed_context,
             self.build_context('seed', 'end', result_type='ok')
         ]
@@ -383,6 +415,12 @@ class TestEventTrackingSuccess(TestEventTracking):
                 label=ANY,
                 context=ANY
             ),
+            call(
+                category='dbt',
+                action='resource_counts',
+                label=ANY,
+                context=ANY
+            ),
             call(
                 category='dbt',
                 action='run_model',
@@ -412,6 +450,7 @@ class TestEventTrackingSuccess(TestEventTracking):
         expected_contexts = [
             self.build_context('run', 'start'),
             self.load_context(),
+            self.resource_counts_context(),
             self.run_context(
                 hashed_contents='1e5789d34cddfbd5da47d7713aa9191c',
                 model_id='4fbacae0e1b69924b22964b457148fb8',
@@ -455,6 +494,12 @@ class TestEventTrackingSuccess(TestEventTracking):
                 label=ANY,
                 context=ANY
             ),
+            call(
+                category='dbt',
+                action='resource_counts',
+                label=ANY,
+                context=ANY
+            ),
             call(
                 category='dbt',
                 action='run_model',
@@ -472,6 +517,7 @@ class TestEventTrackingSuccess(TestEventTracking):
         expected_contexts = [
             self.build_context('run', 'start'),
             self.load_context(),
+            self.resource_counts_context(),
             self.run_context(
                 hashed_contents='4419e809ce0995d99026299e54266037',
                 model_id='576c3d4489593f00fad42b97c278641e',
@@ -508,6 +554,12 @@ class TestEventTrackingSuccess(TestEventTracking):
                 label=ANY,
                 context=ANY
             ),
+            call(
+                category='dbt',
+                action='resource_counts',
+                label=ANY,
+                context=ANY
+            ),
             call(
                 category='dbt',
                 action='invocation',
@@ -519,6 +571,7 @@ class TestEventTrackingSuccess(TestEventTracking):
         expected_contexts = [
             self.build_context('test', 'start'),
             self.load_context(),
+            self.resource_counts_context(),
             self.build_context('test', 'end', result_type='ok')
         ]
 
@@ -619,6 +672,12 @@ class TestEventTrackingUnableToConnect(TestEventTracking):
                 label=ANY,
                 context=ANY
             ),
+            call(
+                category='dbt',
+                action='resource_counts',
+                label=ANY,
+                context=ANY
+            ),
             call(
                 category='dbt',
                 action='invocation',
@@ -630,6 +689,7 @@ class TestEventTrackingUnableToConnect(TestEventTracking):
         expected_contexts = [
             self.build_context('run', 'start'),
             self.load_context(),
+            self.resource_counts_context(),
             self.build_context('run', 'end', result_type='error')
         ]
 
@@ -666,6 +726,12 @@ class TestEventTrackingSnapshot(TestEventTracking):
                 label=ANY,
                 context=ANY
             ),
+            call(
+                category='dbt',
+                action='resource_counts',
+                label=ANY,
+                context=ANY
+            ),
             call(
                 category='dbt',
                 action='run_model',
@@ -684,6 +750,7 @@ class TestEventTrackingSnapshot(TestEventTracking):
         expected_contexts = [
             self.build_context('snapshot', 'start'),
             self.load_context(),
+            self.resource_counts_context(),
             self.run_context(
                 hashed_contents=ANY,
                 model_id='820793a4def8d8a38d109a9709374849',
@@ -721,6 +788,12 @@ class TestEventTrackingCatalogGenerate(TestEventTracking):
                 label=ANY,
                 context=ANY,
             ),
+            call(
+                category='dbt',
+                action='resource_counts',
+                label=ANY,
+                context=ANY,
+            ),
             call(
                 category='dbt',
                 action='invocation',
@@ -732,6 +805,7 @@ class TestEventTrackingCatalogGenerate(TestEventTracking):
         expected_contexts = [
             self.build_context('generate', 'start'),
             self.load_context(),
+            self.resource_counts_context(),
             self.build_context('generate', 'end', result_type='ok')
         ]
 

test/unit/test_bigquery_adapter.py

@@ -659,7 +659,7 @@ class TestBigQueryAdapter(BaseTestBigQueryAdapter):
         self.assertEqual(
             adapter.parse_partition_by({
                 "field": "ts",
-            }).to_dict(), {
+            }).to_dict(omit_none=True), {
                 "field": "ts",
                 "data_type": "date",
                 "granularity": "day"
@@ -670,7 +670,7 @@ class TestBigQueryAdapter(BaseTestBigQueryAdapter):
             adapter.parse_partition_by({
                 "field": "ts",
                 "data_type": "date",
-            }).to_dict(), {
+            }).to_dict(omit_none=True), {
                 "field": "ts",
                 "data_type": "date",
                 "granularity": "day"
@@ -683,7 +683,7 @@ class TestBigQueryAdapter(BaseTestBigQueryAdapter):
                 "data_type": "date",
                 "granularity": "MONTH"
 
-            }).to_dict(), {
+            }).to_dict(omit_none=True), {
                 "field": "ts",
                 "data_type": "date",
                 "granularity": "MONTH"
@@ -696,7 +696,7 @@ class TestBigQueryAdapter(BaseTestBigQueryAdapter):
                 "data_type": "date",
                 "granularity": "YEAR"
 
-            }).to_dict(), {
+            }).to_dict(omit_none=True), {
                 "field": "ts",
                 "data_type": "date",
                 "granularity": "YEAR"
@@ -709,7 +709,7 @@ class TestBigQueryAdapter(BaseTestBigQueryAdapter):
                 "data_type": "timestamp",
                 "granularity": "HOUR"
 
-            }).to_dict(), {
+            }).to_dict(omit_none=True), {
                 "field": "ts",
                 "data_type": "timestamp",
                 "granularity": "HOUR"
@@ -722,7 +722,8 @@ class TestBigQueryAdapter(BaseTestBigQueryAdapter):
                 "data_type": "timestamp",
                 "granularity": "MONTH"
 
-            }).to_dict(), {
+            }).to_dict(omit_none=True
+                ), {
                 "field": "ts",
                 "data_type": "timestamp",
                 "granularity": "MONTH"
@@ -735,7 +736,7 @@ class TestBigQueryAdapter(BaseTestBigQueryAdapter):
                 "data_type": "timestamp",
                 "granularity": "YEAR"
 
-            }).to_dict(), {
+            }).to_dict(omit_none=True), {
                 "field": "ts",
                 "data_type": "timestamp",
                 "granularity": "YEAR"
@@ -748,7 +749,7 @@ class TestBigQueryAdapter(BaseTestBigQueryAdapter):
                 "data_type": "datetime",
                 "granularity": "HOUR"
 
-            }).to_dict(), {
+            }).to_dict(omit_none=True), {
                 "field": "ts",
                 "data_type": "datetime",
                 "granularity": "HOUR"
@@ -761,7 +762,7 @@ class TestBigQueryAdapter(BaseTestBigQueryAdapter):
                 "data_type": "datetime",
                 "granularity": "MONTH"
 
-            }).to_dict(), {
+            }).to_dict(omit_none=True), {
                 "field": "ts",
                 "data_type": "datetime",
                 "granularity": "MONTH"
@@ -774,7 +775,7 @@ class TestBigQueryAdapter(BaseTestBigQueryAdapter):
                 "data_type": "datetime",
                 "granularity": "YEAR"
 
-            }).to_dict(), {
+            }).to_dict(omit_none=True), {
                 "field": "ts",
                 "data_type": "datetime",
                 "granularity": "YEAR"
@@ -795,7 +796,8 @@ class TestBigQueryAdapter(BaseTestBigQueryAdapter):
                     "end": 100,
                     "interval": 20
                 }
-            }).to_dict(), {
+            }).to_dict(omit_none=True
+                ), {
                 "field": "id",
                 "data_type": "int64",
                 "granularity": "day",

test/unit/test_compiler.py

@@ -62,22 +62,12 @@ class CompilerTest(unittest.TestCase):
 
         self.config = config_from_parts_or_dicts(project_cfg, profile_cfg)
 
-        self._generate_runtime_model_patch = patch.object(dbt.compilation, 'generate_runtime_model')
+        self._generate_runtime_model_patch = patch.object(
+            dbt.compilation, 'generate_runtime_model')
         self.mock_generate_runtime_model = self._generate_runtime_model_patch.start()
 
         inject_adapter(Plugin.adapter(self.config), Plugin)
 
-        # self.mock_adapter = PostgresAdapter MagicMock(type=MagicMock(return_value='postgres'))
-        # self.mock_adapter.Relation =
-        # self.mock_adapter.get_compiler.return_value = dbt.compilation.Compiler
-        # self.mock_plugin = MagicMock(
-        #     adapter=MagicMock(
-        #         credentials=MagicMock(return_value='postgres')
-        #     )
-        # )
-        # inject_adapter(self.mock_adapter, self.mock_plugin)
-        # so we can make an adapter
-
         def mock_generate_runtime_model_context(model, config, manifest):
             def ref(name):
                 result = f'__dbt__CTE__{name}'
@@ -98,7 +88,7 @@ class CompilerTest(unittest.TestCase):
         manifest = Manifest(
             macros={},
             nodes={
-                'model.root.view': CompiledModelNode(
+                'model.root.view': ParsedModelNode(
                     name='view',
                     database='dbt',
                     schema='analytics',
@@ -108,23 +98,13 @@ class CompilerTest(unittest.TestCase):
                     fqn=['root', 'view'],
                     package_name='root',
                     root_path='/usr/src/app',
-                    refs=[],
-                    sources=[],
-                    depends_on=DependsOn(nodes=['model.root.ephemeral']),
                     config=self.model_config,
-                    tags=[],
                     path='view.sql',
                     original_file_path='view.sql',
-                    raw_sql='select * from {{ref("ephemeral")}}',
-                    compiled=True,
-                    extra_ctes_injected=False,
-                    extra_ctes=[InjectedCTE(id='model.root.ephemeral', sql='select * from source_table')],
-                    compiled_sql=(
-                        'with cte as (select * from something_else) '
-                        'select * from __dbt__CTE__ephemeral'),
+                    raw_sql='with cte as (select * from something_else) select * from {{ref("ephemeral")}}',
                     checksum=FileHash.from_contents(''),
                 ),
-                'model.root.ephemeral': CompiledModelNode(
+                'model.root.ephemeral': ParsedModelNode(
                     name='ephemeral',
                     database='dbt',
                     schema='analytics',
@@ -134,18 +114,10 @@ class CompilerTest(unittest.TestCase):
                     fqn=['root', 'ephemeral'],
                     package_name='root',
                     root_path='/usr/src/app',
-                    refs=[],
-                    sources=[],
-                    depends_on=DependsOn(),
                     config=ephemeral_config,
-                    tags=[],
                     path='ephemeral.sql',
                     original_file_path='ephemeral.sql',
                     raw_sql='select * from source_table',
-                    compiled=True,
-                    compiled_sql='select * from source_table',
-                    extra_ctes_injected=False,
-                    extra_ctes=[],
                     checksum=FileHash.from_contents(''),
                 ),
             },
@@ -158,11 +130,8 @@ class CompilerTest(unittest.TestCase):
         )
 
         compiler = dbt.compilation.Compiler(self.config)
-        result, _ = compiler._recursively_prepend_ctes(
-            manifest.nodes['model.root.view'],
-            manifest,
-            {}
-        )
+        result = compiler.compile_node(
+            manifest.nodes['model.root.view'], manifest, write=False)
 
         self.assertEqual(result, manifest.nodes['model.root.view'])
         self.assertEqual(result.extra_ctes_injected, True)
@@ -172,16 +141,15 @@ class CompilerTest(unittest.TestCase):
              'select * from source_table'
              '), cte as (select * from something_else) '
              'select * from __dbt__CTE__ephemeral'))
-
         self.assertEqual(
             manifest.nodes['model.root.ephemeral'].extra_ctes_injected,
-            False)
+            True)
 
     def test__prepend_ctes__no_ctes(self):
         manifest = Manifest(
             macros={},
             nodes={
-                'model.root.view': CompiledModelNode(
+                'model.root.view': ParsedModelNode(
                     name='view',
                     database='dbt',
                     schema='analytics',
@@ -191,23 +159,14 @@ class CompilerTest(unittest.TestCase):
                     fqn=['root', 'view'],
                     package_name='root',
                     root_path='/usr/src/app',
-                    refs=[],
-                    sources=[],
-                    depends_on=DependsOn(),
                     config=self.model_config,
-                    tags=[],
                     path='view.sql',
                     original_file_path='view.sql',
                     raw_sql=('with cte as (select * from something_else) '
                              'select * from source_table'),
-                    compiled=True,
-                    extra_ctes_injected=False,
-                    extra_ctes=[],
-                    compiled_sql=('with cte as (select * from something_else) '
-                                  'select * from source_table'),
                     checksum=FileHash.from_contents(''),
                 ),
-                'model.root.view_no_cte': CompiledModelNode(
+                'model.root.view_no_cte': ParsedModelNode(
                     name='view_no_cte',
                     database='dbt',
                     schema='analytics',
@@ -217,18 +176,10 @@ class CompilerTest(unittest.TestCase):
                     fqn=['root', 'view_no_cte'],
                     package_name='root',
                     root_path='/usr/src/app',
-                    refs=[],
-                    sources=[],
-                    depends_on=DependsOn(),
                     config=self.model_config,
-                    tags=[],
                     path='view.sql',
                     original_file_path='view.sql',
                     raw_sql='select * from source_table',
-                    compiled=True,
-                    extra_ctes_injected=False,
-                    extra_ctes=[],
-                    compiled_sql=('select * from source_table'),
                     checksum=FileHash.from_contents(''),
                 ),
             },
@@ -241,11 +192,8 @@ class CompilerTest(unittest.TestCase):
         )
 
         compiler = dbt.compilation.Compiler(self.config)
-        result, _ = compiler._recursively_prepend_ctes(
-            manifest.nodes['model.root.view'],
-            manifest,
-            {}
-        )
+        result = compiler.compile_node(
+            manifest.nodes['model.root.view'], manifest, write=False)
 
         self.assertEqual(
             result,
@@ -253,13 +201,13 @@ class CompilerTest(unittest.TestCase):
         self.assertTrue(result.extra_ctes_injected)
         self.assertEqualIgnoreWhitespace(
             result.compiled_sql,
-            manifest.nodes.get('model.root.view').compiled_sql)
+            ('with cte as (select * from something_else) '
+             'select * from source_table')
+        )
 
         compiler = dbt.compilation.Compiler(self.config)
-        result, _ = compiler._recursively_prepend_ctes(
-            manifest.nodes.get('model.root.view_no_cte'),
-            manifest,
-            {})
+        result = compiler.compile_node(
+            manifest.nodes['model.root.view_no_cte'], manifest, write=False)
 
         self.assertEqual(
             result,
@@ -267,7 +215,8 @@ class CompilerTest(unittest.TestCase):
         self.assertTrue(result.extra_ctes_injected)
         self.assertEqualIgnoreWhitespace(
             result.compiled_sql,
-            manifest.nodes.get('model.root.view_no_cte').compiled_sql)
+            'select * from source_table'
+        )
 
     def test__prepend_ctes(self):
         ephemeral_config = self.model_config.replace(materialized='ephemeral')
@@ -275,7 +224,7 @@ class CompilerTest(unittest.TestCase):
         manifest = Manifest(
             macros={},
             nodes={
-                'model.root.view': CompiledModelNode(
+                'model.root.view': ParsedModelNode(
                     name='view',
                     database='dbt',
                     schema='analytics',
@@ -285,21 +234,13 @@ class CompilerTest(unittest.TestCase):
                     fqn=['root', 'view'],
                     package_name='root',
                     root_path='/usr/src/app',
-                    refs=[],
-                    sources=[],
-                    depends_on=DependsOn(nodes=['model.root.ephemeral']),
                     config=self.model_config,
-                    tags=[],
                     path='view.sql',
                     original_file_path='view.sql',
                     raw_sql='select * from {{ref("ephemeral")}}',
-                    compiled=True,
-                    extra_ctes_injected=False,
-                    extra_ctes=[InjectedCTE(id='model.root.ephemeral', sql='select * from source_table')],
-                    compiled_sql='select * from __dbt__CTE__ephemeral',
                     checksum=FileHash.from_contents(''),
                 ),
-                'model.root.ephemeral': CompiledModelNode(
+                'model.root.ephemeral': ParsedModelNode(
                     name='ephemeral',
                     database='dbt',
                     schema='analytics',
@@ -309,18 +250,10 @@ class CompilerTest(unittest.TestCase):
                     fqn=['root', 'ephemeral'],
                     package_name='root',
                     root_path='/usr/src/app',
-                    refs=[],
-                    sources=[],
-                    depends_on=DependsOn(),
                     config=ephemeral_config,
-                    tags=[],
                     path='ephemeral.sql',
                     original_file_path='ephemeral.sql',
                     raw_sql='select * from source_table',
-                    compiled=True,
-                    extra_ctes_injected=False,
-                    extra_ctes=[],
-                    compiled_sql='select * from source_table',
                     checksum=FileHash.from_contents(''),
                 ),
             },
@@ -333,15 +266,11 @@ class CompilerTest(unittest.TestCase):
         )
 
         compiler = dbt.compilation.Compiler(self.config)
-        result, _ = compiler._recursively_prepend_ctes(
+        result = compiler.compile_node(
             manifest.nodes['model.root.view'],
             manifest,
-            {}
+            write=False
         )
-
-        self.assertEqual(result,
-                         manifest.nodes.get('model.root.view'))
-
         self.assertTrue(result.extra_ctes_injected)
         self.assertEqualIgnoreWhitespace(
             result.compiled_sql,
@@ -349,9 +278,8 @@ class CompilerTest(unittest.TestCase):
              'select * from source_table'
              ') '
              'select * from __dbt__CTE__ephemeral'))
-        print(f"\n---- line 349 ----")
-
-        self.assertFalse(manifest.nodes['model.root.ephemeral'].extra_ctes_injected)
+        self.assertTrue(
+            manifest.nodes['model.root.ephemeral'].extra_ctes_injected)
 
     def test__prepend_ctes__cte_not_compiled(self):
         ephemeral_config = self.model_config.replace(materialized='ephemeral')
@@ -422,7 +350,8 @@ class CompilerTest(unittest.TestCase):
                     raw_sql='select * from {{ref("ephemeral")}}',
                     compiled=True,
                     extra_ctes_injected=False,
-                    extra_ctes=[InjectedCTE(id='model.root.ephemeral', sql='select * from source_table')],
+                    extra_ctes=[InjectedCTE(
+                        id='model.root.ephemeral', sql='select * from source_table')],
                     compiled_sql='select * from __dbt__CTE__ephemeral',
                     checksum=FileHash.from_contents(''),
                 ),
@@ -445,7 +374,8 @@ class CompilerTest(unittest.TestCase):
                 manifest,
                 {}
             )
-            compile_node.assert_called_once_with(parsed_ephemeral, manifest, {})
+            compile_node.assert_called_once_with(
+                parsed_ephemeral, manifest, {})
 
         self.assertEqual(result,
                          manifest.nodes.get('model.root.view'))
@@ -459,7 +389,8 @@ class CompilerTest(unittest.TestCase):
              ') '
              'select * from __dbt__CTE__ephemeral'))
 
-        self.assertTrue(manifest.nodes['model.root.ephemeral'].extra_ctes_injected)
+        self.assertTrue(
+            manifest.nodes['model.root.ephemeral'].extra_ctes_injected)
 
     def test__prepend_ctes__multiple_levels(self):
         ephemeral_config = self.model_config.replace(materialized='ephemeral')
@@ -467,7 +398,7 @@ class CompilerTest(unittest.TestCase):
         manifest = Manifest(
             macros={},
             nodes={
-                'model.root.view': CompiledModelNode(
+                'model.root.view': ParsedModelNode(
                     name='view',
                     database='dbt',
                     schema='analytics',
@@ -477,18 +408,10 @@ class CompilerTest(unittest.TestCase):
                     fqn=['root', 'view'],
                     package_name='root',
                     root_path='/usr/src/app',
-                    refs=[],
-                    sources=[],
-                    depends_on=DependsOn(nodes=['model.root.ephemeral']),
                     config=self.model_config,
-                    tags=[],
                     path='view.sql',
                     original_file_path='view.sql',
                     raw_sql='select * from {{ref("ephemeral")}}',
-                    compiled=True,
-                    extra_ctes_injected=False,
-                    extra_ctes=[InjectedCTE(id='model.root.ephemeral', sql=None)],
-                    compiled_sql='select * from __dbt__CTE__ephemeral',
                     checksum=FileHash.from_contents(''),
 
                 ),
@@ -502,11 +425,7 @@ class CompilerTest(unittest.TestCase):
                     fqn=['root', 'ephemeral'],
                     package_name='root',
                     root_path='/usr/src/app',
-                    refs=[],
-                    sources=[],
-                    depends_on=DependsOn(),
                     config=ephemeral_config,
-                    tags=[],
                     path='ephemeral.sql',
                     original_file_path='ephemeral.sql',
                     raw_sql='select * from {{ref("ephemeral_level_two")}}',
@@ -522,11 +441,7 @@ class CompilerTest(unittest.TestCase):
                     fqn=['root', 'ephemeral_level_two'],
                     package_name='root',
                     root_path='/usr/src/app',
-                    refs=[],
-                    sources=[],
-                    depends_on=DependsOn(),
                     config=ephemeral_config,
-                    tags=[],
                     path='ephemeral_level_two.sql',
                     original_file_path='ephemeral_level_two.sql',
                     raw_sql='select * from source_table',
@@ -542,11 +457,7 @@ class CompilerTest(unittest.TestCase):
         )
 
         compiler = dbt.compilation.Compiler(self.config)
-        result, _ = compiler._recursively_prepend_ctes(
-            manifest.nodes['model.root.view'],
-            manifest,
-            {}
-        )
+        result = compiler.compile_node(manifest.nodes['model.root.view'], manifest, write=False)
 
         self.assertEqual(result, manifest.nodes['model.root.view'])
         self.assertTrue(result.extra_ctes_injected)
@@ -560,6 +471,97 @@ class CompilerTest(unittest.TestCase):
              'select * from __dbt__CTE__ephemeral'))
 
         self.assertTrue(manifest.nodes['model.root.ephemeral'].compiled)
-        self.assertTrue(manifest.nodes['model.root.ephemeral_level_two'].compiled)
-        self.assertTrue(manifest.nodes['model.root.ephemeral'].extra_ctes_injected)
-        self.assertTrue(manifest.nodes['model.root.ephemeral_level_two'].extra_ctes_injected)
+        self.assertTrue(
+            manifest.nodes['model.root.ephemeral_level_two'].compiled)
+        self.assertTrue(
+            manifest.nodes['model.root.ephemeral'].extra_ctes_injected)
+        self.assertTrue(
+            manifest.nodes['model.root.ephemeral_level_two'].extra_ctes_injected)
+
+    def test__prepend_ctes__valid_ephemeral_sql(self):
+        """Assert that the compiled sql for ephemeral models is valid and can be executed on its own"""
+        ephemeral_config = self.model_config.replace(materialized='ephemeral')
+
+        manifest = Manifest(
+            macros={},
+            nodes={
+                'model.root.view': ParsedModelNode(
+                    name='view',
+                    database='dbt',
+                    schema='analytics',
+                    alias='view',
+                    resource_type=NodeType.Model,
+                    unique_id='model.root.view',
+                    fqn=['root', 'view'],
+                    package_name='root',
+                    root_path='/usr/src/app',
+                    config=self.model_config,
+                    path='view.sql',
+                    original_file_path='view.sql',
+                    raw_sql='select * from {{ref("ephemeral")}}',
+                    checksum=FileHash.from_contents(''),
+                ),
+                'model.root.inner_ephemeral': ParsedModelNode(
+                    name='inner_ephemeral',
+                    database='dbt',
+                    schema='analytics',
+                    alias='inner_ephemeral',
+                    resource_type=NodeType.Model,
+                    unique_id='model.root.inner_ephemeral',
+                    fqn=['root', 'inner_ephemeral'],
+                    package_name='root',
+                    root_path='/usr/src/app',
+                    config=ephemeral_config,
+                    path='inner_ephemeral.sql',
+                    original_file_path='inner_ephemeral.sql',
+                    raw_sql='select * from source_table',
+                    checksum=FileHash.from_contents(''),
+                ),
+                 'model.root.ephemeral': ParsedModelNode(
+                    name='ephemeral',
+                    database='dbt',
+                    schema='analytics',
+                    alias='ephemeral',
+                    resource_type=NodeType.Model,
+                    unique_id='model.root.ephemeral',
+                    fqn=['root', 'ephemeral'],
+                    package_name='root',
+                    root_path='/usr/src/app',
+                    config=ephemeral_config,
+                    path='ephemeral.sql',
+                    original_file_path='ephemeral.sql',
+                    raw_sql='select * from {{ ref("inner_ephemeral") }}',
+                    checksum=FileHash.from_contents(''),
+                ),
+            },
+            sources={},
+            docs={},
+            disabled=[],
+            files={},
+            exposures={},
+            selectors={},
+        )
+
+        compiler = dbt.compilation.Compiler(self.config)
+        result = compiler.compile_node(
+            manifest.nodes['model.root.view'],
+            manifest,
+            write=False
+        )
+        self.assertEqualIgnoreWhitespace(
+            result.compiled_sql,
+            ('with __dbt__CTE__inner_ephemeral as ('
+             'select * from source_table'
+             '), '
+             '__dbt__CTE__ephemeral as ('
+             'select * from __dbt__CTE__inner_ephemeral'
+             ') '
+             'select * from __dbt__CTE__ephemeral'))
+        self.assertEqualIgnoreWhitespace(
+            manifest.nodes['model.root.ephemeral'].compiled_sql,
+            ('with __dbt__CTE__inner_ephemeral as ('
+            'select * from source_table'
+            ')  '
+            'select * from __dbt__CTE__inner_ephemeral')
+        )
+

test/unit/test_config.py

@@ -776,7 +776,7 @@ class TestProject(BaseConfigTest):
                 LocalPackage(local='foo'),
                 GitPackage(git='git@example.com:fishtown-analytics/dbt-utils.git', revision='test-rev')
             ]))
-        str(project)
+        str(project)  # this does the equivalent of project.to_project_config(with_packages=True)
         json.dumps(project.to_project_config())
 
     def test_string_run_hooks(self):

test/unit/test_contracts_graph_parsed.py

@@ -1618,7 +1618,7 @@ def test_timestamp_snapshot_ok(basic_timestamp_snapshot_dict, basic_timestamp_sn
 
     assert_symmetric(node, node_dict, ParsedSnapshotNode)
     assert_symmetric(inter, node_dict, IntermediateSnapshotNode)
-    assert ParsedSnapshotNode.from_dict(inter.to_dict()) == node
+    assert ParsedSnapshotNode.from_dict(inter.to_dict(omit_none=True)) == node
     assert node.is_refable is True
     assert node.is_ephemeral is False
     pickle.loads(pickle.dumps(node))
@@ -1631,7 +1631,7 @@ def test_check_snapshot_ok(basic_check_snapshot_dict, basic_check_snapshot_objec
 
     assert_symmetric(node, node_dict, ParsedSnapshotNode)
     assert_symmetric(inter, node_dict, IntermediateSnapshotNode)
-    assert ParsedSnapshotNode.from_dict(inter.to_dict()) == node
+    assert ParsedSnapshotNode.from_dict(inter.to_dict(omit_none=True)) == node
     assert node.is_refable is True
     assert node.is_ephemeral is False
     pickle.loads(pickle.dumps(node))

test/unit/test_docs_generate.py

@@ -32,7 +32,7 @@ class GenerateTest(unittest.TestCase):
             sources=sources,
             errors=None,
         )
-        return result.to_dict(options={'keep_none': True})['nodes']
+        return result.to_dict(omit_none=False)['nodes']
 
     def test__unflatten_empty(self):
         columns = {}

test/unit/test_manifest.py

@@ -212,9 +212,9 @@ class ManifestTest(unittest.TestCase):
             ),
         }
         for node in self.nested_nodes.values():
-            node.validate(node.to_dict())
+            node.validate(node.to_dict(omit_none=True))
         for source in self.sources.values():
-            source.validate(source.to_dict())
+            source.validate(source.to_dict(omit_none=True))
 
         os.environ['DBT_ENV_CUSTOM_ENV_key'] = 'value'
 
@@ -229,7 +229,7 @@ class ManifestTest(unittest.TestCase):
             metadata=ManifestMetadata(generated_at=datetime.utcnow()),
         )
         self.assertEqual(
-            manifest.writable_manifest().to_dict(),
+            manifest.writable_manifest().to_dict(omit_none=True),
             {
                 'nodes': {},
                 'sources': {},
@@ -258,7 +258,7 @@ class ManifestTest(unittest.TestCase):
             exposures={}, selectors={},
             metadata=ManifestMetadata(generated_at=datetime.utcnow()),
         )
-        serialized = manifest.writable_manifest().to_dict()
+        serialized = manifest.writable_manifest().to_dict(omit_none=True)
         self.assertEqual(serialized['metadata']['generated_at'], '2018-02-14T09:15:13Z')
         self.assertEqual(serialized['docs'], {})
         self.assertEqual(serialized['disabled'], [])
@@ -371,7 +371,7 @@ class ManifestTest(unittest.TestCase):
                             metadata=metadata, files={}, exposures={})
 
         self.assertEqual(
-            manifest.writable_manifest().to_dict(),
+            manifest.writable_manifest().to_dict(omit_none=True),
             {
                 'nodes': {},
                 'sources': {},
@@ -612,7 +612,7 @@ class MixedManifestTest(unittest.TestCase):
         manifest = Manifest(nodes={}, sources={}, macros={}, docs={}, selectors={},
                             disabled=[], metadata=metadata, files={}, exposures={})
         self.assertEqual(
-            manifest.writable_manifest().to_dict(),
+            manifest.writable_manifest().to_dict(omit_none=True),
             {
                 'nodes': {},
                 'macros': {},
@@ -640,7 +640,7 @@ class MixedManifestTest(unittest.TestCase):
                             disabled=[], selectors={},
                             metadata=ManifestMetadata(generated_at=datetime.utcnow()),
                             files={}, exposures={})
-        serialized = manifest.writable_manifest().to_dict()
+        serialized = manifest.writable_manifest().to_dict(omit_none=True)
         self.assertEqual(serialized['metadata']['generated_at'], '2018-02-14T09:15:13Z')
         self.assertEqual(serialized['disabled'], [])
         parent_map = serialized['parent_map']

test/unit/utils.py

@@ -145,7 +145,7 @@ class ContractTestCase(TestCase):
         super().setUp()
 
     def assert_to_dict(self, obj, dct):
-        self.assertEqual(obj.to_dict(), dct)
+        self.assertEqual(obj.to_dict(omit_none=True), dct)
 
     def assert_from_dict(self, obj, dct, cls=None):
         if cls is None:
@@ -185,7 +185,7 @@ def compare_dicts(dict1, dict2):
 
 
 def assert_to_dict(obj, dct):
-    assert obj.to_dict() == dct
+    assert obj.to_dict(omit_none=True) == dct
 
 
 def assert_from_dict(obj, dct, cls=None):

third-party-stubs/mashumaro/config.pyi

@@ -0,0 +1,10 @@
+from mashumaro.types import SerializationStrategy as SerializationStrategy
+from typing import Any, Callable, Dict, List, Union
+
+TO_DICT_ADD_OMIT_NONE_FLAG: str
+SerializationStrategyValueType = Union[SerializationStrategy, Dict[str, Union[str, Callable]]]
+
+class BaseConfig:
+    debug: bool = ...
+    code_generation_options: List[str] = ...
+    serialization_strategy: Dict[Any, SerializationStrategyValueType] = ...

third-party-stubs/mashumaro/serializer/base/dict.pyi

@@ -2,10 +2,15 @@ from typing import Any, Mapping, Dict, Optional
 
 class DataClassDictMixin:
     def __init_subclass__(cls, **kwargs: Any) -> None: ...
-    def __pre_serialize__(self, options: Optional[Dict[str, Any]]) -> Any: ...
-    def __post_serialize__(self, dct: Mapping, options: Optional[Dict[str, Any]]) -> Any: ...
+    def __pre_serialize__(self) -> Any: ...
+    def __post_serialize__(self, dct: Mapping) -> Any: ...
     @classmethod
-    def __pre_deserialize__(cls: Any, dct: Mapping, options: Optional[Dict[str, Any]]) -> Any: ...
-    def to_dict( self, use_bytes: bool = False, use_enum: bool = False, use_datetime: bool = False, options: Optional[Dict[str, Any]] = None) -> dict: ...
+    def __pre_deserialize__(cls: Any, dct: Mapping) -> Any: ...
+    # This is absolutely totally wrong. This is *not* the signature of the Mashumaro to_dict
+    # But mypy insists that the DataClassDictMixin to_dict and the JsonSchemaMixin to_dict
+    # must have the same signatures now that we have an 'omit_none' flag on the Mashumaro to_dict.
+    # There is no 'validate = False' in Mashumaro.
+    # Could not find a way to tell mypy to ignore it.
+    def to_dict( self, omit_none = False, validate = False) -> dict: ...  
     @classmethod
-    def from_dict( cls, d: Mapping, use_bytes: bool = False, use_enum: bool = False, use_datetime: bool = False, options: Optional[Dict[str, Any]] = None) -> Any: ...
+    def from_dict( cls, d: Mapping, use_bytes: bool = False, use_enum: bool = False, use_datetime: bool = False) -> Any: ...