remove extra code

core/dbt/contracts/graph/manifest.py

@@ -1157,7 +1157,7 @@ AnyManifest = Union[Manifest, MacroManifest]
 
 
 @dataclass
-@schema_version("manifest", 6)
+@schema_version("manifest", 7)
 class WritableManifest(ArtifactMixin):
     nodes: Mapping[UniqueID, ManifestNode] = field(
         metadata=dict(description=("The nodes defined in the dbt project and its dependencies"))
@@ -1203,7 +1203,7 @@ class WritableManifest(ArtifactMixin):
 
     @classmethod
     def compatible_previous_versions(self):
-        return [("manifest", 4), ("manifest", 5)]
+        return [("manifest", 4), ("manifest", 5), ("manifest", 6)]
 
     def __post_serialize__(self, dct):
         for unique_id, node in dct["nodes"].items():

core/dbt/contracts/graph/model_config.py

@@ -7,7 +7,8 @@ from dbt.dataclass_schema import (
     ValidationError,
     register_pattern,
 )
-from dbt.contracts.graph.unparsed import AdditionalPropertiesAllowed
+from dbt.contracts.graph.unparsed import AdditionalPropertiesAllowed, Docs
+from dbt.contracts.graph.utils import validate_color
 from dbt.exceptions import InternalException, CompilationException
 from dbt.contracts.util import Replaceable, list_str
 from dbt import hooks
@@ -285,7 +286,7 @@ class BaseConfig(AdditionalPropertiesAllowed, Replaceable):
     # 'meta' moved here from node
     mergebehavior = {
         "append": ["pre-hook", "pre_hook", "post-hook", "post_hook", "tags"],
-        "update": ["quoting", "column_types", "meta"],
+        "update": ["quoting", "column_types", "meta", "docs"],
         "dict_key_append": ["grants"],
     }
 
@@ -461,6 +462,20 @@ class NodeConfig(NodeAndTestConfig):
     grants: Dict[str, Any] = field(
         default_factory=dict, metadata=MergeBehavior.DictKeyAppend.meta()
     )
+    docs: Docs = field(
+        default_factory=Docs,
+        metadata=MergeBehavior.Update.meta(),
+    )
+
+    # we validate that node_color has a suitable value to prevent dbt-docs from crashing
+    def __post_init__(self):
+        if self.docs.node_color:
+            node_color = self.docs.node_color
+            if not validate_color(node_color):
+                raise ValidationError(
+                    f"Invalid color name for docs.node_color: {node_color}. "
+                    "It is neither a valid HTML color name nor a valid HEX code."
+                )
 
     @classmethod
     def __pre_deserialize__(cls, data):

core/dbt/contracts/graph/parsed.py

@@ -157,7 +157,6 @@ class ParsedNodeMixins(dbtClassMixin):
         self.created_at = time.time()
         self.description = patch.description
         self.columns = patch.columns
-        self.docs = patch.docs
 
     def get_materialization(self):
         return self.config.materialized

core/dbt/contracts/graph/unparsed.py

@@ -76,6 +76,7 @@ class UnparsedRunHook(UnparsedNode):
 @dataclass
 class Docs(dbtClassMixin, Replaceable):
     show: bool = True
+    node_color: Optional[str] = None
 
 
 @dataclass

core/dbt/contracts/graph/utils.py

@@ -0,0 +1,153 @@
+import re
+
+HTML_COLORS = [
+    "aliceblue",
+    "antiquewhite",
+    "aqua",
+    "aquamarine",
+    "azure",
+    "beige",
+    "bisque",
+    "black",
+    "blanchedalmond",
+    "blue",
+    "blueviolet",
+    "brown",
+    "burlywood",
+    "cadetblue",
+    "chartreuse",
+    "chocolate",
+    "coral",
+    "cornflowerblue",
+    "cornsilk",
+    "crimson",
+    "cyan",
+    "darkblue",
+    "darkcyan",
+    "darkgoldenrod",
+    "darkgray",
+    "darkgreen",
+    "darkkhaki",
+    "darkmagenta",
+    "darkolivegreen",
+    "darkorange",
+    "darkorchid",
+    "darkred",
+    "darksalmon",
+    "darkseagreen",
+    "darkslateblue",
+    "darkslategray",
+    "darkturquoise",
+    "darkviolet",
+    "deeppink",
+    "deepskyblue",
+    "dimgray",
+    "dodgerblue",
+    "firebrick",
+    "floralwhite",
+    "forestgreen",
+    "fuchsia",
+    "gainsboro",
+    "ghostwhite",
+    "gold",
+    "goldenrod",
+    "gray",
+    "green",
+    "greenyellow",
+    "honeydew",
+    "hotpink",
+    "indianred",
+    "indigo",
+    "ivory",
+    "khaki",
+    "lavender",
+    "lavenderblush",
+    "lawngreen",
+    "lemonchiffon",
+    "lightblue",
+    "lightcoral",
+    "lightcyan",
+    "lightgoldenrodyellow",
+    "lightgray",
+    "lightgreen",
+    "lightpink",
+    "lightsalmon",
+    "lightsalmon",
+    "lightseagreen",
+    "lightskyblue",
+    "lightslategray",
+    "lightsteelblue",
+    "lightyellow",
+    "lime",
+    "limegreen",
+    "linen",
+    "magenta",
+    "maroon",
+    "mediumaquamarine",
+    "mediumblue",
+    "mediumorchid",
+    "mediumpurple",
+    "mediumseagreen",
+    "mediumslateblue",
+    "mediumslateblue",
+    "mediumspringgreen",
+    "mediumturquoise",
+    "mediumvioletred",
+    "midnightblue",
+    "mintcream",
+    "mistyrose",
+    "moccasin",
+    "navajowhite",
+    "navy",
+    "oldlace",
+    "olive",
+    "olivedrab",
+    "orange",
+    "orangered",
+    "orchid",
+    "palegoldenrod",
+    "palegreen",
+    "paleturquoise",
+    "palevioletred",
+    "papayawhip",
+    "peachpuff",
+    "peru",
+    "pink",
+    "plum",
+    "powderblue",
+    "purple",
+    "rebeccapurple",
+    "red",
+    "rosybrown",
+    "royalblue",
+    "saddlebrown",
+    "salmon",
+    "sandybrown",
+    "seagreen",
+    "seashell",
+    "sienna",
+    "silver",
+    "skyblue",
+    "slateblue",
+    "slategray",
+    "snow",
+    "springgreen",
+    "steelblue",
+    "tan",
+    "teal",
+    "thistle",
+    "tomato",
+    "turquoise",
+    "violet",
+    "wheat",
+    "white",
+    "whitesmoke",
+    "yellow",
+    "yellowgreen",
+]
+
+
+def validate_color(color: str) -> bool:
+    match_hex = re.search(r"^#(?:[0-9a-f]{3}){1,2}$", color.lower())
+    match_html_color_name = color.lower() in HTML_COLORS
+    return bool(match_hex or match_html_color_name)

core/dbt/parser/base.py

@@ -17,7 +17,7 @@ from dbt.config import Project, RuntimeConfig
 from dbt.context.context_config import ContextConfig
 from dbt.contracts.graph.manifest import Manifest
 from dbt.contracts.graph.parsed import HasUniqueID, ManifestNodes
-from dbt.contracts.graph.unparsed import UnparsedNode
+from dbt.contracts.graph.unparsed import UnparsedNode, Docs
 from dbt.exceptions import ParsingException, validator_error_message, InternalException
 from dbt import hooks
 from dbt.node_types import NodeType
@@ -287,6 +287,22 @@ class ConfiguredParser(
         if "meta" in config_dict and config_dict["meta"]:
             parsed_node.meta = config_dict["meta"]
 
+        # If we have docs in the config, merge with the node level, for backwards
+        # compatibility with earlier node-only config.
+        if "docs" in config_dict and config_dict["docs"]:
+            # we set show at the value of the config if it is set, otherwize, inherit the value
+            docs_show = (
+                config_dict["docs"]["show"]
+                if "show" in config_dict["docs"]
+                else parsed_node.docs.show
+            )
+            if "node_color" in config_dict["docs"]:
+                parsed_node.docs = Docs(
+                    show=docs_show, node_color=config_dict["docs"]["node_color"]
+                )
+            else:
+                parsed_node.docs = Docs(show=docs_show)
+
         # unrendered_config is used to compare the original database/schema/alias
         # values and to handle 'same_config' and 'same_contents' calls
         parsed_node.unrendered_config = config.build_config_dict(rendered=False)

core/dbt/parser/schemas.py

@@ -807,6 +807,7 @@ class NonSourceParser(YamlDocsReader, Generic[NonSourceTarget, Parsed]):
                 if self.key != "macros":
                     # macros don't have the 'config' key support yet
                     self.normalize_meta_attribute(data, path)
+                    self.normalize_docs_attribute(data, path)
                 node = self._target_type().from_dict(data)
             except (ValidationError, JSONValidationException) as exc:
                 msg = error_context(path, self.key, data, exc)
@@ -814,21 +815,27 @@ class NonSourceParser(YamlDocsReader, Generic[NonSourceTarget, Parsed]):
             else:
                 yield node
 
-    # We want to raise an error if 'meta' is in two places, and move 'meta'
+    # We want to raise an error if some attributes are in two places, and move them
     # from toplevel to config if necessary
-    def normalize_meta_attribute(self, data, path):
-        if "meta" in data:
-            if "config" in data and "meta" in data["config"]:
+    def normalize_attribute(self, data, path, attribute):
+        if attribute in data:
+            if "config" in data and attribute in data["config"]:
                 raise ParsingException(
                     f"""
-                    In {path}: found meta dictionary in 'config' dictionary and as top-level key.
+                    In {path}: found {attribute} dictionary in 'config' dictionary and as top-level key.
                     Remove the top-level key and define it under 'config' dictionary only.
                 """.strip()
                 )
             else:
                 if "config" not in data:
                     data["config"] = {}
-                data["config"]["meta"] = data.pop("meta")
+                data["config"][attribute] = data.pop(attribute)
+
+    def normalize_meta_attribute(self, data, path):
+        return self.normalize_attribute(data, path, "meta")
+
+    def normalize_docs_attribute(self, data, path):
+        return self.normalize_attribute(data, path, "docs")
 
     def patch_node_config(self, node, patch):
         # Get the ContextConfig that's used in calculating the config

tests/functional/artifacts/expected_manifest.py

@@ -221,7 +221,7 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False):
     model_raw_sql = read_file_replace_returns(model_sql_path).rstrip("\r\n")
 
     return {
-        "dbt_schema_version": "https://schemas.getdbt.com/dbt/manifest/v6.json",
+        "dbt_schema_version": "https://schemas.getdbt.com/dbt/manifest/v7.json",
         "dbt_version": dbt.version.__version__,
         "nodes": {
             "model.test.model": {
@@ -836,7 +836,7 @@ def expected_references_manifest(project):
     alternate_schema = project.test_schema + "_test"
 
     return {
-        "dbt_schema_version": "https://schemas.getdbt.com/dbt/manifest/v6.json",
+        "dbt_schema_version": "https://schemas.getdbt.com/dbt/manifest/v7.json",
         "dbt_version": dbt.version.__version__,
         "nodes": {
             "model.test.ephemeral_copy": {

tests/functional/artifacts/test_previous_version_state.py

@@ -28,7 +28,7 @@ select 1 as id
 
 
 class TestPreviousVersionState:
-    CURRENT_EXPECTED_MANIFEST_VERSION = 6
+    CURRENT_EXPECTED_MANIFEST_VERSION = 7
 
     @pytest.fixture(scope="class")
     def models(self):

tests/functional/configs/test_custom_node_colors_configs.py

@@ -0,0 +1,185 @@
+import pytest
+import os
+
+# from tests.functional.configs.fixtures import BaseConfigProject
+from dbt.tests.util import run_dbt, get_manifest, write_file
+
+
+models__custom_node_colors__model_sql = """
+{{
+    config(
+        materialized='view',
+        node_color='#c0c0c0'
+    )
+}}
+
+select 1 as id
+
+"""
+
+models__no_node_colors__model_sql = """
+{{
+    config(
+        materialized='view'
+    )
+}}
+
+select 1 as id
+
+"""
+
+models__custom_node_colors__schema_yml = """
+version: 2
+
+models:
+  - name: my_model
+    description: "This is a model description"
+    config:
+        node_color: 'pink'
+"""
+
+dbt_project_yml = """
+models:
+  test:
+    +docs:
+      node_color: "#000000"
+    staging:
+      +docs:
+        node_color: "red"
+"""
+
+
+# class TestCustomNodeColorConfigs(BaseConfigProject):
+#     @pytest.fixture(scope="class")
+#     def project_config_update(self):
+#         return {
+#             "models": {
+#                 "test": {"+node_color": "#c0c0c0"},
+#             },
+#         }
+
+#     def test_custom_color_layering(
+#         self,
+#         project,
+#     ):
+#         pass
+
+
+# python3 -m pytest tests/functional/configs/test_custom_node_colors_configs.py
+class TestNodeColorConfigs:
+    @pytest.fixture(scope="class")
+    def project_config_update(self):
+        return dbt_project_yml
+
+    def test_model_node_color_config(self, project):
+        write_file(
+            models__custom_node_colors__model_sql,
+            project.project_root,
+            "models",
+            "my_model.sql",
+        )
+        run_dbt(["compile"])
+        manifest = get_manifest(project.project_root)
+
+        model_id = "model.test.my_model"
+        model_node_config = manifest.nodes[model_id].config
+        node_color_model_actual = model_node_config._extra["node_color"]
+        node_color_root_actual = model_node_config.docs.node_color
+
+        node_color_model_expected = "#c0c0c0"
+        node_color_root_expected = "#000000"
+
+        assert node_color_model_actual == node_color_model_expected
+        assert node_color_root_actual == node_color_root_expected
+
+    def test_schema_node_color_config(self, project):
+        write_file(
+            models__no_node_colors__model_sql,
+            project.project_root,
+            "models",
+            "my_model.sql",
+        )
+        write_file(
+            models__custom_node_colors__schema_yml,
+            project.project_root,
+            "models",
+            "schema.yml",
+        )
+        run_dbt(["compile"])
+        manifest = get_manifest(project.project_root)
+
+        model_id = "model.test.my_model"
+        model_node_config = manifest.nodes[model_id].config
+        node_color_model_actual = model_node_config._extra["node_color"]
+        node_color_root_actual = model_node_config.docs.node_color
+
+        node_color_model_expected = "pink"
+        node_color_root_expected = "#000000"
+
+        assert node_color_model_actual == node_color_model_expected
+        assert node_color_root_actual == node_color_root_expected
+
+    def test_subfolder_node_color_config(self, project):
+        model_dir = os.path.join(project.project_root, "models", "staging")
+        os.makedirs(model_dir)
+        write_file(
+            models__no_node_colors__model_sql,
+            model_dir,
+            "my_subfolder_model.sql",
+        )
+        run_dbt(["compile"])
+        manifest = get_manifest(project.project_root)
+        print(f"manifest: {manifest.nodes}")
+
+        model_id = "model.test.my_subfolder_model"
+        model_node_config = manifest.nodes[model_id].config
+        node_color_model_actual = model_node_config._extra
+        node_color_root_actual = model_node_config.docs.node_color
+
+        node_color_model_expected = {}
+        node_color_root_expected = "red"
+
+        assert node_color_model_actual == node_color_model_expected
+        assert node_color_root_actual == node_color_root_expected
+
+
+# TODO: node_color in a subfolder overrides global node_color in dbt_project.yml creates a docs object underneath config within the node manifest, and a root level docs object in the node manifest as well
+
+# test = {'model.test.my_model': ParsedModelNode(raw_sql="{{\n    config(\n        materialized='view',\n        node_color='#c0c0c0'\n    )\n}}\n\nselect 1 as id", database='dbt', schema='test16589609566819141019_test_custom_node_colors_configs', fqn=['test', 'my_model'], unique_id='model.test.my_model', package_name='test', root_path='/private/var/folders/hf/80s2zg792jv_n9rtdjt5rnkm0000gp/T/pytest-of-sung/pytest-34/project0', path='my_model.sql', original_file_path='models/my_model.sql', name='my_model', resource_type=<NodeType.Model: 'model'>, alias='my_model', checksum=FileHash(name='sha256', checksum='81429f3e51d581d912f072f258d79ca67bf0d9cd3495d9dceb02ba23c4d4cf0f'), config=NodeConfig(_extra={'node_color': '#c0c0c0'}, enabled=True, alias=None, schema=None, database=None, tags=[], meta={}, materialized='view', persist_docs={}, post_hook=[], pre_hook=[], quoting={}, column_types={}, full_refresh=None, unique_key=None, on_schema_change='ignore', grants={}, docs={'node_color': '#000000'}), _event_status={}, tags=[], refs=[], sources=[], metrics=[], depends_on=DependsOn(macros=[], nodes=[]), description='', columns={}, meta={}, docs={'node_color': '#000000'}, patch_path=None, compiled_path=None, build_path=None, deferred=False, unrendered_config={'docs': {'node_color': '#000000'}, 'materialized': 'view', 'node_color': '#c0c0c0'}, created_at=1658960957.997165, config_call_dict={'materialized': 'view', 'node_color': '#c0c0c0'})}
+
+# manifest = {'model.test.my_model': ParsedModelNode(raw_sql="{{\n    config(\n        materialized='view',\n        node_color='#c0c0c0'\n    )\n}}\n\nselect 1 as id", database='dbt', schema='test16590149696742425564_test_custom_node_colors_configs', fqn=['test', 'my_model'], unique_id='model.test.my_model', package_name='test', root_path='/private/var/folders/hf/80s2zg792jv_n9rtdjt5rnkm0000gp/T/pytest-of-sung/pytest-50/project0', path='my_model.sql', original_file_path='models/my_model.sql', name='my_model', resource_type=<NodeType.Model: 'model'>, alias='my_model', checksum=FileHash(name='sha256', checksum='81429f3e51d581d912f072f258d79ca67bf0d9cd3495d9dceb02ba23c4d4cf0f'), config=NodeConfig(_extra={'node_color': '#c0c0c0'}, enabled=True, alias=None, schema=None, database=None, tags=[], meta={}, materialized='view', incremental_strategy=None, persist_docs={}, post_hook=[], pre_hook=[], quoting={}, column_types={}, full_refresh=None, unique_key=None, on_schema_change='ignore', grants={}, docs=Docs(show=True, node_color='#000000')), _event_status={}, tags=[], refs=[], sources=[], metrics=[], depends_on=DependsOn(macros=[], nodes=[]), description='', columns={}, meta={}, docs=Docs(show=True, node_color='#000000'), patch_path=None, compiled_path=None, build_path=None, deferred=False, unrendered_config={'docs': {'node_color': '#000000'}, 'materialized': 'view', 'node_color': '#c0c0c0'}, created_at=1659014971.114518, config_call_dict={'materialized': 'view', 'node_color': '#c0c0c0'})}
+
+# subfolder_manifest = {'model.test.my_model': ParsedModelNode(raw_sql="{{\n    config(\n        materialized='view'\n    )\n}}\n\nselect 1 as id", database='dbt', schema='test16590332893258542164_test_custom_node_colors_configs', fqn=['test', 'my_model'], unique_id='model.test.my_model', package_name='test', root_path='/private/var/folders/hf/80s2zg792jv_n9rtdjt5rnkm0000gp/T/pytest-of-sung/pytest-73/project0', path='my_model.sql', original_file_path='models/my_model.sql', name='my_model', resource_type=<NodeType.Model: 'model'>, alias='my_model', checksum=FileHash(name='sha256', checksum='96e7dcbd2f14d5f305dec7fd6f22cc46cdb07da2ae14787a7545470fc1e6324c'), config=NodeConfig(_extra={'node_color': 'pink'}, enabled=True, alias=None, schema=None, database=None, tags=[], meta={}, materialized='view', incremental_strategy=None, persist_docs={}, post_hook=[], pre_hook=[], quoting={}, column_types={}, full_refresh=None, unique_key=None, on_schema_change='ignore', grants={}, docs=Docs(show=True, node_color='#000000')), _event_status={}, tags=[], refs=[], sources=[], metrics=[], depends_on=DependsOn(macros=[], nodes=[]), description='This is a model description', columns={}, meta={}, docs=Docs(show=True, node_color='#000000'), patch_path='test://models/schema.yml', compiled_path=None, build_path=None, deferred=False, unrendered_config={'docs': {'node_color': '#000000'}, 'materialized': 'view'}, created_at=1659033290.666083, config_call_dict={'materialized': 'view'}), 'model.test.my_sub_folder_model': ParsedModelNode(raw_sql="{{\n    config(\n        materialized='view'\n    )\n}}\n\nselect 1 as id", database='dbt', schema='test16590332893258542164_test_custom_node_colors_configs', fqn=['test', 'staging', 'my_sub_folder_model'], unique_id='model.test.my_sub_folder_model', package_name='test', root_path='/private/var/folders/hf/80s2zg792jv_n9rtdjt5rnkm0000gp/T/pytest-of-sung/pytest-73/project0', path='staging/my_sub_folder_model.sql', original_file_path='models/staging/my_sub_folder_model.sql', name='my_sub_folder_model', resource_type=<NodeType.Model: 'model'>, alias='my_sub_folder_model', checksum=FileHash(name='sha256', checksum='96e7dcbd2f14d5f305dec7fd6f22cc46cdb07da2ae14787a7545470fc1e6324c'), config=NodeConfig(_extra={}, enabled=True, alias=None, schema=None, database=None, tags=[], meta={}, materialized='view', incremental_strategy=None, persist_docs={}, post_hook=[], pre_hook=[], quoting={}, column_types={}, full_refresh=None, unique_key=None, on_schema_change='ignore', grants={}, docs=Docs(show=True, node_color='red')), _event_status={}, tags=[], refs=[], sources=[], metrics=[], depends_on=DependsOn(macros=[], nodes=[]), description='', columns={}, meta={}, docs=Docs(show=True, node_color='red'), patch_path=None, compiled_path=None, build_path=None, deferred=False, unrendered_config={'docs': {'node_color': 'red'}, 'materialized': 'view'}, created_at=1659033290.957101, config_call_dict={'materialized': 'view'})}
+
+# "model.tpch.dim_customers": {
+#       "raw_sql": "{{\n    config(\n        materialized = 'view',\n        transient=false,\n        node_color = 'pink'\n    )\n}}\n\n\nwith customer as (\n\n    select * from {{ ref('stg_tpch_customers') }}\n\n),\nnation as (\n\n    select * from {{ ref('stg_tpch_nations') }}\n),\nregion as (\n\n    select * from {{ ref('stg_tpch_regions') }}\n\n),\nfinal as (\n    select \n        customer.customer_key,\n        customer.name,\n        customer.address,\n        {# nation.nation_key as nation_key, #}\n        nation.name as nation,\n        {# region.region_key as region_key, #}\n        region.name as region,\n        customer.phone_number,\n        customer.account_balance,\n        customer.market_segment\n        -- new column\n    from\n        customer\n        inner join nation\n            on customer.nation_key = nation.nation_key\n        inner join region\n            on nation.region_key = region.region_key\n)\nselect \n    *\nfrom\n    final\norder by\n    customer_key",
+#       "resource_type": "model",
+#       "depends_on": {
+#         "macros": [],
+#         "nodes": [
+#           "model.tpch.stg_tpch_customers",
+#           "model.tpch.stg_tpch_nations",
+#           "model.tpch.stg_tpch_regions"
+#         ]
+#       },
+#   "config": {
+#     "enabled": true,
+#     "alias": null,
+#     "schema": null,
+#     "database": null,
+#     "tags": [],
+#     "meta": {},
+#     "materialized": "view",
+#     "persist_docs": {},
+#     "quoting": {},
+#     "column_types": {},
+#     "full_refresh": null,
+#     "unique_key": null,
+#     "on_schema_change": "ignore",
+#     "grants": {},
+#     "docs": { "node_color": "#000000" },
+#     "transient": false,
+#     "node_color": "pink",
+#     "post-hook": [],
+#     "pre-hook": []
+#   },