feat: add clickhouse and greenplum support (#460)

Co-authored-by: Grace Goheen <53586774+graciegoheen@users.noreply.github.com>
Co-authored-by: Benoit Perigaud <8754100+b-per@users.noreply.github.com>
This commit is contained in:
igorvoltaic
2024-09-03 15:34:31 +03:00
committed by GitHub
parent eaf8386ced
commit 34209f6a7a
35 changed files with 169 additions and 76 deletions

View File

@@ -22,6 +22,8 @@ Currently, the following adapters are supported:
- DuckDB - DuckDB
- Trino (tested with Iceberg connector) - Trino (tested with Iceberg connector)
- AWS Athena (tested manually) - AWS Athena (tested manually)
- Greenplum (tested manually)
- ClickHouse (tested manually)
## Using This Package ## Using This Package
@@ -29,7 +31,7 @@ Currently, the following adapters are supported:
Check [dbt Hub](https://hub.getdbt.com/dbt-labs/dbt_project_evaluator/latest/) for the latest installation instructions, or [read the docs](https://docs.getdbt.com/docs/package-management) for more information on installing packages. Check [dbt Hub](https://hub.getdbt.com/dbt-labs/dbt_project_evaluator/latest/) for the latest installation instructions, or [read the docs](https://docs.getdbt.com/docs/package-management) for more information on installing packages.
### Additional setup for Databricks/Spark/DuckDB ### Additional setup for Databricks/Spark/DuckDB/Redshift/ClickHouse
In your `dbt_project.yml`, add the following config: In your `dbt_project.yml`, add the following config:

View File

@@ -33,7 +33,7 @@ models:
+materialized: "{{ 'table' if target.type in ['bigquery', 'redshift', 'databricks'] else 'view' }}" +materialized: "{{ 'table' if target.type in ['bigquery', 'redshift', 'databricks'] else 'view' }}"
int_all_dag_relationships: int_all_dag_relationships:
# required for BigQuery, Redshift, and Databricks for performance/memory reasons # required for BigQuery, Redshift, and Databricks for performance/memory reasons
+materialized: "{{ 'table' if target.type in ['bigquery', 'redshift', 'databricks'] else 'view' }}" +materialized: "{{ 'table' if target.type in ['bigquery', 'redshift', 'databricks', 'clickhouse'] else 'view' }}"
dag: dag:
+materialized: table +materialized: table
staging: staging:
@@ -86,7 +86,7 @@ vars:
# -- Execution variables -- # -- Execution variables --
insert_batch_size: "{{ 500 if target.type in ['athena', 'bigquery'] else 10000 }}" insert_batch_size: "{{ 500 if target.type in ['athena', 'bigquery'] else 10000 }}"
max_depth_dag: "{{ 9 if target.type in ['bigquery', 'spark', 'databricks'] else 4 if target.type in ['athena', 'trino'] else -1 }}" max_depth_dag: "{{ 9 if target.type in ['bigquery', 'spark', 'databricks'] else 4 if target.type in ['athena', 'trino', 'clickhouse'] else -1 }}"
# -- Code complexity variables -- # -- Code complexity variables --
comment_chars: ["--"] comment_chars: ["--"]

View File

@@ -23,6 +23,8 @@ Currently, the following adapters are supported:
- DuckDB - DuckDB
- Trino (tested with Iceberg connector) - Trino (tested with Iceberg connector)
- AWS Athena (tested manually) - AWS Athena (tested manually)
- Greenplum (tested manually)
- ClickHouse (tested manually)
## Using This Package ## Using This Package

View File

@@ -72,3 +72,13 @@ integration_tests:
threads: 5 threads: 5
session_properties: session_properties:
query_max_stage_count: 275 query_max_stage_count: 275
clickhouse:
type: clickhouse
host: "{{ env_var('CLICKHOUSE_TEST_HOST') }}"
port: "{{ env_var('CLICKHOUSE_TEST_PORT') | as_number }}"
user: "{{ env_var('CLICKHOUSE_TEST_USER') }}"
password: "{{ env_var('CLICKHOUSE_TEST_PASS') }}"
dbname: "{{ env_var('CLICKHOUSE_TEST_DBNAME') }}"
schema: dbt_project_evaluator_integration_tests_clickhouse
threads: 5

View File

@@ -6,4 +6,4 @@ models:
enforced: true enforced: true
columns: columns:
- name: id - name: id
data_type: integer data_type: "{{ 'UInt8' if target.type in ['clickhouse'] else 'integer' }}"

View File

@@ -1,2 +1,2 @@
-- {{ source('fake_source', 'fake_source') }} -- {{ source('fake_source', 'fake_source') }}
select 1 as id select 1 as id

View File

@@ -5,7 +5,7 @@ models:
enforced: true enforced: true
columns: columns:
- name: id - name: id
data_type: int data_type: "{{ 'UInt8' if target.type in ['clickhouse'] else 'int' }}"
constraints: constraints:
- type: not_null - type: not_null
tests: tests:

View File

@@ -5,5 +5,5 @@
) )
}} }}
-- {{ ref('int_model_5') }}
select * from {{ ref('stg_model_4') }} select * from {{ ref('stg_model_4') }}
-- {{ ref('int_model_5') }}

View File

@@ -20,4 +20,4 @@ models:
enforced: true enforced: true
columns: columns:
- name: id - name: id
data_type: integer data_type: "{{ 'UInt8' if target.type in ['clickhouse'] else 'integer' }}"

View File

@@ -54,4 +54,14 @@ integration_tests:
host: "{{ env_var('DATABRICKS_TEST_HOST') }}" host: "{{ env_var('DATABRICKS_TEST_HOST') }}"
http_path: "{{ env_var('DATABRICKS_TEST_HTTP_PATH') }}" http_path: "{{ env_var('DATABRICKS_TEST_HTTP_PATH') }}"
token: "{{ env_var('DATABRICKS_TEST_ACCESS_TOKEN') }}" token: "{{ env_var('DATABRICKS_TEST_ACCESS_TOKEN') }}"
threads: 10 threads: 10
clickhouse:
type: clickhouse
host: "{{ env_var('CLICKHOUSE_TEST_HOST') }}"
port: "{{ env_var('CLICKHOUSE_TEST_PORT') | as_number }}"
user: "{{ env_var('CLICKHOUSE_TEST_USER') }}"
password: "{{ env_var('CLICKHOUSE_TEST_PASS') }}"
dbname: "{{ env_var('CLICKHOUSE_TEST_DBNAME') }}"
schema: dbt_project_evaluator_integration_tests_clickhouse
threads: 5

View File

@@ -38,4 +38,4 @@ models:
vars: vars:
max_depth_dag: 2 max_depth_dag: 2
chained_views_threshold: 2 chained_views_threshold: 2
primary_key_test_macros: [["my_package.test_my_test", "dbt.test_not_null"]] primary_key_test_macros: [["my_package.test_my_test", "dbt.test_not_null"]]

View File

@@ -0,0 +1,54 @@
{%- macro clickhouse__type_string() -%}
{{ 'Nullable(String)' }}
{%- endmacro %}
{%- macro clickhouse__type_int() -%}
{{ 'Nullable(Int32)' }}
{%- endmacro %}
{%- macro clickhouse__type_float() -%}
{{ 'Nullable(Float32)' }}
{%- endmacro %}
{%- macro clickhouse__type_boolean() -%}
{{ 'Nullable(Bool)' }}
{%- endmacro %}
{% macro clickhouse__replace(string_text, pattern, replacement) -%}
replaceAll(assumeNotNull({{string_text}}), {{pattern}}, {{replacement}})
{%- endmacro %}
{% macro clickhouse__split_part(string_text, delimiter_text, part_number) -%}
splitByChar({{delimiter_text}}, assumeNotNull({{string_text}}))[{{part_number}}]
{%- endmacro %}
{% macro clickhouse__listagg(measure, delimiter_text, order_by_clause, limit_num) -%}
{% if order_by_clause and ' by ' in order_by_clause -%}
{% set order_by_field = order_by_clause.split(' by ')[1] %}
{% set arr = "arrayMap(x -> x.1, arrayReverseSort(x -> x.2, arrayZip(array_agg({}), array_agg({}))))".format(arr, order_by_field) %}
{% else -%}
{% set arr = "array_agg({})".format(measure) %}
{%- endif %}
{% if limit_num -%}
arrayStringConcat(arraySlice({{ arr }}, 1, {{ limit_num }}), {{delimiter_text}})
{% else -%}
arrayStringConcat({{ arr }}, {{delimiter_text}})
{%- endif %}
{%- endmacro %}
{% macro clickhouse__load_csv_rows(model, agate_table) %}
{% set cols_sql = get_seed_column_quoted_csv(model, agate_table.column_names) %}
{% set data_sql = adapter.get_csv_data(agate_table) %}
{% if data_sql %}
{% set sql -%}
insert into {{ this.render() }} ({{ cols_sql }})
{{ adapter.get_model_query_settings(model) }}
format CSV
{{ data_sql }}
{%- endset %}
{% do adapter.add_query(sql, bindings=agate_table, abridge_sql_log=True) %}
{% endif %}
{% endmacro %}

View File

@@ -72,7 +72,7 @@ all_relationships (
is_excluded as child_is_excluded, is_excluded as child_is_excluded,
0 as distance, 0 as distance,
{{ dbt.array_construct(['resource_name']) }} as path, {{ dbt.array_construct(['resource_name']) }} as path,
cast(null as boolean) as is_dependent_on_chain_of_views cast(null as {{ dbt.type_boolean() }}) as is_dependent_on_chain_of_views
from direct_relationships from direct_relationships
-- where direct_parent_id is null {# optional lever to change filtering of anchor clause to only include root resources #} -- where direct_parent_id is null {# optional lever to change filtering of anchor clause to only include root resources #}
@@ -175,7 +175,7 @@ with direct_relationships as (
child_is_excluded, child_is_excluded,
0 as distance, 0 as distance,
{{ dbt.array_construct(['resource_name']) }} as path, {{ dbt.array_construct(['resource_name']) }} as path,
cast(null as boolean) as is_dependent_on_chain_of_views cast(null as {{ dbt.type_boolean() }}) as is_dependent_on_chain_of_views
from get_distinct from get_distinct
) )
@@ -243,7 +243,7 @@ with direct_relationships as (
child.directory_path as child_directory_path, child.directory_path as child_directory_path,
child.file_name as child_file_name, child.file_name as child_file_name,
child.is_excluded as child_is_excluded, child.is_excluded as child_is_excluded,
all_relationships_unioned.distance, cast(all_relationships_unioned.distance as {{ dbt.type_int() }}) as distance,
all_relationships_unioned.path, all_relationships_unioned.path,
all_relationships_unioned.is_dependent_on_chain_of_views all_relationships_unioned.is_dependent_on_chain_of_views
@@ -257,6 +257,10 @@ with direct_relationships as (
{% endmacro %} {% endmacro %}
{% macro clickhouse__recursive_dag() %}
{{ return(bigquery__recursive_dag()) }}
{% endmacro %}
{% macro spark__recursive_dag() %} {% macro spark__recursive_dag() %}
-- as of June 2022 databricks SQL doesn't support "with recursive" in the same way as other DWs -- as of June 2022 databricks SQL doesn't support "with recursive" in the same way as other DWs
{{ return(bigquery__recursive_dag()) }} {{ return(bigquery__recursive_dag()) }}

View File

@@ -17,7 +17,7 @@
wrap_string_with_quotes(node.name), wrap_string_with_quotes(node.name),
wrap_string_with_quotes(node.resource_type), wrap_string_with_quotes(node.resource_type),
wrap_string_with_quotes(node.original_file_path | replace("\\","\\\\")), wrap_string_with_quotes(node.original_file_path | replace("\\","\\\\")),
"cast(" ~ dbt_project_evaluator.is_not_empty_string(node.description) | trim ~ " as boolean)", "cast(" ~ dbt_project_evaluator.is_not_empty_string(node.description) | trim ~ " as " ~ dbt.type_boolean() ~ ")",
wrap_string_with_quotes(node.type), wrap_string_with_quotes(node.type),
wrap_string_with_quotes(node.maturity), wrap_string_with_quotes(node.maturity),
wrap_string_with_quotes(node.package_name), wrap_string_with_quotes(node.package_name),
@@ -35,4 +35,4 @@
{{ return(values) }} {{ return(values) }}
{%- endmacro -%} {%- endmacro -%}

View File

@@ -16,7 +16,7 @@
wrap_string_with_quotes(node.name), wrap_string_with_quotes(node.name),
wrap_string_with_quotes(node.resource_type), wrap_string_with_quotes(node.resource_type),
wrap_string_with_quotes(node.original_file_path | replace("\\","\\\\")), wrap_string_with_quotes(node.original_file_path | replace("\\","\\\\")),
"cast(" ~ dbt_project_evaluator.is_not_empty_string(node.description) | trim ~ " as boolean)", "cast(" ~ dbt_project_evaluator.is_not_empty_string(node.description) | trim ~ " as " ~ dbt.type_boolean() ~ ")",
wrap_string_with_quotes(node.type), wrap_string_with_quotes(node.type),
wrap_string_with_quotes(dbt.escape_single_quotes(node.label)), wrap_string_with_quotes(dbt.escape_single_quotes(node.label)),
wrap_string_with_quotes(node.package_name), wrap_string_with_quotes(node.package_name),

View File

@@ -23,7 +23,7 @@
wrap_string_with_quotes(node.name), wrap_string_with_quotes(node.name),
wrap_string_with_quotes(node.resource_type), wrap_string_with_quotes(node.resource_type),
wrap_string_with_quotes(node.original_file_path | replace("\\","\\\\")), wrap_string_with_quotes(node.original_file_path | replace("\\","\\\\")),
"cast(" ~ node.config.enabled | trim ~ " as boolean)", "cast(" ~ node.config.enabled | trim ~ " as " ~ dbt.type_boolean() ~ ")",
wrap_string_with_quotes(node.config.materialized), wrap_string_with_quotes(node.config.materialized),
wrap_string_with_quotes(node.config.on_schema_change), wrap_string_with_quotes(node.config.on_schema_change),
wrap_string_with_quotes(node.group), wrap_string_with_quotes(node.group),
@@ -31,22 +31,22 @@
wrap_string_with_quotes(node.latest_version), wrap_string_with_quotes(node.latest_version),
wrap_string_with_quotes(node.version), wrap_string_with_quotes(node.version),
wrap_string_with_quotes(node.deprecation_date), wrap_string_with_quotes(node.deprecation_date),
"cast(" ~ contract | trim ~ " as boolean)", "cast(" ~ contract | trim ~ " as " ~ dbt.type_boolean() ~ ")",
node.columns.values() | list | length, node.columns.values() | list | length,
node.columns.values() | list | selectattr('description') | list | length, node.columns.values() | list | selectattr('description') | list | length,
wrap_string_with_quotes(node.database), wrap_string_with_quotes(node.database),
wrap_string_with_quotes(node.schema), wrap_string_with_quotes(node.schema),
wrap_string_with_quotes(node.package_name), wrap_string_with_quotes(node.package_name),
wrap_string_with_quotes(node.alias), wrap_string_with_quotes(node.alias),
"cast(" ~ dbt_project_evaluator.is_not_empty_string(node.description) | trim ~ " as boolean)", "cast(" ~ dbt_project_evaluator.is_not_empty_string(node.description) | trim ~ " as " ~ dbt.type_boolean() ~ ")",
"''" if not node.column_name else wrap_string_with_quotes(dbt.escape_single_quotes(node.column_name)), "''" if not node.column_name else wrap_string_with_quotes(dbt.escape_single_quotes(node.column_name)),
wrap_string_with_quotes(node.meta | tojson), wrap_string_with_quotes(node.meta | tojson),
wrap_string_with_quotes(dbt.escape_single_quotes(hard_coded_references)), wrap_string_with_quotes(dbt.escape_single_quotes(hard_coded_references)),
number_lines, number_lines,
sql_complexity, sql_complexity,
wrap_string_with_quotes(node.get('depends_on',{}).get('macros',[]) | tojson), wrap_string_with_quotes(node.get('depends_on',{}).get('macros',[]) | tojson),
"cast(" ~ dbt_project_evaluator.is_not_empty_string(node.test_metadata) | trim ~ " as boolean)", "cast(" ~ dbt_project_evaluator.is_not_empty_string(node.test_metadata) | trim ~ " as " ~ dbt.type_boolean() ~ ")",
"cast(" ~ exclude_node ~ " as boolean)", "cast(" ~ exclude_node ~ " as " ~ dbt.type_boolean() ~ ")",
] ]
%} %}

View File

@@ -20,9 +20,9 @@
wrap_string_with_quotes(node.alias), wrap_string_with_quotes(node.alias),
wrap_string_with_quotes(node.resource_type), wrap_string_with_quotes(node.resource_type),
wrap_string_with_quotes(node.source_name), wrap_string_with_quotes(node.source_name),
"cast(" ~ dbt_project_evaluator.is_not_empty_string(node.source_description) | trim ~ " as boolean)", "cast(" ~ dbt_project_evaluator.is_not_empty_string(node.source_description) | trim ~ " as " ~ dbt.type_boolean() ~ ")",
"cast(" ~ dbt_project_evaluator.is_not_empty_string(node.description) | trim ~ " as boolean)", "cast(" ~ dbt_project_evaluator.is_not_empty_string(node.description) | trim ~ " as " ~ dbt.type_boolean() ~ ")",
"cast(" ~ node.config.enabled ~ " as boolean)", "cast(" ~ node.config.enabled ~ " as " ~ dbt.type_boolean() ~ ")",
wrap_string_with_quotes(node.loaded_at_field | replace("'", "_")), wrap_string_with_quotes(node.loaded_at_field | replace("'", "_")),
"cast(" ~ ((node.freshness != None) and (dbt_project_evaluator.is_not_empty_string(node.freshness.warn_after.count) "cast(" ~ ((node.freshness != None) and (dbt_project_evaluator.is_not_empty_string(node.freshness.warn_after.count)
or dbt_project_evaluator.is_not_empty_string(node.freshness.error_after.count))) | trim ~ " as boolean)", or dbt_project_evaluator.is_not_empty_string(node.freshness.error_after.count))) | trim ~ " as boolean)",
@@ -32,7 +32,7 @@
wrap_string_with_quotes(node.loader), wrap_string_with_quotes(node.loader),
wrap_string_with_quotes(node.identifier), wrap_string_with_quotes(node.identifier),
wrap_string_with_quotes(node.meta | tojson), wrap_string_with_quotes(node.meta | tojson),
"cast(" ~ exclude_source ~ " as boolean)", "cast(" ~ exclude_source ~ " as " ~ dbt.type_boolean() ~ ")",
] ]
%} %}
@@ -44,4 +44,4 @@
{{ return(values) }} {{ return(values) }}
{%- endmacro -%} {%- endmacro -%}

View File

@@ -62,7 +62,7 @@ joined as (
unioned_with_calc.file_name, unioned_with_calc.file_name,
case case
when unioned_with_calc.resource_type in ('test', 'source', 'metric', 'exposure', 'seed') then null when unioned_with_calc.resource_type in ('test', 'source', 'metric', 'exposure', 'seed') then null
else naming_convention_prefixes.model_type else nullif(naming_convention_prefixes.model_type, '')
end as model_type_prefix, end as model_type_prefix,
case case
when unioned_with_calc.resource_type in ('test', 'source', 'metric', 'exposure', 'seed') then null when unioned_with_calc.resource_type in ('test', 'source', 'metric', 'exposure', 'seed') then null

View File

@@ -45,7 +45,18 @@ direct_metrics_relationships as (
-- for all resources in the graph, find their direct parent -- for all resources in the graph, find their direct parent
direct_relationships as ( direct_relationships as (
select select
all_graph_resources.*, all_graph_resources.resource_id,
all_graph_resources.resource_name,
all_graph_resources.resource_type,
all_graph_resources.file_path,
all_graph_resources.directory_path,
all_graph_resources.file_name,
all_graph_resources.model_type,
all_graph_resources.materialized,
all_graph_resources.is_public,
all_graph_resources.access,
all_graph_resources.source_name,
all_graph_resources.is_excluded,
case case
when all_graph_resources.resource_type = 'source' then null when all_graph_resources.resource_type = 'source' then null
when all_graph_resources.resource_type = 'exposure' then exposures.direct_parent_id when all_graph_resources.resource_type = 'exposure' then exposures.direct_parent_id

View File

@@ -33,7 +33,7 @@ single_use_resources as (
triad_relationships as ( triad_relationships as (
select select
rejoined.parent, rejoined.parent,
rejoined.child as child, rejoined.child,
direct_child.parent as parent_and_child direct_child.parent as parent_and_child
from rejoined from rejoined
left join all_relationships as direct_child left join all_relationships as direct_child
@@ -66,4 +66,4 @@ final_filtered as (
select * from final_filtered select * from final_filtered
{{ filter_exceptions() }} {{ filter_exceptions() }}

View File

@@ -10,7 +10,7 @@ final as (
select select
child as resource_name, child as resource_name,
child_file_path as file_path, child_file_path as file_path,
count(distinct parent) as join_count cast(count(distinct parent) as {{ dbt.type_int() }}) as join_count
from all_dag_relationships from all_dag_relationships
where distance = 1 where distance = 1
group by 1, 2 group by 1, 2

View File

@@ -21,8 +21,8 @@ conversion as (
final as ( final as (
select select
{{ dbt.current_timestamp() if target.type != 'trino' else 'current_timestamp(6)' }} as measured_at, {{ dbt.current_timestamp() if target.type != 'trino' else 'current_timestamp(6)' }} as measured_at,
count(*) as total_models, cast(count(*) as {{ dbt.type_int() }}) as total_models,
sum(is_described_model) as documented_models, cast(sum(is_described_model) as {{ dbt.type_int() }}) as documented_models,
round(sum(is_described_model) * 100.00 / count(*), 2) as documentation_coverage_pct, round(sum(is_described_model) * 100.00 / count(*), 2) as documentation_coverage_pct,
{% for model_type in var('model_types') %} {% for model_type in var('model_types') %}
round( round(

View File

@@ -48,10 +48,10 @@ inappropriate_model_names as (
from models from models
left join appropriate_prefixes left join appropriate_prefixes
on models.model_type = appropriate_prefixes.model_type on models.model_type = appropriate_prefixes.model_type
where models.prefix_value is null where nullif(models.prefix_value, '') is null
) )
select * from inappropriate_model_names select * from inappropriate_model_names
{{ filter_exceptions() }} {{ filter_exceptions() }}

View File

@@ -12,7 +12,7 @@ tests as (
final as ( final as (
select select
* resource_name, is_primary_key_tested, number_of_tests_on_model, number_of_constraints_on_model
from tests from tests
where not(is_primary_key_tested) where not(is_primary_key_tested)
@@ -20,4 +20,4 @@ final as (
select * from final select * from final
{{ filter_exceptions() }} {{ filter_exceptions() }}

View File

@@ -8,7 +8,7 @@ test_counts as (
conversion as ( conversion as (
select select
resource_name, resource_name,
case when number_of_tests_on_model > 0 then 1 else 0 end as is_tested_model, cast(case when number_of_tests_on_model > 0 then 1 else 0 end as {{ dbt.type_boolean() }}) as is_tested_model,
{% for model_type in var('model_types') %} {% for model_type in var('model_types') %}
case when model_type = '{{ model_type }}' then 1.0 else NULL end as is_{{ model_type }}_model, case when model_type = '{{ model_type }}' then 1.0 else NULL end as is_{{ model_type }}_model,
case when number_of_tests_on_model > 0 and model_type = '{{ model_type }}' then 1.0 else 0 end as is_tested_{{ model_type }}_model{% if not loop.last %},{% endif %} case when number_of_tests_on_model > 0 and model_type = '{{ model_type }}' then 1.0 else 0 end as is_tested_{{ model_type }}_model{% if not loop.last %},{% endif %}
@@ -20,10 +20,10 @@ conversion as (
final as ( final as (
select select
{{ dbt.current_timestamp() if target.type != 'trino' else 'current_timestamp(6)' }} as measured_at, {{ dbt.current_timestamp() if target.type != 'trino' else 'current_timestamp(6)' }} as measured_at,
count(*) as total_models, cast(count(*) as {{ dbt.type_int() }}) as total_models,
sum(number_of_tests_on_model) as total_tests, cast(sum(number_of_tests_on_model) as {{ dbt.type_int() }}) as total_tests,
sum(is_tested_model) as tested_models, sum(cast(is_tested_model as {{ dbt.type_int() }})) as tested_models,
round(sum(is_tested_model) * 100.0 / count(*), 2) as test_coverage_pct, round(sum(cast(is_tested_model as {{ dbt.type_int() }})) * 100.0 / count(*), 2) as test_coverage_pct,
{% for model_type in var('model_types') %} {% for model_type in var('model_types') %}
round( round(
{{ dbt_utils.safe_divide( {{ dbt_utils.safe_divide(

View File

@@ -70,7 +70,7 @@ agg_test_relationships as (
select select
direct_parent_id, direct_parent_id,
sum(case cast(sum(case
when ( when (
{%- for test_set in var('primary_key_test_macros') %} {%- for test_set in var('primary_key_test_macros') %}
{%- set compare_value = test_set | length %} {%- set compare_value = test_set | length %}
@@ -81,9 +81,9 @@ agg_test_relationships as (
) then 1 ) then 1
else 0 else 0
end end
) >= 1 as is_primary_key_tested, ) >= 1 as {{ dbt.type_boolean() }}) as is_primary_key_tested,
sum(tests_count) as number_of_tests_on_model, cast(sum(tests_count) as {{ dbt.type_int()}}) as number_of_tests_on_model,
sum(constraints_count) as number_of_constraints_on_model cast(sum(constraints_count) as {{ dbt.type_int()}}) as number_of_constraints_on_model
from combine_column_counts from combine_column_counts
group by 1 group by 1
@@ -94,9 +94,9 @@ final as (
all_graph_resources.resource_name, all_graph_resources.resource_name,
all_graph_resources.resource_type, all_graph_resources.resource_type,
all_graph_resources.model_type, all_graph_resources.model_type,
coalesce(agg_test_relationships.is_primary_key_tested, FALSE) as is_primary_key_tested, cast(coalesce(agg_test_relationships.is_primary_key_tested, FALSE) as {{ dbt.type_boolean()}}) as is_primary_key_tested,
coalesce(agg_test_relationships.number_of_tests_on_model, 0) as number_of_tests_on_model, cast(coalesce(agg_test_relationships.number_of_tests_on_model, 0) as {{ dbt.type_int()}}) as number_of_tests_on_model,
coalesce(agg_test_relationships.number_of_constraints_on_model, 0) as number_of_constraints_on_model cast(coalesce(agg_test_relationships.number_of_constraints_on_model, 0) as {{ dbt.type_int()}}) as number_of_constraints_on_model
from all_graph_resources from all_graph_resources
left join agg_test_relationships left join agg_test_relationships
on all_graph_resources.resource_id = agg_test_relationships.direct_parent_id on all_graph_resources.resource_id = agg_test_relationships.direct_parent_id

View File

@@ -17,7 +17,7 @@ with dummy_cte as (
select select
cast(null as {{ dbt_project_evaluator.type_string_dpe()}}) as resource_id, cast(null as {{ dbt_project_evaluator.type_string_dpe()}}) as resource_id,
cast(null as {{ dbt_project_evaluator.type_string_dpe()}}) as direct_parent_id, cast(null as {{ dbt_project_evaluator.type_string_dpe()}}) as direct_parent_id,
cast(True as boolean) as is_primary_relationship cast(True as {{ dbt.type_boolean() }}) as is_primary_relationship
from dummy_cte from dummy_cte
where false where false

View File

@@ -17,7 +17,7 @@ with dummy_cte as (
select select
cast(null as {{ dbt_project_evaluator.type_string_dpe()}}) as resource_id, cast(null as {{ dbt_project_evaluator.type_string_dpe()}}) as resource_id,
cast(null as {{ dbt_project_evaluator.type_string_dpe()}}) as direct_parent_id, cast(null as {{ dbt_project_evaluator.type_string_dpe()}}) as direct_parent_id,
cast(True as boolean) as is_primary_relationship cast(True as {{ dbt.type_boolean() }}) as is_primary_relationship
from dummy_cte from dummy_cte
where false where false

View File

@@ -19,7 +19,7 @@ select
cast(null as {{ dbt_project_evaluator.type_large_string()}}) as description, cast(null as {{ dbt_project_evaluator.type_large_string()}}) as description,
cast(null as {{ dbt_project_evaluator.type_string_dpe()}}) as data_type, cast(null as {{ dbt_project_evaluator.type_string_dpe()}}) as data_type,
cast(null as {{ dbt_project_evaluator.type_string_dpe()}}) as constraints, cast(null as {{ dbt_project_evaluator.type_string_dpe()}}) as constraints,
cast(True as boolean) as has_not_null_constraint, cast(True as {{ dbt.type_boolean() }}) as has_not_null_constraint,
cast(0 as {{ dbt.type_int() }}) as constraints_count, cast(0 as {{ dbt.type_int() }}) as constraints_count,
cast(null as {{ dbt_project_evaluator.type_string_dpe()}}) as quote cast(null as {{ dbt_project_evaluator.type_string_dpe()}}) as quote

View File

@@ -16,7 +16,7 @@ with dummy_cte as (
select select
cast(null as {{ dbt_project_evaluator.type_string_dpe()}}) as resource_id, cast(null as {{ dbt_project_evaluator.type_string_dpe()}}) as resource_id,
cast(null as {{ dbt_project_evaluator.type_string_dpe()}}) as direct_parent_id, cast(null as {{ dbt_project_evaluator.type_string_dpe()}}) as direct_parent_id,
cast(True as boolean) as is_primary_relationship cast(True as {{ dbt.type_boolean() }}) as is_primary_relationship
from dummy_cte from dummy_cte
where false where false

View File

@@ -19,7 +19,7 @@ select
cast(null as {{ dbt_project_evaluator.type_string_dpe() }} ) as name, cast(null as {{ dbt_project_evaluator.type_string_dpe() }} ) as name,
cast(null as {{ dbt_project_evaluator.type_string_dpe() }} ) as resource_type, cast(null as {{ dbt_project_evaluator.type_string_dpe() }} ) as resource_type,
cast(null as {{ dbt_project_evaluator.type_string_dpe() }} ) as file_path, cast(null as {{ dbt_project_evaluator.type_string_dpe() }} ) as file_path,
cast(True as boolean) as is_described, cast(True as {{ dbt.type_boolean() }}) as is_described,
cast(null as {{ dbt_project_evaluator.type_string_dpe() }} ) as exposure_type, cast(null as {{ dbt_project_evaluator.type_string_dpe() }} ) as exposure_type,
cast(null as {{ dbt_project_evaluator.type_string_dpe() }} ) as maturity, cast(null as {{ dbt_project_evaluator.type_string_dpe() }} ) as maturity,
cast(null as {{ dbt_project_evaluator.type_string_dpe() }} ) as package_name, cast(null as {{ dbt_project_evaluator.type_string_dpe() }} ) as package_name,
@@ -29,4 +29,4 @@ select
cast(null as {{ dbt_project_evaluator.type_string_dpe() }} ) as meta cast(null as {{ dbt_project_evaluator.type_string_dpe() }} ) as meta
from dummy_cte from dummy_cte
where false where false

View File

@@ -19,7 +19,7 @@ select
cast(null as {{ dbt_project_evaluator.type_string_dpe() }}) as name, cast(null as {{ dbt_project_evaluator.type_string_dpe() }}) as name,
cast(null as {{ dbt_project_evaluator.type_string_dpe() }}) as resource_type, cast(null as {{ dbt_project_evaluator.type_string_dpe() }}) as resource_type,
cast(null as {{ dbt_project_evaluator.type_string_dpe() }}) as file_path, cast(null as {{ dbt_project_evaluator.type_string_dpe() }}) as file_path,
cast(True as boolean) as is_described, cast(True as {{ dbt.type_boolean() }}) as is_described,
cast(null as {{ dbt_project_evaluator.type_string_dpe() }}) as metric_type, cast(null as {{ dbt_project_evaluator.type_string_dpe() }}) as metric_type,
cast(null as {{ dbt_project_evaluator.type_string_dpe() }}) as label, cast(null as {{ dbt_project_evaluator.type_string_dpe() }}) as label,
cast(null as {{ dbt_project_evaluator.type_string_dpe() }}) as package_name, cast(null as {{ dbt_project_evaluator.type_string_dpe() }}) as package_name,
@@ -34,4 +34,4 @@ select
cast(null as {{ dbt_project_evaluator.type_string_dpe() }}) as meta cast(null as {{ dbt_project_evaluator.type_string_dpe() }}) as meta
from dummy_cte from dummy_cte
where false where false

View File

@@ -20,7 +20,7 @@ select
cast(null as {{ dbt_project_evaluator.type_string_dpe() }}) as name, cast(null as {{ dbt_project_evaluator.type_string_dpe() }}) as name,
cast(null as {{ dbt_project_evaluator.type_string_dpe() }}) as resource_type, cast(null as {{ dbt_project_evaluator.type_string_dpe() }}) as resource_type,
cast(null as {{ dbt_project_evaluator.type_string_dpe() }}) as file_path, cast(null as {{ dbt_project_evaluator.type_string_dpe() }}) as file_path,
cast(True as boolean) as is_enabled, cast(True as {{ dbt.type_boolean() }}) as is_enabled,
cast(null as {{ dbt_project_evaluator.type_string_dpe() }}) as materialized, cast(null as {{ dbt_project_evaluator.type_string_dpe() }}) as materialized,
cast(null as {{ dbt_project_evaluator.type_string_dpe() }}) as on_schema_change, cast(null as {{ dbt_project_evaluator.type_string_dpe() }}) as on_schema_change,
cast(null as {{ dbt_project_evaluator.type_string_dpe() }}) as model_group, cast(null as {{ dbt_project_evaluator.type_string_dpe() }}) as model_group,
@@ -28,22 +28,22 @@ select
cast(null as {{ dbt_project_evaluator.type_string_dpe() }}) as latest_version, cast(null as {{ dbt_project_evaluator.type_string_dpe() }}) as latest_version,
cast(null as {{ dbt_project_evaluator.type_string_dpe() }}) as version, cast(null as {{ dbt_project_evaluator.type_string_dpe() }}) as version,
cast(null as {{ dbt_project_evaluator.type_string_dpe() }}) as deprecation_date, cast(null as {{ dbt_project_evaluator.type_string_dpe() }}) as deprecation_date,
cast(True as boolean) as is_contract_enforced, cast(True as {{ dbt.type_boolean() }}) as is_contract_enforced,
cast(0 as {{ dbt.type_int() }}) as total_defined_columns, cast(0 as {{ dbt.type_int() }}) as total_defined_columns,
cast(0 as {{ dbt.type_int() }}) as total_described_columns, cast(0 as {{ dbt.type_int() }}) as total_described_columns,
cast(null as {{ dbt_project_evaluator.type_string_dpe() }}) as database, cast(null as {{ dbt_project_evaluator.type_string_dpe() }}) as database,
cast(null as {{ dbt_project_evaluator.type_string_dpe() }}) as schema, cast(null as {{ dbt_project_evaluator.type_string_dpe() }}) as schema,
cast(null as {{ dbt_project_evaluator.type_string_dpe() }}) as package_name, cast(null as {{ dbt_project_evaluator.type_string_dpe() }}) as package_name,
cast(null as {{ dbt_project_evaluator.type_string_dpe() }}) as alias, cast(null as {{ dbt_project_evaluator.type_string_dpe() }}) as alias,
cast(True as boolean) as is_described, cast(True as {{ dbt.type_boolean() }}) as is_described,
cast(null as {{ dbt_project_evaluator.type_string_dpe() }}) as column_name, cast(null as {{ dbt_project_evaluator.type_string_dpe() }}) as column_name,
cast(null as {{ dbt_project_evaluator.type_string_dpe() }}) as meta, cast(null as {{ dbt_project_evaluator.type_string_dpe() }}) as meta,
cast(null as {{ dbt_project_evaluator.type_string_dpe() }}) as hard_coded_references, cast(null as {{ dbt_project_evaluator.type_string_dpe() }}) as hard_coded_references,
cast(null as {{ dbt.type_int() }}) as number_lines, cast(null as {{ dbt.type_int() }}) as number_lines,
cast(null as {{ dbt.type_float() }}) as sql_complexity, cast(null as {{ dbt.type_float() }}) as sql_complexity,
cast(null as {{ dbt_project_evaluator.type_string_dpe() }}) as macro_dependencies, cast(null as {{ dbt_project_evaluator.type_string_dpe() }}) as macro_dependencies,
cast(True as boolean) as is_generic_test, cast(True as {{ dbt.type_boolean() }}) as is_generic_test,
cast(True as boolean) as is_excluded cast(True as {{ dbt.type_boolean() }}) as is_excluded
from dummy_cte from dummy_cte
where false where false

View File

@@ -22,18 +22,18 @@ select
cast(null as {{ dbt_project_evaluator.type_string_dpe() }}) as alias, cast(null as {{ dbt_project_evaluator.type_string_dpe() }}) as alias,
cast(null as {{ dbt_project_evaluator.type_string_dpe() }}) as resource_type, cast(null as {{ dbt_project_evaluator.type_string_dpe() }}) as resource_type,
cast(null as {{ dbt_project_evaluator.type_string_dpe() }}) as source_name, cast(null as {{ dbt_project_evaluator.type_string_dpe() }}) as source_name,
cast(True as boolean) as is_source_described, cast(True as {{ dbt.type_boolean() }}) as is_source_described,
cast(True as boolean) as is_described, cast(True as {{ dbt.type_boolean() }}) as is_described,
cast(True as boolean) as is_enabled, cast(True as {{ dbt.type_boolean() }}) as is_enabled,
cast(null as {{ dbt_project_evaluator.type_string_dpe() }}) as loaded_at_field, cast(null as {{ dbt_project_evaluator.type_string_dpe() }}) as loaded_at_field,
cast(True as boolean) as is_freshness_enabled, cast(True as {{ dbt.type_boolean() }}) as is_freshness_enabled,
cast(null as {{ dbt_project_evaluator.type_string_dpe() }}) as database, cast(null as {{ dbt_project_evaluator.type_string_dpe() }}) as database,
cast(null as {{ dbt_project_evaluator.type_string_dpe() }}) as schema, cast(null as {{ dbt_project_evaluator.type_string_dpe() }}) as schema,
cast(null as {{ dbt_project_evaluator.type_string_dpe() }}) as package_name, cast(null as {{ dbt_project_evaluator.type_string_dpe() }}) as package_name,
cast(null as {{ dbt_project_evaluator.type_string_dpe() }}) as loader, cast(null as {{ dbt_project_evaluator.type_string_dpe() }}) as loader,
cast(null as {{ dbt_project_evaluator.type_string_dpe() }}) as identifier, cast(null as {{ dbt_project_evaluator.type_string_dpe() }}) as identifier,
cast(null as {{ dbt_project_evaluator.type_string_dpe() }}) as meta, cast(null as {{ dbt_project_evaluator.type_string_dpe() }}) as meta,
cast(True as boolean) as is_excluded cast(True as {{ dbt.type_boolean() }}) as is_excluded
from dummy_cte from dummy_cte
where false where false

View File

@@ -6,10 +6,10 @@ seeds:
config: config:
column_types: column_types:
fct_name: "{{ 'varchar' if target.type in ['redshift', 'postgres', 'snowflake', 'trino'] else 'string' }}" fct_name: "{{ 'varchar' if target.type in ['redshift', 'postgres', 'snowflake', 'trino', 'greenplum'] else 'String' if target.type in ['clickhouse'] else 'string' }}"
column_name: "{{ 'varchar' if target.type in ['redshift', 'postgres', 'snowflake', 'trino'] else 'string' }}" column_name: "{{ 'varchar' if target.type in ['redshift', 'postgres', 'snowflake', 'trino', 'greenplum'] else 'String' if target.type in ['clickhouse'] else 'string' }}"
id_to_exclude: "{{ 'varchar' if target.type in ['redshift', 'postgres', 'snowflake', 'trino'] else 'string' }}" id_to_exclude: "{{ 'varchar' if target.type in ['redshift', 'postgres', 'snowflake', 'trino', 'greenplum'] else 'String' if target.type in ['clickhouse'] else 'string' }}"
comment: "{{ 'varchar' if target.type in ['redshift', 'postgres', 'snowflake', 'trino'] else 'string' }}" comment: "{{ 'varchar' if target.type in ['redshift', 'postgres', 'snowflake', 'trino', 'greenplum'] else 'String' if target.type in ['clickhouse'] else 'string' }}"
columns: columns:
- name: fct_name - name: fct_name