diff --git a/.gitignore b/.gitignore index 8df0b8a..3d84d54 100644 --- a/.gitignore +++ b/.gitignore @@ -27,4 +27,5 @@ integration_tests/package-lock.yml # duckdb *.db -*.duckdb \ No newline at end of file +*.duckdb +dbt_internal_packages/ diff --git a/dbt_project.yml b/dbt_project.yml index 7cacc73..d83989e 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -2,7 +2,7 @@ name: 'dbt_project_evaluator' version: '1.0.0' config-version: 2 -require-dbt-version: [">=1.9.0", "<3.0.0"] +require-dbt-version: [">=1.10.6", "<3.0.0"] model-paths: ["models"] analysis-paths: ["analysis"] diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index f0ca576..4fc6b1f 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -31,6 +31,7 @@ dispatch: flags: require_nested_cumulative_type_params: True require_yaml_configuration_for_mf_time_spines: True + require_generic_test_arguments_property: True models: dbt_project_evaluator_integration_tests: diff --git a/integration_tests/exclude_package/models/staging/_models.yml b/integration_tests/exclude_package/models/staging/_models.yml index 8794dc1..45afcd0 100644 --- a/integration_tests/exclude_package/models/staging/_models.yml +++ b/integration_tests/exclude_package/models/staging/_models.yml @@ -1,7 +1,7 @@ models: - name: excluded_model - access: public config: + access: public contract: enforced: true columns: diff --git a/integration_tests/models/marts/exposures.yml b/integration_tests/models/marts/exposures.yml index 9272c8b..01f6a2d 100644 --- a/integration_tests/models/marts/exposures.yml +++ b/integration_tests/models/marts/exposures.yml @@ -6,10 +6,11 @@ exposures: type: dashboard url: dave.com/metrics maturity: low - tags: ['proserv'] owner: name: dave's davey email: proserv@dbt.com + config: + tags: ['proserv'] depends_on: - ref('fct_model_6') diff --git a/integration_tests/models/reports/reports.yml b/integration_tests/models/reports/reports.yml index 6b2c7a7..16be58b 100644 --- a/integration_tests/models/reports/reports.yml +++ b/integration_tests/models/reports/reports.yml @@ -1,10 +1,15 @@ version: 2 +groups: + - name: my_awesome_group + owner: + email: thegoat@jaffleshop.com + models: - name: report_1 description: tom cruise's minority report - access: public config: + access: public materialized: table columns: - name: col @@ -13,9 +18,10 @@ models: - name: report_2 description: the colbert report - access: public config: + access: public materialized: table + group: my_awesome_group contract: enforced: true columns: diff --git a/integration_tests/models/staging/source_1/schema.yml b/integration_tests/models/staging/source_1/schema.yml index 78c7bc7..6e3de69 100644 --- a/integration_tests/models/staging/source_1/schema.yml +++ b/integration_tests/models/staging/source_1/schema.yml @@ -13,9 +13,10 @@ models: - name: stg_model_3 data_tests: - dbt_utils.unique_combination_of_columns: - combination_of_columns: - - id - - color + arguments: + combination_of_columns: + - id + - color - name: stg_model_2 columns: - name: id diff --git a/integration_tests/models/staging/staging.yml b/integration_tests/models/staging/staging.yml index 1904c80..941d178 100644 --- a/integration_tests/models/staging/staging.yml +++ b/integration_tests/models/staging/staging.yml @@ -8,8 +8,9 @@ models: - unique - not_null - relationships: - to: ref('stg_model_2') - field: id + arguments: + to: ref('stg_model_2') + field: id - name: "concat(coalesce('fake_column', ' '),'for_testing')" data_tests: - not_null diff --git a/integration_tests/seeds/dag/dag_seeds.yml b/integration_tests/seeds/dag/dag_seeds.yml index b963404..34e74b9 100644 --- a/integration_tests/seeds/dag/dag_seeds.yml +++ b/integration_tests/seeds/dag/dag_seeds.yml @@ -4,100 +4,122 @@ seeds: - name: test_fct_multiple_sources_joined data_tests: - dbt_utils.equality: - name: equality_fct_multiple_sources_joined - compare_model: ref('fct_multiple_sources_joined') - compare_columns: - - child - - "{{ 'source_parents' if target.type != 'databricks' else 'child' }}" + config: + name: equality_fct_multiple_sources_joined + arguments: + compare_model: ref('fct_multiple_sources_joined') + compare_columns: + - child + - "{{ 'source_parents' if target.type != 'databricks' else 'child' }}" - name: test_fct_direct_join_to_source data_tests: - dbt_utils.equality: - name: equality_fct_direct_join_to_source - compare_model: ref('fct_direct_join_to_source') - compare_columns: - - parent - - parent_resource_type - - child - - child_resource_type - - distance + config: + name: equality_fct_direct_join_to_source + arguments: + compare_model: ref('fct_direct_join_to_source') + compare_columns: + - parent + - parent_resource_type + - child + - child_resource_type + - distance - name: test_fct_duplicate_sources data_tests: - dbt_utils.equality: - name: equality_fct_duplicate_sources - compare_model: ref('fct_duplicate_sources') - compare_columns: - - source_names + config: + name: equality_fct_duplicate_sources + arguments: + compare_model: ref('fct_duplicate_sources') + compare_columns: + - source_names - name: test_fct_root_models data_tests: - dbt_utils.equality: - name: equality_fct_root_models - compare_model: ref('fct_root_models') + config: + name: equality_fct_root_models + arguments: + compare_model: ref('fct_root_models') - name: test_fct_unused_sources data_tests: - dbt_utils.equality: - name: equality_fct_unused_sources - compare_model: ref('fct_unused_sources') + config: + name: equality_fct_unused_sources + arguments: + compare_model: ref('fct_unused_sources') - name: test_fct_source_fanout data_tests: - dbt_utils.equality: - name: equality_fct_source_fanout - compare_model: ref('fct_source_fanout') - compare_columns: - - parent - - "{{ 'model_children' if target.type != 'databricks' else 'parent' }}" + config: + name: equality_fct_source_fanout + arguments: + compare_model: ref('fct_source_fanout') + compare_columns: + - parent + - "{{ 'model_children' if target.type != 'databricks' else 'parent' }}" - name: test_fct_model_fanout data_tests: - dbt_utils.equality: - name: equality_fct_model_fanout - compare_model: ref('fct_model_fanout') - compare_columns: - - parent - - parent_model_type - - "{{ 'leaf_children' if target.type != 'databricks' else 'parent_model_type' }}" + config: + name: equality_fct_model_fanout + arguments: + compare_model: ref('fct_model_fanout') + compare_columns: + - parent + - parent_model_type + - "{{ 'leaf_children' if target.type != 'databricks' else 'parent_model_type' }}" - name: test_fct_staging_dependent_on_staging data_tests: - dbt_utils.equality: - name: equality_fct_staging_dependent_on_staging - compare_model: ref('fct_staging_dependent_on_staging') - compare_columns: - - parent - - parent_model_type - - child - - child_model_type + config: + name: equality_fct_staging_dependent_on_staging + arguments: + compare_model: ref('fct_staging_dependent_on_staging') + compare_columns: + - parent + - parent_model_type + - child + - child_model_type - name: test_fct_rejoining_of_upstream_concepts data_tests: - dbt_utils.equality: - name: equality_fct_rejoining_of_upstream_concepts - compare_model: ref('fct_rejoining_of_upstream_concepts') - compare_columns: - - parent - - child - - parent_and_child - - is_loop_independent + config: + name: equality_fct_rejoining_of_upstream_concepts + arguments: + compare_model: ref('fct_rejoining_of_upstream_concepts') + compare_columns: + - parent + - child + - parent_and_child + - is_loop_independent - name: test_fct_hard_coded_references data_tests: - dbt_utils.equality: - name: equality_fct_hard_coded_references - compare_model: ref('fct_hard_coded_references') - compare_columns: - - model - - hard_coded_references + config: + name: equality_fct_hard_coded_references + arguments: + compare_model: ref('fct_hard_coded_references') + compare_columns: + - model + - hard_coded_references - name: test_fct_too_many_joins data_tests: - dbt_utils.equality: - name: equality_fct_too_many_joins - compare_model: ref('fct_too_many_joins') - compare_columns: - - resource_name - - file_path - - join_count \ No newline at end of file + config: + name: equality_fct_too_many_joins + arguments: + compare_model: ref('fct_too_many_joins') + compare_columns: + - resource_name + - file_path + - join_count \ No newline at end of file diff --git a/integration_tests/seeds/docs/docs_seeds.yml b/integration_tests/seeds/docs/docs_seeds.yml index b414a0f..89e1711 100644 --- a/integration_tests/seeds/docs/docs_seeds.yml +++ b/integration_tests/seeds/docs/docs_seeds.yml @@ -7,8 +7,10 @@ seeds: - docs data_tests: - dbt_utils.equality: - name: equality_fct_undocumented_models - compare_model: ref('fct_undocumented_models') + config: + name: equality_fct_undocumented_models + arguments: + compare_model: ref('fct_undocumented_models') - name: test_fct_documentation_coverage config: @@ -21,16 +23,18 @@ seeds: - docs data_tests: - dbt_utils.equality: - name: equality_fct_documentation_coverage - compare_model: ref('fct_documentation_coverage') - compare_columns: - - total_models - - documented_models - - documentation_coverage_pct - - staging_documentation_coverage_pct - - intermediate_documentation_coverage_pct - - marts_documentation_coverage_pct - - other_documentation_coverage_pct + config: + name: equality_fct_documentation_coverage + arguments: + compare_model: ref('fct_documentation_coverage') + compare_columns: + - total_models + - documented_models + - documentation_coverage_pct + - staging_documentation_coverage_pct + - intermediate_documentation_coverage_pct + - marts_documentation_coverage_pct + - other_documentation_coverage_pct - name: test_fct_undocumented_source_tables config: @@ -38,8 +42,10 @@ seeds: - docs data_tests: - dbt_utils.equality: - name: equality_fct_undocumented_source_tables - compare_model: ref('fct_undocumented_source_tables') + config: + name: equality_fct_undocumented_source_tables + arguments: + compare_model: ref('fct_undocumented_source_tables') - name: test_fct_undocumented_sources config: @@ -47,5 +53,7 @@ seeds: - docs data_tests: - dbt_utils.equality: - name: equality_fct_undocumented_sources - compare_model: ref('fct_undocumented_sources') + config: + name: equality_fct_undocumented_sources + arguments: + compare_model: ref('fct_undocumented_sources') diff --git a/integration_tests/seeds/governance/_governance_seeds.yml b/integration_tests/seeds/governance/_governance_seeds.yml index c675390..eb66ded 100644 --- a/integration_tests/seeds/governance/_governance_seeds.yml +++ b/integration_tests/seeds/governance/_governance_seeds.yml @@ -4,14 +4,19 @@ seeds: - name: test_fct_public_models_without_contract data_tests: - dbt_utils.equality: - compare_model: ref('fct_public_models_without_contract') + arguments: + compare_model: ref('fct_public_models_without_contract') - name: test_fct_undocumented_public_models data_tests: - dbt_utils.equality: - compare_model: ref('fct_undocumented_public_models') + config: + name: equality_fct_undocumented_public_models + arguments: + compare_model: ref('fct_undocumented_public_models') - name: test_fct_exposures_dependent_on_private_models data_tests: - dbt_utils.equality: - compare_model: ref('fct_exposures_dependent_on_private_models') \ No newline at end of file + arguments: + compare_model: ref('fct_exposures_dependent_on_private_models') \ No newline at end of file diff --git a/integration_tests/seeds/performance/performance_seeds.yml b/integration_tests/seeds/performance/performance_seeds.yml index 66cb0b1..b9255a8 100644 --- a/integration_tests/seeds/performance/performance_seeds.yml +++ b/integration_tests/seeds/performance/performance_seeds.yml @@ -4,14 +4,17 @@ seeds: - name: test_fct_chained_views_dependencies data_tests: - dbt_utils.equality: - name: equality_fct_chained_views_dependencies - compare_model: ref('fct_chained_views_dependencies') - compare_columns: - - parent - - child - - distance + config: + name: equality_fct_chained_views_dependencies + arguments: + compare_model: ref('fct_chained_views_dependencies') + compare_columns: + - parent + - child + - distance - name: test_fct_exposure_parents_materializations data_tests: - dbt_utils.equality: - compare_model: ref('fct_exposure_parents_materializations') \ No newline at end of file + arguments: + compare_model: ref('fct_exposure_parents_materializations') \ No newline at end of file diff --git a/integration_tests/seeds/structure/structure_seeds.yml b/integration_tests/seeds/structure/structure_seeds.yml index a92c03a..e85854b 100644 --- a/integration_tests/seeds/structure/structure_seeds.yml +++ b/integration_tests/seeds/structure/structure_seeds.yml @@ -4,37 +4,45 @@ seeds: - name: test_fct_model_directories data_tests: - dbt_utils.equality: - name: equality_fct_model_directories - compare_model: ref('fct_model_directories') - compare_columns: - - resource_name - - current_file_path - - change_file_path_to + config: + name: equality_fct_model_directories + arguments: + compare_model: ref('fct_model_directories') + compare_columns: + - resource_name + - current_file_path + - change_file_path_to - name: test_fct_model_naming_conventions data_tests: - dbt_utils.equality: - name: equality_fct_model_naming_conventions - compare_model: ref('fct_model_naming_conventions') - compare_columns: - - resource_name - - model_type - - "{{ 'appropriate_prefixes' if target.type != 'databricks' else 'model_type' }}" + config: + name: equality_fct_model_naming_conventions + arguments: + compare_model: ref('fct_model_naming_conventions') + compare_columns: + - resource_name + - model_type + - "{{ 'appropriate_prefixes' if target.type != 'databricks' else 'model_type' }}" - name: test_fct_source_directories data_tests: - dbt_utils.equality: - name: equality_fct_source_directories - compare_model: ref('fct_source_directories') - compare_columns: - - resource_name - - current_file_path - - change_file_path_to + config: + name: equality_fct_source_directories + arguments: + compare_model: ref('fct_source_directories') + compare_columns: + - resource_name + - current_file_path + - change_file_path_to - name: test_fct_test_directories data_tests: - dbt_utils.equality: - name: equality_fct_test_directories - compare_model: ref('fct_test_directories') - compare_columns: - - test_name - - model_name - - current_test_directory - - change_test_directory_to + config: + name: equality_fct_test_directories + arguments: + compare_model: ref('fct_test_directories') + compare_columns: + - test_name + - model_name + - current_test_directory + - change_test_directory_to diff --git a/integration_tests/seeds/tests/tests_seeds.yml b/integration_tests/seeds/tests/tests_seeds.yml index 7789108..93dc070 100644 --- a/integration_tests/seeds/tests/tests_seeds.yml +++ b/integration_tests/seeds/tests/tests_seeds.yml @@ -4,11 +4,13 @@ seeds: - name: test_fct_missing_primary_key_tests data_tests: - dbt_utils.equality: - name: equality_fct_missing_primary_key_tests - compare_model: ref('fct_missing_primary_key_tests') - exclude_columns: - - resource_type - - model_type + config: + name: equality_fct_missing_primary_key_tests + arguments: + compare_model: ref('fct_missing_primary_key_tests') + exclude_columns: + - resource_type + - model_type - name: test_fct_test_coverage config: @@ -20,21 +22,25 @@ seeds: other_test_coverage_pct: *float data_tests: - dbt_utils.equality: - name: equality_fct_test_coverage - compare_model: ref('fct_test_coverage') - compare_columns: - - total_models - - total_tests - - tested_models - - "{{ 'test_coverage_pct' if not target.name in ['duckdb', 'databricks', 'trino'] else 'tested_models' }}" - - test_to_model_ratio - - staging_test_coverage_pct - - intermediate_test_coverage_pct - - marts_test_coverage_pct - - other_test_coverage_pct + config: + name: equality_fct_test_coverage + arguments: + compare_model: ref('fct_test_coverage') + compare_columns: + - total_models + - total_tests + - tested_models + - "{{ 'test_coverage_pct' if not target.name in ['duckdb', 'databricks', 'trino'] else 'tested_models' }}" + - test_to_model_ratio + - staging_test_coverage_pct + - intermediate_test_coverage_pct + - marts_test_coverage_pct + - other_test_coverage_pct - name: test_fct_sources_without_freshness data_tests: - dbt_utils.equality: - name: equality_fct_sources_without_freshness - compare_model: ref('fct_sources_without_freshness') + config: + name: equality_fct_sources_without_freshness + arguments: + compare_model: ref('fct_sources_without_freshness') diff --git a/integration_tests_2/dbt_project.yml b/integration_tests_2/dbt_project.yml index 31e90b8..6fc2902 100644 --- a/integration_tests_2/dbt_project.yml +++ b/integration_tests_2/dbt_project.yml @@ -33,6 +33,7 @@ on-run-end: "{{ dbt_project_evaluator.print_dbt_project_evaluator_issues() }}" flags: require_nested_cumulative_type_params: True require_yaml_configuration_for_mf_time_spines: True + require_generic_test_arguments_property: True models: dbt_project_evaluator_integration_tests_2: diff --git a/models/marts/documentation/documentation.yml b/models/marts/documentation/documentation.yml index b3a0704..45a3285 100644 --- a/models/marts/documentation/documentation.yml +++ b/models/marts/documentation/documentation.yml @@ -10,9 +10,11 @@ models: description: the number of models in the project with a description divided by the total number of models in the project data_tests: - dbt_utils.accepted_range: - name: valid_documentation_coverage - min_value: "{{ var('documentation_coverage_target') }}" - severity: warn + config: + name: valid_documentation_coverage + severity: warn + arguments: + min_value: "{{ var('documentation_coverage_target') }}" - name: fct_undocumented_models description: > This model contains all models that do not have a description configured in a YML file. diff --git a/models/marts/performance/performance.yml b/models/marts/performance/performance.yml index d9941d7..4a4ef25 100644 --- a/models/marts/performance/performance.yml +++ b/models/marts/performance/performance.yml @@ -8,4 +8,5 @@ models: the chain to table or incremental. data_tests: - is_empty: - severity: warn \ No newline at end of file + config: + severity: warn \ No newline at end of file diff --git a/models/marts/tests/testing.yml b/models/marts/tests/testing.yml index 36161a9..1b189c6 100644 --- a/models/marts/tests/testing.yml +++ b/models/marts/tests/testing.yml @@ -14,9 +14,11 @@ models: description: the number of models in the project with at least one test configured divided by the total number of models in the project data_tests: - dbt_utils.accepted_range: - name: valid_test_coverage - min_value: "{{ var('test_coverage_target') }}" - severity: warn + config: + name: valid_test_coverage + severity: warn + arguments: + min_value: "{{ var('test_coverage_target') }}" - name: fct_missing_primary_key_tests description: this model has one record for every model without unique and not null tests configured on a single column data_tests: