mirror of
https://github.com/dbt-labs/dbt-core
synced 2025-12-22 10:01:27 +00:00
Compare commits
19 Commits
enable-pos
...
court/defa
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
f020cfb09a | ||
|
|
b71a738bd0 | ||
|
|
287434813f | ||
|
|
adb997a714 | ||
|
|
97351e0a34 | ||
|
|
1f31af1f94 | ||
|
|
4be1870afb | ||
|
|
c07a152052 | ||
|
|
96b79a87ee | ||
|
|
9ab091584f | ||
|
|
d3e33625fc | ||
|
|
40bb8e7e03 | ||
|
|
c9507c1804 | ||
|
|
3e6f775791 | ||
|
|
f660f32987 | ||
|
|
e32cadb5a7 | ||
|
|
2cd734cc2e | ||
|
|
ee9016eda0 | ||
|
|
4745814081 |
6
.changes/unreleased/Features-20240625-095107.yaml
Normal file
6
.changes/unreleased/Features-20240625-095107.yaml
Normal file
@@ -0,0 +1,6 @@
|
||||
kind: Features
|
||||
body: Support cumulative_type_params & sub-daily granularities in semantic manifest.
|
||||
time: 2024-06-25T09:51:07.983248-07:00
|
||||
custom:
|
||||
Author: courtneyholcomb
|
||||
Issue: "10360"
|
||||
6
.changes/unreleased/Features-20240627-162953.yaml
Normal file
6
.changes/unreleased/Features-20240627-162953.yaml
Normal file
@@ -0,0 +1,6 @@
|
||||
kind: Features
|
||||
body: Add default_granularity to metric spec.
|
||||
time: 2024-06-27T16:29:53.500917-07:00
|
||||
custom:
|
||||
Author: courtneyholcomb
|
||||
Issue: "10376"
|
||||
6
.changes/unreleased/Fixes-20240624-171729.yaml
Normal file
6
.changes/unreleased/Fixes-20240624-171729.yaml
Normal file
@@ -0,0 +1,6 @@
|
||||
kind: Fixes
|
||||
body: Fix setting `silence` of `warn_error_options` via `dbt_project.yaml` flags
|
||||
time: 2024-06-24T17:17:29.464865-07:00
|
||||
custom:
|
||||
Author: QMalcolm
|
||||
Issue: "10160"
|
||||
@@ -20,10 +20,12 @@ repos:
|
||||
- "markdown"
|
||||
- id: check-case-conflict
|
||||
- repo: https://github.com/pycqa/isort
|
||||
rev: 5.12.0
|
||||
# rev must match what's in dev-requirements.txt
|
||||
rev: 5.13.2
|
||||
hooks:
|
||||
- id: isort
|
||||
- repo: https://github.com/psf/black
|
||||
# rev must match what's in dev-requirements.txt
|
||||
rev: 22.3.0
|
||||
hooks:
|
||||
- id: black
|
||||
@@ -34,6 +36,7 @@ repos:
|
||||
- "--check"
|
||||
- "--diff"
|
||||
- repo: https://github.com/pycqa/flake8
|
||||
# rev must match what's in dev-requirements.txt
|
||||
rev: 4.0.1
|
||||
hooks:
|
||||
- id: flake8
|
||||
@@ -41,6 +44,7 @@ repos:
|
||||
alias: flake8-check
|
||||
stages: [manual]
|
||||
- repo: https://github.com/pre-commit/mirrors-mypy
|
||||
# rev must match what's in dev-requirements.txt
|
||||
rev: v1.4.1
|
||||
hooks:
|
||||
- id: mypy
|
||||
|
||||
26
codecov.yml
26
codecov.yml
@@ -1,6 +1,7 @@
|
||||
ignore:
|
||||
- ".github"
|
||||
- ".changes"
|
||||
|
||||
coverage:
|
||||
status:
|
||||
project:
|
||||
@@ -11,3 +12,28 @@ coverage:
|
||||
default:
|
||||
target: auto
|
||||
threshold: 80%
|
||||
|
||||
comment:
|
||||
layout: "header, diff, flags, components" # show component info in the PR comment
|
||||
|
||||
component_management:
|
||||
default_rules: # default rules that will be inherited by all components
|
||||
statuses:
|
||||
- type: project # in this case every component that doens't have a status defined will have a project type one
|
||||
target: auto
|
||||
threshold: 0.1%
|
||||
- type: patch
|
||||
target: 80%
|
||||
individual_components:
|
||||
- component_id: unittests
|
||||
name: "Unit Tests"
|
||||
flag_regexes:
|
||||
- "unit"
|
||||
statuses:
|
||||
- type: patch
|
||||
target: 80%
|
||||
threshold: 5%
|
||||
- component_id: integrationtests
|
||||
name: "Integration Tests"
|
||||
flag_regexes:
|
||||
- "integration"
|
||||
|
||||
@@ -38,6 +38,7 @@ from dbt.artifacts.resources.v1.macro import Macro, MacroArgument, MacroDependsO
|
||||
from dbt.artifacts.resources.v1.metric import (
|
||||
ConstantPropertyInput,
|
||||
ConversionTypeParams,
|
||||
CumulativeTypeParams,
|
||||
Metric,
|
||||
MetricConfig,
|
||||
MetricInput,
|
||||
|
||||
@@ -6,6 +6,7 @@ from dbt_semantic_interfaces.references import MeasureReference, MetricReference
|
||||
from dbt_semantic_interfaces.type_enums import (
|
||||
ConversionCalculationType,
|
||||
MetricType,
|
||||
PeriodAggregation,
|
||||
TimeGranularity,
|
||||
)
|
||||
|
||||
@@ -80,6 +81,13 @@ class ConversionTypeParams(dbtClassMixin):
|
||||
constant_properties: Optional[List[ConstantPropertyInput]] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class CumulativeTypeParams(dbtClassMixin):
|
||||
window: Optional[MetricTimeWindow] = None
|
||||
grain_to_date: Optional[TimeGranularity] = None
|
||||
period_agg: PeriodAggregation = PeriodAggregation.FIRST
|
||||
|
||||
|
||||
@dataclass
|
||||
class MetricTypeParams(dbtClassMixin):
|
||||
measure: Optional[MetricInputMeasure] = None
|
||||
@@ -91,6 +99,7 @@ class MetricTypeParams(dbtClassMixin):
|
||||
grain_to_date: Optional[TimeGranularity] = None
|
||||
metrics: Optional[List[MetricInput]] = None
|
||||
conversion_type_params: Optional[ConversionTypeParams] = None
|
||||
cumulative_type_params: Optional[CumulativeTypeParams] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -113,6 +122,7 @@ class Metric(GraphResource):
|
||||
type_params: MetricTypeParams
|
||||
filter: Optional[WhereFilterIntersection] = None
|
||||
metadata: Optional[SourceFileMetadata] = None
|
||||
default_granularity: TimeGranularity = TimeGranularity.DAY
|
||||
resource_type: Literal[NodeType.Metric]
|
||||
meta: Dict[str, Any] = field(default_factory=dict, metadata=MergeBehavior.Update.meta())
|
||||
tags: List[str] = field(default_factory=list)
|
||||
|
||||
@@ -57,6 +57,7 @@ def convert_config(config_name, config_value):
|
||||
ret = WarnErrorOptions(
|
||||
include=config_value.get("include", []),
|
||||
exclude=config_value.get("exclude", []),
|
||||
silence=config_value.get("silence", []),
|
||||
valid_error_names=ALL_EVENT_NAMES,
|
||||
)
|
||||
return ret
|
||||
|
||||
@@ -15,5 +15,6 @@ DEPENDENCIES_FILE_NAME = "dependencies.yml"
|
||||
PACKAGE_LOCK_FILE_NAME = "package-lock.yml"
|
||||
MANIFEST_FILE_NAME = "manifest.json"
|
||||
SEMANTIC_MANIFEST_FILE_NAME = "semantic_manifest.json"
|
||||
TIME_SPINE_MODEL_NAME = "metricflow_time_spine"
|
||||
PARTIAL_PARSE_FILE_NAME = "partial_parse.msgpack"
|
||||
PACKAGE_LOCK_HASH_KEY = "sha1_hash"
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
## Artifacts
|
||||
|
||||
### Generating JSON schemas
|
||||
A helper script, `sripts/collect-artifact-schema.py` is available to generate json schemas corresponding to versioned artifacts (`ArtifactMixin`s).
|
||||
A helper script, `scripts/collect-artifact-schema.py` is available to generate json schemas corresponding to versioned artifacts (`ArtifactMixin`s).
|
||||
|
||||
This script is necessary to run when a new artifact schema version is created, or when changes are made to existing artifact versions, and writes json schema to `schema/dbt/<artifact>/v<version>.json`.
|
||||
|
||||
|
||||
@@ -15,6 +15,7 @@ from dbt_semantic_interfaces.validations.semantic_manifest_validator import (
|
||||
SemanticManifestValidator,
|
||||
)
|
||||
|
||||
from dbt.constants import TIME_SPINE_MODEL_NAME
|
||||
from dbt.events.types import SemanticValidationFailure
|
||||
from dbt.exceptions import ParsingError
|
||||
from dbt_common.clients.system import write_file
|
||||
@@ -82,8 +83,7 @@ class SemanticManifest:
|
||||
# Look for time-spine table model and create time spine table configuration
|
||||
if self.manifest.semantic_models:
|
||||
# Get model for time_spine_table
|
||||
time_spine_model_name = "metricflow_time_spine"
|
||||
model = self.manifest.ref_lookup.find(time_spine_model_name, None, None, self.manifest)
|
||||
model = self.manifest.ref_lookup.find(TIME_SPINE_MODEL_NAME, None, None, self.manifest)
|
||||
if not model:
|
||||
raise ParsingError(
|
||||
"The semantic layer requires a 'metricflow_time_spine' model in the project, but none was found. "
|
||||
|
||||
@@ -4,7 +4,11 @@ from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Literal, Optional, Sequence, Union
|
||||
|
||||
from dbt_semantic_interfaces.type_enums import ConversionCalculationType
|
||||
from dbt_semantic_interfaces.type_enums import (
|
||||
ConversionCalculationType,
|
||||
PeriodAggregation,
|
||||
TimeGranularity,
|
||||
)
|
||||
|
||||
# trigger the PathEncoder
|
||||
import dbt_common.helper_types # noqa:F401
|
||||
@@ -532,6 +536,13 @@ class UnparsedConversionTypeParams(dbtClassMixin):
|
||||
constant_properties: Optional[List[ConstantPropertyInput]] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class UnparsedCumulativeTypeParams(dbtClassMixin):
|
||||
window: Optional[str] = None
|
||||
grain_to_date: Optional[str] = None
|
||||
period_agg: str = PeriodAggregation.FIRST.value
|
||||
|
||||
|
||||
@dataclass
|
||||
class UnparsedMetricTypeParams(dbtClassMixin):
|
||||
measure: Optional[Union[UnparsedMetricInputMeasure, str]] = None
|
||||
@@ -542,6 +553,7 @@ class UnparsedMetricTypeParams(dbtClassMixin):
|
||||
grain_to_date: Optional[str] = None # str is really a TimeGranularity Enum
|
||||
metrics: Optional[List[Union[UnparsedMetricInput, str]]] = None
|
||||
conversion_type_params: Optional[UnparsedConversionTypeParams] = None
|
||||
cumulative_type_params: Optional[UnparsedCumulativeTypeParams] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -553,6 +565,7 @@ class UnparsedMetric(dbtClassMixin):
|
||||
description: str = ""
|
||||
# Note: `Union` must be the outermost part of the type annotation for serialization to work properly.
|
||||
filter: Union[str, List[str], None] = None
|
||||
default_granularity: str = TimeGranularity.DAY.value
|
||||
# metadata: Optional[Unparsedetadata] = None # TODO
|
||||
meta: Dict[str, Any] = field(default_factory=dict)
|
||||
tags: List[str] = field(default_factory=list)
|
||||
|
||||
@@ -6,11 +6,13 @@ from dbt_semantic_interfaces.type_enums import (
|
||||
DimensionType,
|
||||
EntityType,
|
||||
MetricType,
|
||||
PeriodAggregation,
|
||||
TimeGranularity,
|
||||
)
|
||||
|
||||
from dbt.artifacts.resources import (
|
||||
ConversionTypeParams,
|
||||
CumulativeTypeParams,
|
||||
Dimension,
|
||||
DimensionTypeParams,
|
||||
Entity,
|
||||
@@ -42,6 +44,7 @@ from dbt.context.providers import (
|
||||
from dbt.contracts.graph.nodes import Exposure, Group, Metric, SavedQuery, SemanticModel
|
||||
from dbt.contracts.graph.unparsed import (
|
||||
UnparsedConversionTypeParams,
|
||||
UnparsedCumulativeTypeParams,
|
||||
UnparsedDimension,
|
||||
UnparsedDimensionTypeParams,
|
||||
UnparsedEntity,
|
||||
@@ -221,9 +224,19 @@ class MetricParser(YamlReader):
|
||||
|
||||
return input_measures
|
||||
|
||||
def _get_time_window(
|
||||
self,
|
||||
unparsed_window: Optional[str],
|
||||
def _get_period_agg(self, unparsed_period_agg: str) -> PeriodAggregation:
|
||||
return PeriodAggregation(unparsed_period_agg)
|
||||
|
||||
def _get_optional_grain_to_date(
|
||||
self, unparsed_grain_to_date: Optional[str]
|
||||
) -> Optional[TimeGranularity]:
|
||||
if not unparsed_grain_to_date:
|
||||
return None
|
||||
|
||||
return TimeGranularity(unparsed_grain_to_date)
|
||||
|
||||
def _get_optional_time_window(
|
||||
self, unparsed_window: Optional[str]
|
||||
) -> Optional[MetricTimeWindow]:
|
||||
if unparsed_window is not None:
|
||||
parts = unparsed_window.split(" ")
|
||||
@@ -277,7 +290,7 @@ class MetricParser(YamlReader):
|
||||
name=unparsed.name,
|
||||
filter=parse_where_filter(unparsed.filter),
|
||||
alias=unparsed.alias,
|
||||
offset_window=self._get_time_window(unparsed.offset_window),
|
||||
offset_window=self._get_optional_time_window(unparsed.offset_window),
|
||||
offset_to_grain=offset_to_grain,
|
||||
)
|
||||
|
||||
@@ -311,11 +324,48 @@ class MetricParser(YamlReader):
|
||||
conversion_measure=self._get_input_measure(unparsed.conversion_measure),
|
||||
entity=unparsed.entity,
|
||||
calculation=ConversionCalculationType(unparsed.calculation),
|
||||
window=self._get_time_window(unparsed.window),
|
||||
window=self._get_optional_time_window(unparsed.window),
|
||||
constant_properties=unparsed.constant_properties,
|
||||
)
|
||||
|
||||
def _get_metric_type_params(self, type_params: UnparsedMetricTypeParams) -> MetricTypeParams:
|
||||
def _get_optional_cumulative_type_params(
|
||||
self, unparsed_metric: UnparsedMetric
|
||||
) -> Optional[CumulativeTypeParams]:
|
||||
unparsed_type_params = unparsed_metric.type_params
|
||||
if unparsed_metric.type.lower() == MetricType.CUMULATIVE.value:
|
||||
if not unparsed_type_params.cumulative_type_params:
|
||||
unparsed_type_params.cumulative_type_params = UnparsedCumulativeTypeParams()
|
||||
|
||||
if (
|
||||
unparsed_type_params.window
|
||||
and not unparsed_type_params.cumulative_type_params.window
|
||||
):
|
||||
unparsed_type_params.cumulative_type_params.window = unparsed_type_params.window
|
||||
if (
|
||||
unparsed_type_params.grain_to_date
|
||||
and not unparsed_type_params.cumulative_type_params.grain_to_date
|
||||
):
|
||||
unparsed_type_params.cumulative_type_params.grain_to_date = (
|
||||
unparsed_type_params.grain_to_date
|
||||
)
|
||||
|
||||
return CumulativeTypeParams(
|
||||
window=self._get_optional_time_window(
|
||||
unparsed_type_params.cumulative_type_params.window
|
||||
),
|
||||
grain_to_date=self._get_optional_grain_to_date(
|
||||
unparsed_type_params.cumulative_type_params.grain_to_date
|
||||
),
|
||||
period_agg=self._get_period_agg(
|
||||
unparsed_type_params.cumulative_type_params.period_agg
|
||||
),
|
||||
)
|
||||
|
||||
return None
|
||||
|
||||
def _get_metric_type_params(self, unparsed_metric: UnparsedMetric) -> MetricTypeParams:
|
||||
type_params = unparsed_metric.type_params
|
||||
|
||||
grain_to_date: Optional[TimeGranularity] = None
|
||||
if type_params.grain_to_date is not None:
|
||||
grain_to_date = TimeGranularity(type_params.grain_to_date)
|
||||
@@ -325,12 +375,15 @@ class MetricParser(YamlReader):
|
||||
numerator=self._get_optional_metric_input(type_params.numerator),
|
||||
denominator=self._get_optional_metric_input(type_params.denominator),
|
||||
expr=str(type_params.expr) if type_params.expr is not None else None,
|
||||
window=self._get_time_window(type_params.window),
|
||||
window=self._get_optional_time_window(type_params.window),
|
||||
grain_to_date=grain_to_date,
|
||||
metrics=self._get_metric_inputs(type_params.metrics),
|
||||
conversion_type_params=self._get_optional_conversion_type_params(
|
||||
type_params.conversion_type_params
|
||||
)
|
||||
),
|
||||
cumulative_type_params=self._get_optional_cumulative_type_params(
|
||||
unparsed_metric=unparsed_metric,
|
||||
),
|
||||
# input measures are calculated via metric processing post parsing
|
||||
# input_measures=?,
|
||||
)
|
||||
@@ -380,7 +433,8 @@ class MetricParser(YamlReader):
|
||||
description=unparsed.description,
|
||||
label=unparsed.label,
|
||||
type=MetricType(unparsed.type),
|
||||
type_params=self._get_metric_type_params(unparsed.type_params),
|
||||
type_params=self._get_metric_type_params(unparsed),
|
||||
default_granularity=TimeGranularity(unparsed.default_granularity),
|
||||
filter=parse_where_filter(unparsed.filter),
|
||||
meta=unparsed.meta,
|
||||
tags=unparsed.tags,
|
||||
|
||||
@@ -69,7 +69,7 @@ setup(
|
||||
# Accept patches but avoid automatically updating past a set minor version range.
|
||||
"dbt-extractor>=0.5.0,<=0.6",
|
||||
"minimal-snowplow-tracker>=0.0.2,<0.1",
|
||||
"dbt-semantic-interfaces>=0.5.1,<0.7",
|
||||
"dbt-semantic-interfaces>=0.6.1,<0.7",
|
||||
# Minor versions for these are expected to be backwards-compatible
|
||||
"dbt-common>=1.3.0,<2.0",
|
||||
"dbt-adapters>=1.1.1,<2.0",
|
||||
|
||||
@@ -2,16 +2,20 @@ git+https://github.com/dbt-labs/dbt-adapters.git@main
|
||||
git+https://github.com/dbt-labs/dbt-adapters.git@main#subdirectory=dbt-tests-adapter
|
||||
git+https://github.com/dbt-labs/dbt-common.git@main
|
||||
git+https://github.com/dbt-labs/dbt-postgres.git@main
|
||||
black>=24.3.0,<25.0
|
||||
# black must match what's in .pre-commit-config.yaml to be sure local env matches CI
|
||||
black==22.3.0
|
||||
bumpversion
|
||||
ddtrace==2.3.0
|
||||
docutils
|
||||
flake8
|
||||
# flake8 must match what's in .pre-commit-config.yaml to be sure local env matches CI
|
||||
flake8==4.0.1
|
||||
flaky
|
||||
freezegun>=1.4.0,<1.5
|
||||
hypothesis
|
||||
ipdb
|
||||
isort>=5.12,<6
|
||||
# isort must match what's in .pre-commit-config.yaml to be sure local env matches CI
|
||||
isort==5.13.2
|
||||
# mypy must match what's in .pre-commit-config.yaml to be sure local env matches CI
|
||||
mypy==1.4.1
|
||||
pip-tools
|
||||
pre-commit
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
},
|
||||
"dbt_version": {
|
||||
"type": "string",
|
||||
"default": "1.8.0a1"
|
||||
"default": "1.9.0a1"
|
||||
},
|
||||
"generated_at": {
|
||||
"type": "string"
|
||||
|
||||
@@ -13,7 +13,7 @@
|
||||
},
|
||||
"dbt_version": {
|
||||
"type": "string",
|
||||
"default": "1.8.0b3"
|
||||
"default": "1.9.0a1"
|
||||
},
|
||||
"generated_at": {
|
||||
"type": "string"
|
||||
@@ -4388,18 +4388,10 @@
|
||||
"default": null
|
||||
},
|
||||
"primary_key": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"additionalProperties": false,
|
||||
@@ -8257,6 +8249,12 @@
|
||||
},
|
||||
"granularity": {
|
||||
"enum": [
|
||||
"nanosecond",
|
||||
"microsecond",
|
||||
"millisecond",
|
||||
"second",
|
||||
"minute",
|
||||
"hour",
|
||||
"day",
|
||||
"week",
|
||||
"month",
|
||||
@@ -8281,6 +8279,12 @@
|
||||
"anyOf": [
|
||||
{
|
||||
"enum": [
|
||||
"nanosecond",
|
||||
"microsecond",
|
||||
"millisecond",
|
||||
"second",
|
||||
"minute",
|
||||
"hour",
|
||||
"day",
|
||||
"week",
|
||||
"month",
|
||||
@@ -8371,6 +8375,12 @@
|
||||
},
|
||||
"granularity": {
|
||||
"enum": [
|
||||
"nanosecond",
|
||||
"microsecond",
|
||||
"millisecond",
|
||||
"second",
|
||||
"minute",
|
||||
"hour",
|
||||
"day",
|
||||
"week",
|
||||
"month",
|
||||
@@ -8395,6 +8405,12 @@
|
||||
"anyOf": [
|
||||
{
|
||||
"enum": [
|
||||
"nanosecond",
|
||||
"microsecond",
|
||||
"millisecond",
|
||||
"second",
|
||||
"minute",
|
||||
"hour",
|
||||
"day",
|
||||
"week",
|
||||
"month",
|
||||
@@ -8442,6 +8458,12 @@
|
||||
},
|
||||
"granularity": {
|
||||
"enum": [
|
||||
"nanosecond",
|
||||
"microsecond",
|
||||
"millisecond",
|
||||
"second",
|
||||
"minute",
|
||||
"hour",
|
||||
"day",
|
||||
"week",
|
||||
"month",
|
||||
@@ -8466,6 +8488,12 @@
|
||||
"anyOf": [
|
||||
{
|
||||
"enum": [
|
||||
"nanosecond",
|
||||
"microsecond",
|
||||
"millisecond",
|
||||
"second",
|
||||
"minute",
|
||||
"hour",
|
||||
"day",
|
||||
"week",
|
||||
"month",
|
||||
@@ -8546,6 +8574,12 @@
|
||||
},
|
||||
"granularity": {
|
||||
"enum": [
|
||||
"nanosecond",
|
||||
"microsecond",
|
||||
"millisecond",
|
||||
"second",
|
||||
"minute",
|
||||
"hour",
|
||||
"day",
|
||||
"week",
|
||||
"month",
|
||||
@@ -8570,6 +8604,12 @@
|
||||
"anyOf": [
|
||||
{
|
||||
"enum": [
|
||||
"nanosecond",
|
||||
"microsecond",
|
||||
"millisecond",
|
||||
"second",
|
||||
"minute",
|
||||
"hour",
|
||||
"day",
|
||||
"week",
|
||||
"month",
|
||||
@@ -8769,6 +8809,12 @@
|
||||
},
|
||||
"granularity": {
|
||||
"enum": [
|
||||
"nanosecond",
|
||||
"microsecond",
|
||||
"millisecond",
|
||||
"second",
|
||||
"minute",
|
||||
"hour",
|
||||
"day",
|
||||
"week",
|
||||
"month",
|
||||
@@ -8830,6 +8876,89 @@
|
||||
}
|
||||
],
|
||||
"default": null
|
||||
},
|
||||
"cumulative_type_params": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "object",
|
||||
"title": "CumulativeTypeParams",
|
||||
"properties": {
|
||||
"window": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "object",
|
||||
"title": "MetricTimeWindow",
|
||||
"properties": {
|
||||
"count": {
|
||||
"type": "integer"
|
||||
},
|
||||
"granularity": {
|
||||
"enum": [
|
||||
"nanosecond",
|
||||
"microsecond",
|
||||
"millisecond",
|
||||
"second",
|
||||
"minute",
|
||||
"hour",
|
||||
"day",
|
||||
"week",
|
||||
"month",
|
||||
"quarter",
|
||||
"year"
|
||||
]
|
||||
}
|
||||
},
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"count",
|
||||
"granularity"
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null
|
||||
},
|
||||
"grain_to_date": {
|
||||
"anyOf": [
|
||||
{
|
||||
"enum": [
|
||||
"nanosecond",
|
||||
"microsecond",
|
||||
"millisecond",
|
||||
"second",
|
||||
"minute",
|
||||
"hour",
|
||||
"day",
|
||||
"week",
|
||||
"month",
|
||||
"quarter",
|
||||
"year"
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null
|
||||
},
|
||||
"period_agg": {
|
||||
"enum": [
|
||||
"first",
|
||||
"last",
|
||||
"average"
|
||||
],
|
||||
"default": "first"
|
||||
}
|
||||
},
|
||||
"additionalProperties": false
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null
|
||||
}
|
||||
},
|
||||
"additionalProperties": false
|
||||
@@ -8915,6 +9044,22 @@
|
||||
],
|
||||
"default": null
|
||||
},
|
||||
"default_granularity": {
|
||||
"enum": [
|
||||
"nanosecond",
|
||||
"microsecond",
|
||||
"millisecond",
|
||||
"second",
|
||||
"minute",
|
||||
"hour",
|
||||
"day",
|
||||
"week",
|
||||
"month",
|
||||
"quarter",
|
||||
"year"
|
||||
],
|
||||
"default": "day"
|
||||
},
|
||||
"meta": {
|
||||
"type": "object",
|
||||
"propertyNames": {
|
||||
@@ -13453,18 +13598,10 @@
|
||||
"default": null
|
||||
},
|
||||
"primary_key": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"additionalProperties": false,
|
||||
@@ -17104,6 +17241,12 @@
|
||||
},
|
||||
"granularity": {
|
||||
"enum": [
|
||||
"nanosecond",
|
||||
"microsecond",
|
||||
"millisecond",
|
||||
"second",
|
||||
"minute",
|
||||
"hour",
|
||||
"day",
|
||||
"week",
|
||||
"month",
|
||||
@@ -17128,6 +17271,12 @@
|
||||
"anyOf": [
|
||||
{
|
||||
"enum": [
|
||||
"nanosecond",
|
||||
"microsecond",
|
||||
"millisecond",
|
||||
"second",
|
||||
"minute",
|
||||
"hour",
|
||||
"day",
|
||||
"week",
|
||||
"month",
|
||||
@@ -17218,6 +17367,12 @@
|
||||
},
|
||||
"granularity": {
|
||||
"enum": [
|
||||
"nanosecond",
|
||||
"microsecond",
|
||||
"millisecond",
|
||||
"second",
|
||||
"minute",
|
||||
"hour",
|
||||
"day",
|
||||
"week",
|
||||
"month",
|
||||
@@ -17242,6 +17397,12 @@
|
||||
"anyOf": [
|
||||
{
|
||||
"enum": [
|
||||
"nanosecond",
|
||||
"microsecond",
|
||||
"millisecond",
|
||||
"second",
|
||||
"minute",
|
||||
"hour",
|
||||
"day",
|
||||
"week",
|
||||
"month",
|
||||
@@ -17289,6 +17450,12 @@
|
||||
},
|
||||
"granularity": {
|
||||
"enum": [
|
||||
"nanosecond",
|
||||
"microsecond",
|
||||
"millisecond",
|
||||
"second",
|
||||
"minute",
|
||||
"hour",
|
||||
"day",
|
||||
"week",
|
||||
"month",
|
||||
@@ -17313,6 +17480,12 @@
|
||||
"anyOf": [
|
||||
{
|
||||
"enum": [
|
||||
"nanosecond",
|
||||
"microsecond",
|
||||
"millisecond",
|
||||
"second",
|
||||
"minute",
|
||||
"hour",
|
||||
"day",
|
||||
"week",
|
||||
"month",
|
||||
@@ -17393,6 +17566,12 @@
|
||||
},
|
||||
"granularity": {
|
||||
"enum": [
|
||||
"nanosecond",
|
||||
"microsecond",
|
||||
"millisecond",
|
||||
"second",
|
||||
"minute",
|
||||
"hour",
|
||||
"day",
|
||||
"week",
|
||||
"month",
|
||||
@@ -17417,6 +17596,12 @@
|
||||
"anyOf": [
|
||||
{
|
||||
"enum": [
|
||||
"nanosecond",
|
||||
"microsecond",
|
||||
"millisecond",
|
||||
"second",
|
||||
"minute",
|
||||
"hour",
|
||||
"day",
|
||||
"week",
|
||||
"month",
|
||||
@@ -17616,6 +17801,12 @@
|
||||
},
|
||||
"granularity": {
|
||||
"enum": [
|
||||
"nanosecond",
|
||||
"microsecond",
|
||||
"millisecond",
|
||||
"second",
|
||||
"minute",
|
||||
"hour",
|
||||
"day",
|
||||
"week",
|
||||
"month",
|
||||
@@ -17677,6 +17868,89 @@
|
||||
}
|
||||
],
|
||||
"default": null
|
||||
},
|
||||
"cumulative_type_params": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "object",
|
||||
"title": "CumulativeTypeParams",
|
||||
"properties": {
|
||||
"window": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "object",
|
||||
"title": "MetricTimeWindow",
|
||||
"properties": {
|
||||
"count": {
|
||||
"type": "integer"
|
||||
},
|
||||
"granularity": {
|
||||
"enum": [
|
||||
"nanosecond",
|
||||
"microsecond",
|
||||
"millisecond",
|
||||
"second",
|
||||
"minute",
|
||||
"hour",
|
||||
"day",
|
||||
"week",
|
||||
"month",
|
||||
"quarter",
|
||||
"year"
|
||||
]
|
||||
}
|
||||
},
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"count",
|
||||
"granularity"
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null
|
||||
},
|
||||
"grain_to_date": {
|
||||
"anyOf": [
|
||||
{
|
||||
"enum": [
|
||||
"nanosecond",
|
||||
"microsecond",
|
||||
"millisecond",
|
||||
"second",
|
||||
"minute",
|
||||
"hour",
|
||||
"day",
|
||||
"week",
|
||||
"month",
|
||||
"quarter",
|
||||
"year"
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null
|
||||
},
|
||||
"period_agg": {
|
||||
"enum": [
|
||||
"first",
|
||||
"last",
|
||||
"average"
|
||||
],
|
||||
"default": "first"
|
||||
}
|
||||
},
|
||||
"additionalProperties": false
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"default": null
|
||||
}
|
||||
},
|
||||
"additionalProperties": false
|
||||
@@ -17762,6 +18036,22 @@
|
||||
],
|
||||
"default": null
|
||||
},
|
||||
"default_granularity": {
|
||||
"enum": [
|
||||
"nanosecond",
|
||||
"microsecond",
|
||||
"millisecond",
|
||||
"second",
|
||||
"minute",
|
||||
"hour",
|
||||
"day",
|
||||
"week",
|
||||
"month",
|
||||
"quarter",
|
||||
"year"
|
||||
],
|
||||
"default": "day"
|
||||
},
|
||||
"meta": {
|
||||
"type": "object",
|
||||
"propertyNames": {
|
||||
@@ -17930,26 +18220,7 @@
|
||||
"type": "string"
|
||||
},
|
||||
"resource_type": {
|
||||
"enum": [
|
||||
"model",
|
||||
"analysis",
|
||||
"test",
|
||||
"snapshot",
|
||||
"operation",
|
||||
"seed",
|
||||
"rpc",
|
||||
"sql_operation",
|
||||
"doc",
|
||||
"source",
|
||||
"macro",
|
||||
"exposure",
|
||||
"metric",
|
||||
"group",
|
||||
"saved_query",
|
||||
"semantic_model",
|
||||
"unit_test",
|
||||
"fixture"
|
||||
]
|
||||
"const": "saved_query"
|
||||
},
|
||||
"package_name": {
|
||||
"type": "string"
|
||||
@@ -18741,6 +19012,12 @@
|
||||
"properties": {
|
||||
"time_granularity": {
|
||||
"enum": [
|
||||
"nanosecond",
|
||||
"microsecond",
|
||||
"millisecond",
|
||||
"second",
|
||||
"minute",
|
||||
"hour",
|
||||
"day",
|
||||
"week",
|
||||
"month",
|
||||
@@ -19468,26 +19745,7 @@
|
||||
"type": "string"
|
||||
},
|
||||
"resource_type": {
|
||||
"enum": [
|
||||
"model",
|
||||
"analysis",
|
||||
"test",
|
||||
"snapshot",
|
||||
"operation",
|
||||
"seed",
|
||||
"rpc",
|
||||
"sql_operation",
|
||||
"doc",
|
||||
"source",
|
||||
"macro",
|
||||
"exposure",
|
||||
"metric",
|
||||
"group",
|
||||
"saved_query",
|
||||
"semantic_model",
|
||||
"unit_test",
|
||||
"fixture"
|
||||
]
|
||||
"const": "saved_query"
|
||||
},
|
||||
"package_name": {
|
||||
"type": "string"
|
||||
@@ -20286,6 +20544,12 @@
|
||||
"properties": {
|
||||
"time_granularity": {
|
||||
"enum": [
|
||||
"nanosecond",
|
||||
"microsecond",
|
||||
"millisecond",
|
||||
"second",
|
||||
"minute",
|
||||
"hour",
|
||||
"day",
|
||||
"week",
|
||||
"month",
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
},
|
||||
"dbt_version": {
|
||||
"type": "string",
|
||||
"default": "1.8.0a1"
|
||||
"default": "1.9.0a1"
|
||||
},
|
||||
"generated_at": {
|
||||
"type": "string"
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
},
|
||||
"dbt_version": {
|
||||
"type": "string",
|
||||
"default": "1.8.0a1"
|
||||
"default": "1.9.0a1"
|
||||
},
|
||||
"generated_at": {
|
||||
"type": "string"
|
||||
|
||||
@@ -53,9 +53,8 @@ class TestWarnErrorOptionsFromCLI:
|
||||
assert_deprecation_warning(result, catcher)
|
||||
|
||||
catcher.flush()
|
||||
runner.invoke(
|
||||
["run", "--warn-error-options", "{'include': 'all', 'silence': ['DeprecatedModel']}"]
|
||||
)
|
||||
result = runner.invoke(["run", "--warn-error-options", "{'silence': ['DeprecatedModel']}"])
|
||||
assert result.success
|
||||
assert len(catcher.caught_events) == 0
|
||||
|
||||
def test_can_raise_warning_to_error(
|
||||
@@ -131,13 +130,12 @@ class TestWarnErrorOptionsFromProject:
|
||||
result = runner.invoke(["run"])
|
||||
assert_deprecation_warning(result, catcher)
|
||||
|
||||
silence_options = {
|
||||
"flags": {"warn_error_options": {"include": "all", "silence": ["DeprecatedModel"]}}
|
||||
}
|
||||
silence_options = {"flags": {"warn_error_options": {"silence": ["DeprecatedModel"]}}}
|
||||
update_config_file(silence_options, project_root, "dbt_project.yml")
|
||||
|
||||
catcher.flush()
|
||||
runner.invoke(["run"])
|
||||
result = runner.invoke(["run"])
|
||||
assert result.success
|
||||
assert len(catcher.caught_events) == 0
|
||||
|
||||
def test_can_raise_warning_to_error(
|
||||
|
||||
@@ -113,6 +113,7 @@ metrics:
|
||||
type: simple
|
||||
type_params:
|
||||
measure: people
|
||||
default_granularity: month
|
||||
config:
|
||||
meta:
|
||||
my_meta_config: 'config'
|
||||
@@ -744,6 +745,59 @@ semantic_models:
|
||||
|
||||
"""
|
||||
|
||||
cumulative_metric_yml = """
|
||||
version: 2
|
||||
metrics:
|
||||
- name: weekly_visits
|
||||
label: Rolling sum of visits over the last 7 days
|
||||
type: cumulative
|
||||
type_params:
|
||||
measure: num_visits
|
||||
cumulative_type_params:
|
||||
window: 7 days
|
||||
period_agg: average
|
||||
- name: cumulative_orders
|
||||
label: Rolling total of orders (all time)
|
||||
type: cumulative
|
||||
type_params:
|
||||
measure: num_orders
|
||||
cumulative_type_params:
|
||||
period_agg: last
|
||||
- name: orders_ytd
|
||||
label: Total orders since the start of the year
|
||||
type: cumulative
|
||||
type_params:
|
||||
measure: num_orders
|
||||
cumulative_type_params:
|
||||
grain_to_date: year
|
||||
period_agg: first
|
||||
- name: monthly_orders
|
||||
label: Orders in the past month
|
||||
type: cumulative
|
||||
type_params:
|
||||
measure: num_orders
|
||||
window: 1 month
|
||||
cumulative_type_params:
|
||||
period_agg: average
|
||||
- name: yearly_orders
|
||||
label: Orders in the past year
|
||||
type: cumulative
|
||||
type_params:
|
||||
measure: num_orders
|
||||
window: 1 year
|
||||
- name: visits_mtd
|
||||
label: Visits since start of month
|
||||
type: cumulative
|
||||
type_params:
|
||||
measure: num_visits
|
||||
grain_to_date: month
|
||||
- name: cumulative_visits
|
||||
label: Rolling total of visits (all time)
|
||||
type: cumulative
|
||||
type_params:
|
||||
measure: num_visits
|
||||
"""
|
||||
|
||||
conversion_metric_yml = """
|
||||
version: 2
|
||||
metrics:
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
import pytest
|
||||
from dbt_semantic_interfaces.type_enums.period_agg import PeriodAggregation
|
||||
from dbt_semantic_interfaces.type_enums.time_granularity import TimeGranularity
|
||||
|
||||
from dbt.artifacts.resources.v1.metric import CumulativeTypeParams, MetricTimeWindow
|
||||
from dbt.cli.main import dbtRunner
|
||||
from dbt.contracts.graph.manifest import Manifest
|
||||
from dbt.exceptions import ParsingError
|
||||
@@ -8,6 +11,7 @@ from tests.functional.metrics.fixtures import (
|
||||
basic_metrics_yml,
|
||||
conversion_metric_yml,
|
||||
conversion_semantic_model_purchasing_yml,
|
||||
cumulative_metric_yml,
|
||||
derived_metric_yml,
|
||||
downstream_model_sql,
|
||||
duplicate_measure_metric_yml,
|
||||
@@ -79,6 +83,14 @@ class TestSimpleMetrics:
|
||||
)
|
||||
== 2
|
||||
)
|
||||
assert (
|
||||
manifest.metrics["metric.test.number_of_people"].default_granularity
|
||||
== TimeGranularity.MONTH
|
||||
)
|
||||
assert (
|
||||
manifest.metrics["metric.test.collective_tenure"].default_granularity
|
||||
== TimeGranularity.DAY
|
||||
)
|
||||
|
||||
|
||||
class TestInvalidRefMetrics:
|
||||
@@ -402,6 +414,66 @@ class TestConversionMetric:
|
||||
)
|
||||
|
||||
|
||||
class TestCumulativeMetric:
|
||||
@pytest.fixture(scope="class")
|
||||
def models(self):
|
||||
return {
|
||||
"purchasing.sql": purchasing_model_sql,
|
||||
"metricflow_time_spine.sql": metricflow_time_spine_sql,
|
||||
"semantic_models.yml": conversion_semantic_model_purchasing_yml,
|
||||
"conversion_metric.yml": cumulative_metric_yml,
|
||||
}
|
||||
|
||||
@pytest.fixture(scope="class")
|
||||
def seeds(self):
|
||||
return {"mock_purchase_data.csv": mock_purchase_data_csv}
|
||||
|
||||
def test_cumulative_metric(self, project):
|
||||
# initial parse
|
||||
runner = dbtRunner()
|
||||
result = runner.invoke(["parse"])
|
||||
assert result.success
|
||||
assert isinstance(result.result, Manifest)
|
||||
|
||||
manifest = get_manifest(project.project_root)
|
||||
metric_ids = set(manifest.metrics.keys())
|
||||
expected_metric_ids_to_cumulative_type_params = {
|
||||
"metric.test.weekly_visits": CumulativeTypeParams(
|
||||
window=MetricTimeWindow(count=7, granularity=TimeGranularity.DAY),
|
||||
period_agg=PeriodAggregation.AVERAGE,
|
||||
),
|
||||
"metric.test.cumulative_orders": CumulativeTypeParams(
|
||||
period_agg=PeriodAggregation.LAST
|
||||
),
|
||||
"metric.test.orders_ytd": CumulativeTypeParams(
|
||||
grain_to_date=TimeGranularity.YEAR, period_agg=PeriodAggregation.FIRST
|
||||
),
|
||||
"metric.test.monthly_orders": CumulativeTypeParams(
|
||||
window=MetricTimeWindow(count=1, granularity=TimeGranularity.MONTH),
|
||||
period_agg=PeriodAggregation.AVERAGE,
|
||||
),
|
||||
"metric.test.yearly_orders": CumulativeTypeParams(
|
||||
window=MetricTimeWindow(count=1, granularity=TimeGranularity.YEAR),
|
||||
period_agg=PeriodAggregation.FIRST,
|
||||
),
|
||||
"metric.test.visits_mtd": CumulativeTypeParams(
|
||||
grain_to_date=TimeGranularity.MONTH, period_agg=PeriodAggregation.FIRST
|
||||
),
|
||||
"metric.test.cumulative_visits": CumulativeTypeParams(
|
||||
period_agg=PeriodAggregation.FIRST
|
||||
),
|
||||
}
|
||||
assert metric_ids == set(expected_metric_ids_to_cumulative_type_params.keys())
|
||||
for (
|
||||
metric_id,
|
||||
expected_cumulative_type_params,
|
||||
) in expected_metric_ids_to_cumulative_type_params.items():
|
||||
assert (
|
||||
manifest.metrics[metric_id].type_params.cumulative_type_params
|
||||
== expected_cumulative_type_params
|
||||
), f"Found unexpected cumulative type params for {metric_id}"
|
||||
|
||||
|
||||
class TestFilterParsing:
|
||||
@pytest.fixture(scope="class")
|
||||
def models(self):
|
||||
|
||||
@@ -896,6 +896,7 @@ class TestUnparsedMetric(ContractTestCase):
|
||||
},
|
||||
},
|
||||
"config": {},
|
||||
"default_granularity": "day",
|
||||
"tags": [],
|
||||
"meta": {"is_okr": True},
|
||||
}
|
||||
|
||||
@@ -23,6 +23,7 @@ from hypothesis.strategies import builds, none, text
|
||||
from dbt.artifacts.resources import (
|
||||
ConstantPropertyInput,
|
||||
ConversionTypeParams,
|
||||
CumulativeTypeParams,
|
||||
Defaults,
|
||||
Dimension,
|
||||
DimensionTypeParams,
|
||||
@@ -245,9 +246,18 @@ def conversion_type_params(
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def cumulative_type_params() -> CumulativeTypeParams:
|
||||
return CumulativeTypeParams()
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def complex_metric_type_params(
|
||||
metric_time_window, simple_metric_input, simple_metric_input_measure
|
||||
metric_time_window,
|
||||
simple_metric_input,
|
||||
simple_metric_input_measure,
|
||||
conversion_type_params,
|
||||
cumulative_type_params,
|
||||
) -> MetricTypeParams:
|
||||
return MetricTypeParams(
|
||||
measure=simple_metric_input_measure,
|
||||
@@ -258,6 +268,7 @@ def complex_metric_type_params(
|
||||
grain_to_date=TimeGranularity.DAY,
|
||||
metrics=[simple_metric_input],
|
||||
conversion_type_params=conversion_type_params,
|
||||
cumulative_type_params=cumulative_type_params,
|
||||
)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user