repo-mirrors/dlt
1
0
Fork 0
mirror of https://github.com/dlt-hub/dlt.git synced 2025-12-17 19:31:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

data quality checks cell in dashboard (#3413)

Browse Source 
* adds hub extra

* makes hub module more user friendly when hub not installed

* test and lint fixes

* adds plugin version check util function

* basic cell appearing if installed

* use data quality cell

* show raw data too

* adds dlt-runtime to hub extra, minimal import tests

* bumps to dlthub 0.20.0 alpha

* lists pipelines with cli using the same functions as dashboard, dlt pipeline will list pipelines by default

* adds configured propfiles method on context so only profiles with configs or pipelines are listed

* adds list of locations that contained actual configs to provider interface

* improves workspace and profile commands

* test fixes

* fixes tests

* update text

* adds quality widget as python functions

* adds data_quality as module to hub

* adds hub extra to docs deps

* fixes dashboard imports

* bumps to alpha x.20.0a1

---------

Co-authored-by: Marcin Rudolf <rudolfix@rudolfix.org>
This commit is contained in:
djudjuu
2025-12-07 12:59:21 +01:00
committed by GitHub
parent c3d8afe51a
commit 289e00dece
13 changed files with 397 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/test_docs.yml vendored
View File

@@ -83,7 +83,7 @@ jobs:
run: cd docs && make dev
- name: Install dlthub incl alpha releases
run: cd docs && uv run pip install --pre dlthub
run: cd docs
- name: lint docs
run: cd docs && make lint

229
dlt/_workspace/helpers/dashboard/dlt_dashboard.py
View File

@@ -38,6 +38,14 @@ def build_header_controls(dlt_profile_select: mo.ui.dropdown) -> Union[List[Any]
return None
@app.function(hide_code=True)
def detect_dlt_hub():
try:
return dlt.hub.__found__
except ImportError:
return False
@app.function
def build_home_header_row(
dlt_profile_select: mo.ui.dropdown,
@@ -446,6 +454,223 @@ def section_schema(
return
@app.cell(hide_code=True)
def ui_data_quality_controls(
dlt_pipeline: dlt.Pipeline,
dlt_section_data_quality_switch: mo.ui.switch,
):
"""
Create data quality filter controls (separate cell for marimo reactivity)
Import the function from the dashboard module and call it.
"""
dlt_data_quality_show_failed_filter: mo.ui.checkbox = None
dlt_data_quality_table_filter: mo.ui.dropdown = None
dlt_data_quality_rate_filter: mo.ui.slider = None
dlt_data_quality_checks_arrow = None
# Create controls whenever dlthub is detected and pipeline exists
# The switch controls whether widget content is shown, not whether controls exist
if detect_dlt_hub() and dlt_pipeline:
try:
# Import the function from the dashboard module
from dlthub.data_quality._dashboard import create_data_quality_controls
# Call the function - returns (checkbox, dropdown, slider, checks_arrow)
(
dlt_data_quality_show_failed_filter,
dlt_data_quality_table_filter,
dlt_data_quality_rate_filter,
dlt_data_quality_checks_arrow,
) = create_data_quality_controls(dlt_pipeline)
except Exception:
pass
return (
dlt_data_quality_show_failed_filter,
dlt_data_quality_table_filter,
dlt_data_quality_rate_filter,
dlt_data_quality_checks_arrow,
)
@app.cell(hide_code=True)
def section_data_quality(
dlt_pipeline: dlt.Pipeline,
dlt_section_data_quality_switch: mo.ui.switch,
dlt_data_quality_show_failed_filter: mo.ui.checkbox,
dlt_data_quality_table_filter: mo.ui.dropdown,
dlt_data_quality_rate_filter: mo.ui.slider,
dlt_data_quality_checks_arrow,
):
"""
Show data quality of the currently selected pipeline
only if dlt.hub is installed
Import the widget function from the dashboard module and call it.
"""
if not detect_dlt_hub():
_result = None
else:
_result = [
ui.section_marker(
strings.data_quality_section_name, has_content=dlt_pipeline is not None
)
]
_result.extend(
ui.build_page_header(
dlt_pipeline,
strings.data_quality_title,
strings.data_quality_subtitle,
strings.data_quality_subtitle,
dlt_section_data_quality_switch,
)
)
if dlt_pipeline and dlt_section_data_quality_switch.value:
try:
# Import the widget function from the dashboard module
from dlthub.data_quality._dashboard import data_quality_widget
# Extract values from controls (must be in separate cell from where controls are created)
show_failed_value = (
dlt_data_quality_show_failed_filter.value
if dlt_data_quality_show_failed_filter is not None
else False
)
table_value = None
if (
dlt_data_quality_table_filter is not None
and dlt_data_quality_table_filter.value != "All"
):
table_value = dlt_data_quality_table_filter.value
rate_value = (
dlt_data_quality_rate_filter.value
if dlt_data_quality_rate_filter is not None
else None
)
# Call the widget function
widget_output = data_quality_widget(
dlt_pipeline=dlt_pipeline,
failure_rate_slider=dlt_data_quality_rate_filter,
failure_rate_filter_value=rate_value,
show_only_failed_checkbox=dlt_data_quality_show_failed_filter,
show_only_failed_value=show_failed_value,
table_dropdown=dlt_data_quality_table_filter,
table_name_filter_value=table_value,
checks_arrow=dlt_data_quality_checks_arrow,
)
if widget_output is not None:
_result.append(widget_output)
# Only show raw table switch if there is data to display
if (
dlt_data_quality_checks_arrow is not None
and dlt_data_quality_checks_arrow.num_rows > 0
):
dlt_data_quality_show_raw_table_switch: mo.ui.switch = mo.ui.switch(
value=False,
label="<small>Show Raw Table</small>",
)
_result.append(
mo.hstack([dlt_data_quality_show_raw_table_switch], justify="start")
)
else:
dlt_data_quality_show_raw_table_switch = None
except ImportError:
_result.append(mo.md("**DLT Hub data quality module is not available.**"))
dlt_data_quality_show_raw_table_switch = None
except Exception as exc:
_result.append(
ui.build_error_callout(
f"Error loading data quality checks: {exc}",
traceback_string=traceback.format_exc(),
)
)
dlt_data_quality_show_raw_table_switch = None
else:
dlt_data_quality_show_raw_table_switch = None
mo.vstack(_result) if _result else None
return dlt_data_quality_show_raw_table_switch
@app.cell(hide_code=True)
def section_data_quality_raw_table(
dlt_pipeline: dlt.Pipeline,
dlt_section_data_quality_switch: mo.ui.switch,
dlt_data_quality_show_raw_table_switch: mo.ui.switch,
dlt_get_last_query_result,
dlt_set_last_query_result,
):
"""
Display the raw data quality checks table with _dlt_load_id column
"""
_result = []
if (
dlt_pipeline
and dlt_section_data_quality_switch.value
and dlt_data_quality_show_raw_table_switch is not None
and dlt_data_quality_show_raw_table_switch.value
):
try:
# Import constants from data_quality module (using private names to avoid conflicts)
from dlthub.data_quality.storage import (
DLT_CHECKS_RESULTS_TABLE_NAME as _DLT_CHECKS_RESULTS_TABLE_NAME,
DLT_DATA_QUALITY_SCHEMA_NAME as _DLT_DATA_QUALITY_SCHEMA_NAME,
)
_error_message: str = None
with mo.status.spinner(title="Loading raw data quality checks table..."):
try:
# Build query to select all columns including _dlt_load_id
_raw_dataset = dlt_pipeline.dataset(schema=_DLT_DATA_QUALITY_SCHEMA_NAME)
_raw_sql_query = (
_raw_dataset.table(_DLT_CHECKS_RESULTS_TABLE_NAME)
.limit(1000)
.to_sql(pretty=True, _raw_query=True)
)
# Execute query
_raw_query_result, _error_message, _traceback_string = utils.get_query_result(
dlt_pipeline, _raw_sql_query
)
dlt_set_last_query_result(_raw_query_result)
except Exception as exc:
_error_message = str(exc)
_traceback_string = traceback.format_exc()
# Display error message if encountered
if _error_message:
_result.append(
ui.build_error_callout(
f"Error loading raw table: {_error_message}",
traceback_string=_traceback_string,
)
)
# Always display result table
_last_result = dlt_get_last_query_result()
if _last_result is not None:
_result.append(mo.ui.table(_last_result, selection=None))
except ImportError:
_result.append(
mo.callout(
mo.md("DLT Hub data quality module is not available."),
kind="warn",
)
)
except Exception as exc:
_result.append(
ui.build_error_callout(
f"Error loading raw table: {exc}",
traceback_string=traceback.format_exc(),
)
)
mo.vstack(_result) if _result else None
return
@app.cell(hide_code=True)
def section_browse_data_table_list(
dlt_clear_query_cache: mo.ui.run_button,
@@ -1151,6 +1376,9 @@ def ui_controls(mo_cli_arg_with_test_identifiers: bool):
dlt_section_ibis_browser_switch: mo.ui.switch = mo.ui.switch(
value=False, label="ibis" if mo_cli_arg_with_test_identifiers else ""
)
dlt_section_data_quality_switch: mo.ui.switch = mo.ui.switch(
value=False, label="data_quality" if mo_cli_arg_with_test_identifiers else ""
)
# other switches
dlt_schema_show_dlt_tables: mo.ui.switch = mo.ui.switch(
@@ -1191,6 +1419,7 @@ def ui_controls(mo_cli_arg_with_test_identifiers: bool):
dlt_schema_show_row_counts,
dlt_schema_show_type_hints,
dlt_section_browse_data_switch,
dlt_section_data_quality_switch,
dlt_section_ibis_browser_switch,
dlt_section_loads_switch,
dlt_section_info_switch,

5
dlt/_workspace/helpers/dashboard/dlt_dashboard_styles.css
View File

@@ -74,7 +74,8 @@
#App .marimo-cell .output-area:has([data-section="home_section"].has-content),
#App .marimo-cell .output-area:has([data-section="schema_section"].has-content),
#App .marimo-cell .output-area:has([data-section="state_section"].has-content),
#App .marimo-cell .output-area:has([data-section="loads_section"].has-content) {
#App .marimo-cell .output-area:has([data-section="loads_section"].has-content),
#App .marimo-cell .output-area:has([data-section="data_quality_section"].has-content) {
background-color: var(--dlt-color-aqua-background);
border: 1px dashed var(--dlt-color-aqua);
}
@@ -157,4 +158,4 @@ marimo-callout-output .border {
.status-badge-grey {
background-color: var(--grey-bg);
color: var(--grey-text);
}
}

6
dlt/_workspace/helpers/dashboard/strings.py
View File

@@ -180,6 +180,12 @@ state_section_name = "state_section"
state_title = "Pipeline State"
state_subtitle = "A raw view of the currently stored pipeline state."
#
# Data quality page
#
data_quality_section_name = "data_quality_section"
data_quality_title = "Data Quality"
data_quality_subtitle = "View the results of your data quality checks"
#
# Last trace page

3
dlt/hub/__init__.py
View File

@@ -7,8 +7,9 @@ __exception__ = None
try:
from dlthub import transformation, runner, data_quality
from dlthub import transformation, runner
from . import current
from . import data_quality
__found__ = True
__all__ = ("transformation", "current", "runner", "data_quality")

2
dlt/hub/current.py
View File

@@ -1,3 +1 @@
"""A collection of dltHub Features"""
from dlthub.current import * # noqa

1
dlt/hub/data_quality.py Normal file
View File

@@ -0,0 +1 @@
from dlthub.data_quality import * # noqa

2
docs/pyproject.toml
View File

@@ -7,7 +7,7 @@ requires-python = ">=3.10, <3.13" # databind not available over 3.10, we need t
# NOTE: working here is always dev enviroment, so we don't need a dev group
dependencies = [
"dlt[duckdb,postgres,bigquery,mssql,databricks,qdrant,sql_database,workspace,weaviate]",
"dlt[duckdb,postgres,bigquery,mssql,databricks,qdrant,sql_database,workspace,weaviate,hub]",
"docstring-parser>=0.11",
"flake8>=7.0.0,<8",
"modal>=1.2.1",

141
docs/uv.lock generated
View File

@@ -473,6 +473,18 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/08/d0/2af09c4077e0d357f33384e4d6fc2c34a3d33e473ae7f939a6c58769774d/connectorx-0.4.4-cp312-none-win_amd64.whl", hash = "sha256:dcf4fb9d1e94ebe0bb4b72a18aeba119895d2fa66b4fe69a8ece97942748c3b0", size = 34561589, upload-time = "2025-08-19T05:38:14.81Z" },
]
[[package]]
name = "cron-descriptor"
version = "2.0.6"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "typing-extensions" },
]
sdist = { url = "https://files.pythonhosted.org/packages/7c/31/0b21d1599656b2ffa6043e51ca01041cd1c0f6dacf5a3e2b620ed120e7d8/cron_descriptor-2.0.6.tar.gz", hash = "sha256:e39d2848e1d8913cfb6e3452e701b5eec662ee18bea8cc5aa53ee1a7bb217157", size = 49456, upload-time = "2025-09-03T16:30:22.434Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/21/cc/361326a54ad92e2e12845ad15e335a4e14b8953665007fb514d3393dfb0f/cron_descriptor-2.0.6-py3-none-any.whl", hash = "sha256:3a1c0d837c0e5a32e415f821b36cf758eb92d510e6beff8fbfe4fa16573d93d6", size = 74446, upload-time = "2025-09-03T16:30:21.397Z" },
]
[[package]]
name = "cryptography"
version = "46.0.3"
@@ -822,7 +834,7 @@ wheels = [
[[package]]
name = "dlt"
version = "1.19.1"
version = "1.20.0a0"
source = { editable = "../" }
dependencies = [
{ name = "click" },
@@ -868,6 +880,10 @@ databricks = [
duckdb = [
{ name = "duckdb" },
]
hub = [
{ name = "dlt-runtime" },
{ name = "dlthub" },
]
mssql = [
{ name = "pyodbc" },
]
@@ -912,6 +928,8 @@ requires-dist = [
{ name = "db-dtypes", marker = "extra == 'bigquery'", specifier = ">=1.2.0" },
{ name = "db-dtypes", marker = "extra == 'gcp'", specifier = ">=1.2.0" },
{ name = "deltalake", marker = "extra == 'deltalake'", specifier = ">=0.25.1" },
{ name = "dlt-runtime", marker = "python_full_version >= '3.10' and extra == 'hub'", specifier = ">=0.19.0a0,<0.21" },
{ name = "dlthub", marker = "python_full_version >= '3.10' and extra == 'hub'", specifier = ">=0.19.0a0,<0.21" },
{ name = "duckdb", marker = "extra == 'duckdb'", specifier = ">=0.9" },
{ name = "duckdb", marker = "extra == 'ducklake'", specifier = ">=1.2.0" },
{ name = "duckdb", marker = "extra == 'motherduck'", specifier = ">=0.9" },
@@ -992,7 +1010,7 @@ requires-dist = [
{ name = "weaviate-client", marker = "extra == 'weaviate'", specifier = ">=3.26.7,<4.0.0" },
{ name = "win-precise-time", marker = "python_full_version < '3.13' and os_name == 'nt'", specifier = ">=1.4.2" },
]
provides-extras = ["gcp", "bigquery", "postgres", "redshift", "parquet", "duckdb", "ducklake", "filesystem", "s3", "gs", "az", "sftp", "http", "snowflake", "motherduck", "cli", "athena", "weaviate", "mssql", "synapse", "qdrant", "databricks", "clickhouse", "dremio", "lancedb", "deltalake", "sql-database", "sqlalchemy", "pyiceberg", "postgis", "workspace", "dbml"]
provides-extras = ["gcp", "bigquery", "postgres", "redshift", "parquet", "duckdb", "ducklake", "filesystem", "s3", "gs", "az", "sftp", "http", "snowflake", "motherduck", "cli", "athena", "weaviate", "mssql", "synapse", "qdrant", "databricks", "clickhouse", "dremio", "lancedb", "deltalake", "sql-database", "sqlalchemy", "pyiceberg", "postgis", "workspace", "hub", "dbml"]
[package.metadata.requires-dev]
adbc = [
@@ -1097,7 +1115,7 @@ dependencies = [
{ name = "databind" },
{ name = "dbt-core" },
{ name = "dbt-duckdb" },
{ name = "dlt", extra = ["bigquery", "databricks", "duckdb", "mssql", "postgres", "qdrant", "sql-database", "weaviate", "workspace"] },
{ name = "dlt", extra = ["bigquery", "databricks", "duckdb", "hub", "mssql", "postgres", "qdrant", "sql-database", "weaviate", "workspace"] },
{ name = "docstring-parser" },
{ name = "flake8" },
{ name = "google-api-python-client" },
@@ -1136,7 +1154,7 @@ requires-dist = [
{ name = "databind", specifier = ">=4.5.2" },
{ name = "dbt-core", specifier = ">=1.5.0" },
{ name = "dbt-duckdb", specifier = ">=1.5.0" },
{ name = "dlt", extras = ["duckdb", "postgres", "bigquery", "mssql", "databricks", "qdrant", "sql-database", "workspace", "weaviate"], editable = "../" },
{ name = "dlt", extras = ["duckdb", "postgres", "bigquery", "mssql", "databricks", "qdrant", "sql-database", "workspace", "weaviate", "hub"], editable = "../" },
{ name = "docstring-parser", specifier = ">=0.11" },
{ name = "flake8", specifier = ">=7.0.0,<8" },
{ name = "google-api-python-client", specifier = ">=1.7.11" },
@@ -1169,6 +1187,36 @@ requires-dist = [
{ name = "weaviate-client", specifier = ">=3.26.7,<4.0.0" },
]
[[package]]
name = "dlt-runtime"
version = "0.20.0a0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "attrs" },
{ name = "cron-descriptor" },
{ name = "httpx" },
{ name = "pathspec" },
{ name = "python-jose" },
{ name = "tabulate" },
]
sdist = { url = "https://files.pythonhosted.org/packages/1d/f5/d5c74ba2560507493b9d5d0c98a8482e15f036f338bb2832730065679e22/dlt_runtime-0.20.0a0.tar.gz", hash = "sha256:3e9d5df91f03152c251e5f874e5a13ac1bd66d5fbac0357c11996bf4e8279c8c", size = 48300, upload-time = "2025-12-04T15:51:51.192Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/21/1f/ebcfea0c69a697d64f836140bfdb0f73d0be2880547dc00ff267eaad7569/dlt_runtime-0.20.0a0-py3-none-any.whl", hash = "sha256:d6498d4078980c833ea9c5cbfc8b7146488beddb77146ae6d3a2e7d7345dfb5a", size = 118233, upload-time = "2025-12-04T15:51:49.67Z" },
]
[[package]]
name = "dlthub"
version = "0.20.0a0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "python-jose" },
{ name = "ruamel-yaml" },
]
sdist = { url = "https://files.pythonhosted.org/packages/da/ff/3484810a588516053ead898e86cf05bc416ca0a06035cfe11440153a518c/dlthub-0.20.0a0.tar.gz", hash = "sha256:34aa26c8103e54913f92bcbbbc55df516e32cfcaf24f176e7562141fabc8c1e6", size = 154938, upload-time = "2025-12-04T18:55:23.034Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/1c/a8/2116d92df6fa1c660eb3ae699f9409bfe968700d6842ad9c7a4c53ce7326/dlthub-0.20.0a0-py3-none-any.whl", hash = "sha256:5c3abc352b5525d5f84f744b511339ff9d728234812824dc6065b29b86615aee", size = 205718, upload-time = "2025-12-04T18:55:20.143Z" },
]
[[package]]
name = "dnspython"
version = "2.8.0"
@@ -1247,6 +1295,18 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/c0/21/08f10706d30252753349ec545833fc0cea67c11abd0b5223acf2827f1056/duckdb-1.4.1-cp312-cp312-win_amd64.whl", hash = "sha256:567f3b3a785a9e8650612461893c49ca799661d2345a6024dda48324ece89ded", size = 12336422, upload-time = "2025-10-07T10:36:57.521Z" },
]
[[package]]
name = "ecdsa"
version = "0.19.1"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "six" },
]
sdist = { url = "https://files.pythonhosted.org/packages/c0/1f/924e3caae75f471eae4b26bd13b698f6af2c44279f67af317439c2f4c46a/ecdsa-0.19.1.tar.gz", hash = "sha256:478cba7b62555866fcb3bb3fe985e06decbdb68ef55713c4e5ab98c57d508e61", size = 201793, upload-time = "2025-03-13T11:52:43.25Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/cb/a3/460c57f094a4a165c84a1341c373b0a4f5ec6ac244b998d5021aade89b77/ecdsa-0.19.1-py2.py3-none-any.whl", hash = "sha256:30638e27cf77b7e15c4c4cc1973720149e1033827cfd00661ca5c8cc0cdb24c3", size = 150607, upload-time = "2025-03-13T11:52:41.757Z" },
]
[[package]]
name = "et-xmlfile"
version = "2.0.0"
@@ -3825,6 +3885,20 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/14/1b/a298b06749107c305e1fe0f814c6c74aea7b2f1e10989cb30f544a1b3253/python_dotenv-1.2.1-py3-none-any.whl", hash = "sha256:b81ee9561e9ca4004139c6cbba3a238c32b03e4894671e181b671e8cb8425d61", size = 21230, upload-time = "2025-10-26T15:12:09.109Z" },
]
[[package]]
name = "python-jose"
version = "3.5.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "ecdsa" },
{ name = "pyasn1" },
{ name = "rsa" },
]
sdist = { url = "https://files.pythonhosted.org/packages/c6/77/3a1c9039db7124eb039772b935f2244fbb73fc8ee65b9acf2375da1c07bf/python_jose-3.5.0.tar.gz", hash = "sha256:fb4eaa44dbeb1c26dcc69e4bd7ec54a1cb8dd64d3b4d81ef08d90ff453f2b01b", size = 92726, upload-time = "2025-05-28T17:31:54.288Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/d9/c3/0bd11992072e6a1c513b16500a5d07f91a24017c5909b02c72c62d7ad024/python_jose-3.5.0-py2.py3-none-any.whl", hash = "sha256:abd1202f23d34dfad2c3d28cb8617b90acf34132c7afd60abd0b0b7d3cb55771", size = 34624, upload-time = "2025-05-28T17:31:52.802Z" },
]
[[package]]
name = "python-multipart"
version = "0.0.20"
@@ -4145,6 +4219,56 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/64/8d/0133e4eb4beed9e425d9a98ed6e081a55d195481b7632472be1af08d2f6b/rsa-4.9.1-py3-none-any.whl", hash = "sha256:68635866661c6836b8d39430f97a996acbd61bfa49406748ea243539fe239762", size = 34696, upload-time = "2025-04-16T09:51:17.142Z" },
]
[[package]]
name = "ruamel-yaml"
version = "0.18.16"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "ruamel-yaml-clib", marker = "platform_python_implementation == 'CPython'" },
]
sdist = { url = "https://files.pythonhosted.org/packages/9f/c7/ee630b29e04a672ecfc9b63227c87fd7a37eb67c1bf30fe95376437f897c/ruamel.yaml-0.18.16.tar.gz", hash = "sha256:a6e587512f3c998b2225d68aa1f35111c29fad14aed561a26e73fab729ec5e5a", size = 147269, upload-time = "2025-10-22T17:54:02.346Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/0f/73/bb1bc2529f852e7bf64a2dec885e89ff9f5cc7bbf6c9340eed30ff2c69c5/ruamel.yaml-0.18.16-py3-none-any.whl", hash = "sha256:048f26d64245bae57a4f9ef6feb5b552a386830ef7a826f235ffb804c59efbba", size = 119858, upload-time = "2025-10-22T17:53:59.012Z" },
]
[[package]]
name = "ruamel-yaml-clib"
version = "0.2.15"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/ea/97/60fda20e2fb54b83a61ae14648b0817c8f5d84a3821e40bfbdae1437026a/ruamel_yaml_clib-0.2.15.tar.gz", hash = "sha256:46e4cc8c43ef6a94885f72512094e482114a8a706d3c555a34ed4b0d20200600", size = 225794, upload-time = "2025-11-16T16:12:59.761Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/f7/5a/4ab767cd42dcd65b83c323e1620d7c01ee60a52f4032fb7b61501f45f5c2/ruamel_yaml_clib-0.2.15-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:88eea8baf72f0ccf232c22124d122a7f26e8a24110a0273d9bcddcb0f7e1fa03", size = 147454, upload-time = "2025-11-16T16:13:02.54Z" },
{ url = "https://files.pythonhosted.org/packages/40/44/184173ac1e74fd35d308108bcbf83904d6ef8439c70763189225a166b238/ruamel_yaml_clib-0.2.15-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9b6f7d74d094d1f3a4e157278da97752f16ee230080ae331fcc219056ca54f77", size = 132467, upload-time = "2025-11-16T16:13:03.539Z" },
{ url = "https://files.pythonhosted.org/packages/49/1b/2d2077a25fe682ae335007ca831aff42e3cbc93c14066675cf87a6c7fc3e/ruamel_yaml_clib-0.2.15-cp310-cp310-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:4be366220090d7c3424ac2b71c90d1044ea34fca8c0b88f250064fd06087e614", size = 693454, upload-time = "2025-11-16T20:22:41.083Z" },
{ url = "https://files.pythonhosted.org/packages/90/16/e708059c4c429ad2e33be65507fc1730641e5f239fb2964efc1ba6edea94/ruamel_yaml_clib-0.2.15-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1f66f600833af58bea694d5892453f2270695b92200280ee8c625ec5a477eed3", size = 700345, upload-time = "2025-11-16T16:13:04.771Z" },
{ url = "https://files.pythonhosted.org/packages/d9/79/0e8ef51df1f0950300541222e3332f20707a9c210b98f981422937d1278c/ruamel_yaml_clib-0.2.15-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:da3d6adadcf55a93c214d23941aef4abfd45652110aed6580e814152f385b862", size = 731306, upload-time = "2025-11-16T16:13:06.312Z" },
{ url = "https://files.pythonhosted.org/packages/a6/f4/2cdb54b142987ddfbd01fc45ac6bd882695fbcedb9d8bbf796adc3fc3746/ruamel_yaml_clib-0.2.15-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:e9fde97ecb7bb9c41261c2ce0da10323e9227555c674989f8d9eb7572fc2098d", size = 692415, upload-time = "2025-11-16T16:13:07.465Z" },
{ url = "https://files.pythonhosted.org/packages/a0/07/40b5fc701cce8240a3e2d26488985d3bbdc446e9fe397c135528d412fea6/ruamel_yaml_clib-0.2.15-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:05c70f7f86be6f7bee53794d80050a28ae7e13e4a0087c1839dcdefd68eb36b6", size = 705007, upload-time = "2025-11-16T20:22:42.856Z" },
{ url = "https://files.pythonhosted.org/packages/82/19/309258a1df6192fb4a77ffa8eae3e8150e8d0ffa56c1b6fa92e450ba2740/ruamel_yaml_clib-0.2.15-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:6f1d38cbe622039d111b69e9ca945e7e3efebb30ba998867908773183357f3ed", size = 723974, upload-time = "2025-11-16T16:13:08.72Z" },
{ url = "https://files.pythonhosted.org/packages/67/3a/d6ee8263b521bfceb5cd2faeb904a15936480f2bb01c7ff74a14ec058ca4/ruamel_yaml_clib-0.2.15-cp310-cp310-win32.whl", hash = "sha256:fe239bdfdae2302e93bd6e8264bd9b71290218fff7084a9db250b55caaccf43f", size = 102836, upload-time = "2025-11-16T16:13:10.27Z" },
{ url = "https://files.pythonhosted.org/packages/ed/03/92aeb5c69018387abc49a8bb4f83b54a0471d9ef48e403b24bac68f01381/ruamel_yaml_clib-0.2.15-cp310-cp310-win_amd64.whl", hash = "sha256:468858e5cbde0198337e6a2a78eda8c3fb148bdf4c6498eaf4bc9ba3f8e780bd", size = 121917, upload-time = "2025-11-16T16:13:12.145Z" },
{ url = "https://files.pythonhosted.org/packages/2c/80/8ce7b9af532aa94dd83360f01ce4716264db73de6bc8efd22c32341f6658/ruamel_yaml_clib-0.2.15-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c583229f336682b7212a43d2fa32c30e643d3076178fb9f7a6a14dde85a2d8bd", size = 147998, upload-time = "2025-11-16T16:13:13.241Z" },
{ url = "https://files.pythonhosted.org/packages/53/09/de9d3f6b6701ced5f276d082ad0f980edf08ca67114523d1b9264cd5e2e0/ruamel_yaml_clib-0.2.15-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:56ea19c157ed8c74b6be51b5fa1c3aff6e289a041575f0556f66e5fb848bb137", size = 132743, upload-time = "2025-11-16T16:13:14.265Z" },
{ url = "https://files.pythonhosted.org/packages/0e/f7/73a9b517571e214fe5c246698ff3ed232f1ef863c8ae1667486625ec688a/ruamel_yaml_clib-0.2.15-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:5fea0932358e18293407feb921d4f4457db837b67ec1837f87074667449f9401", size = 731459, upload-time = "2025-11-16T20:22:44.338Z" },
{ url = "https://files.pythonhosted.org/packages/9b/a2/0dc0013169800f1c331a6f55b1282c1f4492a6d32660a0cf7b89e6684919/ruamel_yaml_clib-0.2.15-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ef71831bd61fbdb7aa0399d5c4da06bea37107ab5c79ff884cc07f2450910262", size = 749289, upload-time = "2025-11-16T16:13:15.633Z" },
{ url = "https://files.pythonhosted.org/packages/aa/ed/3fb20a1a96b8dc645d88c4072df481fe06e0289e4d528ebbdcc044ebc8b3/ruamel_yaml_clib-0.2.15-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:617d35dc765715fa86f8c3ccdae1e4229055832c452d4ec20856136acc75053f", size = 777630, upload-time = "2025-11-16T16:13:16.898Z" },
{ url = "https://files.pythonhosted.org/packages/60/50/6842f4628bc98b7aa4733ab2378346e1441e150935ad3b9f3c3c429d9408/ruamel_yaml_clib-0.2.15-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1b45498cc81a4724a2d42273d6cfc243c0547ad7c6b87b4f774cb7bcc131c98d", size = 744368, upload-time = "2025-11-16T16:13:18.117Z" },
{ url = "https://files.pythonhosted.org/packages/d3/b0/128ae8e19a7d794c2e36130a72b3bb650ce1dd13fb7def6cf10656437dcf/ruamel_yaml_clib-0.2.15-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:def5663361f6771b18646620fca12968aae730132e104688766cf8a3b1d65922", size = 745233, upload-time = "2025-11-16T20:22:45.833Z" },
{ url = "https://files.pythonhosted.org/packages/75/05/91130633602d6ba7ce3e07f8fc865b40d2a09efd4751c740df89eed5caf9/ruamel_yaml_clib-0.2.15-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:014181cdec565c8745b7cbc4de3bf2cc8ced05183d986e6d1200168e5bb59490", size = 770963, upload-time = "2025-11-16T16:13:19.344Z" },
{ url = "https://files.pythonhosted.org/packages/fd/4b/fd4542e7f33d7d1bc64cc9ac9ba574ce8cf145569d21f5f20133336cdc8c/ruamel_yaml_clib-0.2.15-cp311-cp311-win32.whl", hash = "sha256:d290eda8f6ada19e1771b54e5706b8f9807e6bb08e873900d5ba114ced13e02c", size = 102640, upload-time = "2025-11-16T16:13:20.498Z" },
{ url = "https://files.pythonhosted.org/packages/bb/eb/00ff6032c19c7537371e3119287999570867a0eafb0154fccc80e74bf57a/ruamel_yaml_clib-0.2.15-cp311-cp311-win_amd64.whl", hash = "sha256:bdc06ad71173b915167702f55d0f3f027fc61abd975bd308a0968c02db4a4c3e", size = 121996, upload-time = "2025-11-16T16:13:21.855Z" },
{ url = "https://files.pythonhosted.org/packages/72/4b/5fde11a0722d676e469d3d6f78c6a17591b9c7e0072ca359801c4bd17eee/ruamel_yaml_clib-0.2.15-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:cb15a2e2a90c8475df45c0949793af1ff413acfb0a716b8b94e488ea95ce7cff", size = 149088, upload-time = "2025-11-16T16:13:22.836Z" },
{ url = "https://files.pythonhosted.org/packages/85/82/4d08ac65ecf0ef3b046421985e66301a242804eb9a62c93ca3437dc94ee0/ruamel_yaml_clib-0.2.15-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:64da03cbe93c1e91af133f5bec37fd24d0d4ba2418eaf970d7166b0a26a148a2", size = 134553, upload-time = "2025-11-16T16:13:24.151Z" },
{ url = "https://files.pythonhosted.org/packages/b9/cb/22366d68b280e281a932403b76da7a988108287adff2bfa5ce881200107a/ruamel_yaml_clib-0.2.15-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:f6d3655e95a80325b84c4e14c080b2470fe4f33b6846f288379ce36154993fb1", size = 737468, upload-time = "2025-11-16T20:22:47.335Z" },
{ url = "https://files.pythonhosted.org/packages/71/73/81230babf8c9e33770d43ed9056f603f6f5f9665aea4177a2c30ae48e3f3/ruamel_yaml_clib-0.2.15-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:71845d377c7a47afc6592aacfea738cc8a7e876d586dfba814501d8c53c1ba60", size = 753349, upload-time = "2025-11-16T16:13:26.269Z" },
{ url = "https://files.pythonhosted.org/packages/61/62/150c841f24cda9e30f588ef396ed83f64cfdc13b92d2f925bb96df337ba9/ruamel_yaml_clib-0.2.15-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:11e5499db1ccbc7f4b41f0565e4f799d863ea720e01d3e99fa0b7b5fcd7802c9", size = 788211, upload-time = "2025-11-16T16:13:27.441Z" },
{ url = "https://files.pythonhosted.org/packages/30/93/e79bd9cbecc3267499d9ead919bd61f7ddf55d793fb5ef2b1d7d92444f35/ruamel_yaml_clib-0.2.15-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4b293a37dc97e2b1e8a1aec62792d1e52027087c8eea4fc7b5abd2bdafdd6642", size = 743203, upload-time = "2025-11-16T16:13:28.671Z" },
{ url = "https://files.pythonhosted.org/packages/8d/06/1eb640065c3a27ce92d76157f8efddb184bd484ed2639b712396a20d6dce/ruamel_yaml_clib-0.2.15-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:512571ad41bba04eac7268fe33f7f4742210ca26a81fe0c75357fa682636c690", size = 747292, upload-time = "2025-11-16T20:22:48.584Z" },
{ url = "https://files.pythonhosted.org/packages/a5/21/ee353e882350beab65fcc47a91b6bdc512cace4358ee327af2962892ff16/ruamel_yaml_clib-0.2.15-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e5e9f630c73a490b758bf14d859a39f375e6999aea5ddd2e2e9da89b9953486a", size = 771624, upload-time = "2025-11-16T16:13:29.853Z" },
{ url = "https://files.pythonhosted.org/packages/57/34/cc1b94057aa867c963ecf9ea92ac59198ec2ee3a8d22a126af0b4d4be712/ruamel_yaml_clib-0.2.15-cp312-cp312-win32.whl", hash = "sha256:f4421ab780c37210a07d138e56dd4b51f8642187cdfb433eb687fe8c11de0144", size = 100342, upload-time = "2025-11-16T16:13:31.067Z" },
{ url = "https://files.pythonhosted.org/packages/b3/e5/8925a4208f131b218f9a7e459c0d6fcac8324ae35da269cb437894576366/ruamel_yaml_clib-0.2.15-cp312-cp312-win_amd64.whl", hash = "sha256:2b216904750889133d9222b7b873c199d48ecbb12912aca78970f84a5aa1a4bc", size = 119013, upload-time = "2025-11-16T16:13:32.164Z" },
]
[[package]]
name = "ruff"
version = "0.3.7"
@@ -4415,6 +4539,15 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/f8/f9/ce041b9531022a0b5999a47e6da14485239f7bce9c595d1bfb387fe60e89/synchronicity-0.10.2-py3-none-any.whl", hash = "sha256:4ba1f8c02ca582ef068033300201e3c403e08d81e42553554f4e67b27f0d9bb1", size = 38766, upload-time = "2025-07-30T20:23:18.04Z" },
]
[[package]]
name = "tabulate"
version = "0.9.0"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/ec/fe/802052aecb21e3797b8f7902564ab6ea0d60ff8ca23952079064155d1ae1/tabulate-0.9.0.tar.gz", hash = "sha256:0095b12bf5966de529c0feb1fa08671671b3368eec77d7ef7ab114be2c068b3c", size = 81090, upload-time = "2022-10-06T17:21:48.54Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/40/44/4a5f08c96eb108af5cb50b41f76142f0afa346dfa99d5296fe7202a11854/tabulate-0.9.0-py3-none-any.whl", hash = "sha256:024ca478df22e9340661486f85298cff5f6dcdba14f3813e8830015b9ed1948f", size = 35252, upload-time = "2022-10-06T17:21:44.262Z" },
]
[[package]]
name = "tenacity"
version = "9.1.2"

4
pyproject.toml
View File

@@ -1,6 +1,6 @@
[project]
name = "dlt"
version = "1.20.0a0"
version = "1.20.0a1"
description = "dlt is an open-source python-first scalable data loading library that does not require any backend to run."
authors = [{ name = "dltHub Inc.", email = "services@dlthub.com" }]
requires-python = ">=3.9.2, <3.15"
@@ -188,7 +188,7 @@ workspace = [
"pathspec>=0.11.2",
]
hub = [
"dlthub>=0.19.0a0,<0.21 ; python_version >= '3.10'",
"dlthub>=0.20.0a1,<0.21 ; python_version >= '3.10'",
"dlt-runtime>=0.19.0a0,<0.21 ; python_version >= '3.10'",
]

2
tests/e2e/helpers/dashboard/test_e2e.py
View File

@@ -367,7 +367,7 @@ def test_workspace_profile_dev(page: Page):
page.goto(f"http://localhost:{test_port}/?profile=dev&pipeline=fruit_pipeline")
expect(page.get_by_role("switch", name="overview")).to_be_visible()
expect(page.get_by_role("switch", name="overview")).to_be_visible(timeout=20000)
page.get_by_role("switch", name="loads").check()
expect(page.get_by_role("row", name="fruitshop").first).to_be_visible()

19
tests/hub/test_data_quality.py
View File

@@ -1,12 +1,11 @@
import pytest
import dlt
def test_direct_module_import():
"""It's currently not possible to import the module directly"""
with pytest.raises(ModuleNotFoundError):
import dlt.hub.data_quality # type: ignore[import-not-found]
# NOTE: this is still re-import so submodule structure is not importable
from dlt.hub import data_quality as dq
dq.checks.is_in("payment_methods", ["card", "cash", "voucher"]) # type: ignore[attr-defined,unused-ignore]
def test_from_module_import():
@@ -18,8 +17,8 @@ def test_data_quality_entrypoints():
import dlthub.data_quality as dq
# access a single check
assert dlt.hub.data_quality is dq
assert dlt.hub.data_quality.checks is dq.checks
assert dlt.hub.data_quality.checks.is_not_null is dq.checks.is_not_null
assert dlt.hub.data_quality.CheckSuite is dq.CheckSuite
assert dlt.hub.data_quality.prepare_checks is dq.prepare_checks
assert dlt.hub.data_quality is not dq
assert dlt.hub.data_quality.checks is dq.checks # type: ignore[attr-defined,unused-ignore]
assert dlt.hub.data_quality.checks.is_not_null is dq.checks.is_not_null # type: ignore[attr-defined,unused-ignore]
assert dlt.hub.data_quality.CheckSuite is dq.CheckSuite # type: ignore[attr-defined,unused-ignore]
assert dlt.hub.data_quality.prepare_checks is dq.prepare_checks # type: ignore[attr-defined,unused-ignore]

10
uv.lock generated
View File

@@ -2058,7 +2058,7 @@ wheels = [
[[package]]
name = "dlt"
version = "1.20.0a0"
version = "1.20.0a1"
source = { editable = "." }
dependencies = [
{ name = "click" },
@@ -2357,7 +2357,7 @@ requires-dist = [
{ name = "db-dtypes", marker = "extra == 'gcp'", specifier = ">=1.2.0" },
{ name = "deltalake", marker = "extra == 'deltalake'", specifier = ">=0.25.1" },
{ name = "dlt-runtime", marker = "python_full_version >= '3.10' and extra == 'hub'", specifier = ">=0.19.0a0,<0.21" },
{ name = "dlthub", marker = "python_full_version >= '3.10' and extra == 'hub'", specifier = ">=0.19.0a0,<0.21" },
{ name = "dlthub", marker = "python_full_version >= '3.10' and extra == 'hub'", specifier = ">=0.20.0a1,<0.21" },
{ name = "duckdb", marker = "extra == 'duckdb'", specifier = ">=0.9" },
{ name = "duckdb", marker = "extra == 'ducklake'", specifier = ">=1.2.0" },
{ name = "duckdb", marker = "extra == 'motherduck'", specifier = ">=0.9" },
@@ -2552,15 +2552,15 @@ wheels = [
[[package]]
name = "dlthub"
version = "0.20.0a0"
version = "0.20.0a1"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "python-jose", marker = "python_full_version >= '3.10'" },
{ name = "ruamel-yaml", marker = "python_full_version >= '3.10'" },
]
sdist = { url = "https://files.pythonhosted.org/packages/da/ff/3484810a588516053ead898e86cf05bc416ca0a06035cfe11440153a518c/dlthub-0.20.0a0.tar.gz", hash = "sha256:34aa26c8103e54913f92bcbbbc55df516e32cfcaf24f176e7562141fabc8c1e6", size = 154938, upload-time = "2025-12-04T18:55:23.034Z" }
sdist = { url = "https://files.pythonhosted.org/packages/13/20/eb0246c88dd80102a4b02ad807b3459c31855ac0c983f69a2b8fe027a88c/dlthub-0.20.0a1.tar.gz", hash = "sha256:bf8e90bd22bad85ed67d497f97d763678e74a41c70bb3c17e3ae2da7897a0bde", size = 158441, upload-time = "2025-12-07T11:26:45.312Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/1c/a8/2116d92df6fa1c660eb3ae699f9409bfe968700d6842ad9c7a4c53ce7326/dlthub-0.20.0a0-py3-none-any.whl", hash = "sha256:5c3abc352b5525d5f84f744b511339ff9d728234812824dc6065b29b86615aee", size = 205718, upload-time = "2025-12-04T18:55:20.143Z" },
{ url = "https://files.pythonhosted.org/packages/40/7b/307126b81ecefbf2dd7569e2da81b2a7b9124dbdcfa6039f3e350bd097c3/dlthub-0.20.0a1-py3-none-any.whl", hash = "sha256:de1a20567da74943931e8fd3551babd345c9f7dcb1f3782d200b308a79c660ce", size = 209794, upload-time = "2025-12-07T11:26:43.808Z" },
]
[[package]]