mirror of
https://github.com/dlt-hub/dlt.git
synced 2025-12-17 19:31:30 +00:00
docs: add runtime docs to CLI reference (#3445)
* bumps to version 1.20.0 * update the hub reference docs, add CI check * use dependency specifier in hub for plugin version check * minimum dlt runtime cli check * rollaback to old fsspec min version * fixes test_hub ci workflow * fixes flaky test * bumps hub extra * updates cli docs linting * fixes docs lock --------- Co-authored-by: Marcin Rudolf <rudolfix@rudolfix.org> Co-authored-by: ivasio <ivan@dlthub.com>
This commit is contained in:
ivasio
GitHub
11
.github/workflows/test_hub.yml
vendored
11
.github/workflows/test_hub.yml
vendored
@@ -94,6 +94,17 @@ jobs:
|
||||
run: pytest tests/hub
|
||||
# if: matrix.os != 'macos-latest'
|
||||
|
||||
- name: Test runtime client
|
||||
run: |
|
||||
mkdir .dlt && touch .dlt/.workspace
|
||||
dlt runtime --help
|
||||
|
||||
# DISABLED: because docs rendering happens in non-deterministic order (of plugin discovery)
|
||||
# must be fixed
|
||||
# - name: Check that dlthub cli docs are up to date
|
||||
# run: cd docs/tools/dlthub_cli && make check-cli-docs
|
||||
# if: ${{ matrix.python-version == '3.11' && matrix.os == 'ubuntu-latest' }}
|
||||
|
||||
matrix_job_required_check:
|
||||
name: hub | dlthub features tests
|
||||
needs: run_hub_features
|
||||
|
||||
@@ -34,7 +34,7 @@ COPY dist/dlt-${IMAGE_VERSION}.tar.gz .
|
||||
RUN mkdir -p /app
|
||||
WORKDIR /app
|
||||
RUN uv venv && uv pip install /tmp/pydlt/dlt-${IMAGE_VERSION}.tar.gz --resolution lowest-direct && uv pip install typing-extensions==4.8.0
|
||||
RUN rm -r /tmp/pydlt
|
||||
# RUN rm -r /tmp/pydlt
|
||||
|
||||
# make sure dlt can be actually imported
|
||||
RUN uv run python -c 'import dlt;import pendulum;'
|
||||
@@ -50,7 +50,12 @@ RUN uv run dlt pipeline fruit_pipeline info
|
||||
|
||||
# enable workspace
|
||||
RUN mkdir -p .dlt && touch .dlt/.workspace
|
||||
# RUN dlt pipeline fruit_pipeline info
|
||||
RUN uv run dlt workspace info
|
||||
RUN uv run dlt workspace -v info
|
||||
RUN uv run python minimal_pipeline.py
|
||||
RUN uv run dlt pipeline fruit_pipeline info
|
||||
RUN uv run dlt pipeline fruit_pipeline info
|
||||
|
||||
# install hub
|
||||
RUN uv pip install /tmp/pydlt/dlt-${IMAGE_VERSION}.tar.gz[hub] --resolution lowest-direct && uv pip install typing-extensions==4.8.0
|
||||
RUN uv run python minimal_pipeline.py
|
||||
RUN uv run dlt --non-interactive license issue dlthub.transformation
|
||||
RUN uv run dlt runtime --help
|
||||
@@ -6,7 +6,7 @@ from types import ModuleType
|
||||
from typing import Any, Dict, Iterator, List, Optional
|
||||
from urllib.parse import urlencode
|
||||
|
||||
from packaging.version import Version
|
||||
from packaging.specifiers import SpecifierSet
|
||||
|
||||
from dlt.common import known_env
|
||||
from dlt.common.configuration.container import Container
|
||||
@@ -246,9 +246,14 @@ def ensure_plugin_version_match(
|
||||
plugin_version: str,
|
||||
plugin_module_name: str,
|
||||
dlt_extra: str,
|
||||
dlt_version_specifier: Optional[SpecifierSet] = None,
|
||||
) -> None:
|
||||
"""Ensures that installed dlt version matches plugin version. Plugins are tightly bound to `dlt`
|
||||
and released together. Both major and minor version must match. For alpha plugins version may be 0.
|
||||
"""Ensures that installed plugin version matches dlt requirements. Plugins are tightly bound
|
||||
to `dlt` and released together.
|
||||
|
||||
If `dlt_version_specifier` is provided, it is used to check if the plugin version satisfies
|
||||
the specifier. Otherwise, the specifier is read from dlt's package metadata (Requires-Dist).
|
||||
If specifier cannot be determined, the function returns without checking.
|
||||
|
||||
Args:
|
||||
pkg_name: Name of the plugin package (e.g., "dlthub")
|
||||
@@ -256,30 +261,37 @@ def ensure_plugin_version_match(
|
||||
plugin_version: The installed plugin version string
|
||||
plugin_module_name: The module name for MissingDependencyException (e.g., "dlthub")
|
||||
dlt_extra: The dlt extra to install the plugin (e.g., "hub")
|
||||
dlt_version_specifier: Optional version specifier for the plugin. If not provided,
|
||||
reads from dlt's package metadata.
|
||||
|
||||
Raises:
|
||||
MissingDependencyException: If version mismatch is detected
|
||||
"""
|
||||
installed = Version(plugin_version)
|
||||
dlt_installed = Version(dlt_version)
|
||||
# Get specifier from dlt's package metadata if not provided
|
||||
if dlt_version_specifier is None:
|
||||
from dlt.version import get_dependency_requirement
|
||||
|
||||
# currently packages must match on minor version
|
||||
if installed.minor != dlt_installed.minor or (
|
||||
installed.major != dlt_installed.major and installed.major != 0
|
||||
):
|
||||
req = get_dependency_requirement(pkg_name)
|
||||
if req is not None:
|
||||
dlt_version_specifier = req.specifier
|
||||
|
||||
# If specifier still not available, exit without checking
|
||||
if dlt_version_specifier is None or len(dlt_version_specifier) == 0:
|
||||
return
|
||||
|
||||
# Use specifier.contains() for proper version check (allowing prereleases)
|
||||
if not dlt_version_specifier.contains(plugin_version, prereleases=True):
|
||||
from dlt.common.exceptions import MissingDependencyException
|
||||
|
||||
custom_msg = (
|
||||
f"`{pkg_name}` is a `dlt` plugin and must be installed together with `dlt` with a "
|
||||
f"matching version. `dlt` {dlt_installed.major}.{dlt_installed.minor}.x requires "
|
||||
f"`{pkg_name}` 0.{dlt_installed.minor}.x but you have "
|
||||
f"{plugin_version}. Please install the right version of {pkg_name} with:\n\n"
|
||||
f"`{pkg_name}` is a `dlt` plugin and must satisfy version requirement "
|
||||
f"`{dlt_version_specifier}` but you have {plugin_version}. "
|
||||
f"Please install the right version of {pkg_name} with:\n\n"
|
||||
f'pip install "dlt[{dlt_extra}]=={dlt_version}"\n\n'
|
||||
"or if you are upgrading the plugin:\n\n"
|
||||
f'pip install "dlt[{dlt_extra}]=={dlt_version}" -U {pkg_name}'
|
||||
)
|
||||
missing_dep_ex = MissingDependencyException(plugin_module_name, [])
|
||||
# ImportError uses `msg` attribute for __str__, not just args
|
||||
missing_dep_ex.args = (custom_msg,)
|
||||
missing_dep_ex.msg = custom_msg
|
||||
raise missing_dep_ex
|
||||
|
||||
@@ -65,10 +65,16 @@ MTIME_DISPATCH = {
|
||||
"az": lambda f: ensure_pendulum_datetime_utc(f["last_modified"]),
|
||||
"gcs": lambda f: ensure_pendulum_datetime_utc(f["updated"]),
|
||||
"https": lambda f: cast(
|
||||
pendulum.DateTime, pendulum.parse(f["Last-Modified"], exact=True, strict=False)
|
||||
pendulum.DateTime,
|
||||
pendulum.parse(
|
||||
f.get("Last-Modified", pendulum.now().isoformat()), exact=True, strict=False
|
||||
),
|
||||
),
|
||||
"http": lambda f: cast(
|
||||
pendulum.DateTime, pendulum.parse(f["Last-Modified"], exact=True, strict=False)
|
||||
pendulum.DateTime,
|
||||
pendulum.parse(
|
||||
f.get("Last-Modified", pendulum.now().isoformat()), exact=True, strict=False
|
||||
),
|
||||
),
|
||||
"file": lambda f: ensure_pendulum_datetime_utc(f["mtime"]),
|
||||
"memory": lambda f: ensure_pendulum_datetime_utc(f["created"]),
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
from importlib.metadata import version as pkg_version, distribution as pkg_distribution
|
||||
from typing import Optional
|
||||
from urllib.request import url2pathname
|
||||
from urllib.parse import urlparse
|
||||
from packaging.requirements import Requirement
|
||||
|
||||
DLT_IMPORT_NAME = "dlt"
|
||||
PKG_NAME = DLT_PKG_NAME = "dlt"
|
||||
@@ -30,3 +32,19 @@ def get_installed_requirement_string(
|
||||
else:
|
||||
package_requirement = f"{package}{ver_selector}{pkg_version(package)}"
|
||||
return package_requirement
|
||||
|
||||
|
||||
def get_dependency_requirement(
|
||||
dependency_name: str, package: str = DLT_PKG_NAME
|
||||
) -> Optional[Requirement]:
|
||||
"""Find a specific dependency requirement from package metadata"""
|
||||
dist = pkg_distribution(package)
|
||||
|
||||
if dist.requires is None:
|
||||
return None
|
||||
|
||||
for req_str in dist.requires:
|
||||
req = Requirement(req_str)
|
||||
if req.name == dependency_name:
|
||||
return req
|
||||
return None
|
||||
|
||||
@@ -1,11 +1,14 @@
|
||||
.PHONY: install-dlthub, update-cli-docs, check-cli-docs
|
||||
.PHONY: update-cli-docs, check-cli-docs, dev
|
||||
# this must be run from `dlthub_cli` to see workspace commands
|
||||
# it will use dlthub and dlt-runtime versions from dlt/docs/pyproject.toml to generate docs
|
||||
|
||||
install-dlthub:
|
||||
uv pip install dlthub
|
||||
dev:
|
||||
uv sync
|
||||
|
||||
update-cli-docs: install-dlthub
|
||||
uv run dlt --debug render-docs ../../website/docs/hub/command-line-interface.md --commands license workspace profile
|
||||
update-cli-docs: dev
|
||||
# generate as there's no license
|
||||
RUNTIME__LICENSE="" uv run dlt --debug render-docs ../../website/docs/hub/command-line-interface.md --commands license workspace profile runtime
|
||||
|
||||
check-cli-docs: install-dlthub
|
||||
uv run dlt --debug render-docs ../../website/docs/hub/command-line-interface.md --compare --commands license workspace profile
|
||||
check-cli-docs: dev
|
||||
# generate as there's no license
|
||||
RUNTIME__LICENSE="" uv run dlt --debug render-docs ../../website/docs/hub/command-line-interface.md --compare --commands license workspace profile runtime
|
||||
|
||||
18
docs/uv.lock
generated
18
docs/uv.lock
generated
@@ -834,7 +834,7 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "dlt"
|
||||
version = "1.20.0a1"
|
||||
version = "1.20.0"
|
||||
source = { editable = "../" }
|
||||
dependencies = [
|
||||
{ name = "click" },
|
||||
@@ -928,13 +928,13 @@ requires-dist = [
|
||||
{ name = "db-dtypes", marker = "extra == 'bigquery'", specifier = ">=1.2.0" },
|
||||
{ name = "db-dtypes", marker = "extra == 'gcp'", specifier = ">=1.2.0" },
|
||||
{ name = "deltalake", marker = "extra == 'deltalake'", specifier = ">=0.25.1" },
|
||||
{ name = "dlt-runtime", marker = "python_full_version >= '3.10' and extra == 'hub'", specifier = ">=0.19.0a0,<0.21" },
|
||||
{ name = "dlt-runtime", marker = "python_full_version >= '3.10' and extra == 'hub'", specifier = ">=0.20.0a0,<0.21" },
|
||||
{ name = "dlthub", marker = "python_full_version >= '3.10' and extra == 'hub'", specifier = ">=0.20.0a1,<0.21" },
|
||||
{ name = "duckdb", marker = "extra == 'duckdb'", specifier = ">=0.9" },
|
||||
{ name = "duckdb", marker = "extra == 'ducklake'", specifier = ">=1.2.0" },
|
||||
{ name = "duckdb", marker = "extra == 'motherduck'", specifier = ">=0.9" },
|
||||
{ name = "duckdb", marker = "extra == 'workspace'", specifier = ">=0.9" },
|
||||
{ name = "fsspec", specifier = ">=2025.9.0" },
|
||||
{ name = "fsspec", specifier = ">=2022.4.0" },
|
||||
{ name = "gcsfs", marker = "extra == 'bigquery'", specifier = ">=2022.4.0" },
|
||||
{ name = "gcsfs", marker = "extra == 'clickhouse'", specifier = ">=2022.4.0" },
|
||||
{ name = "gcsfs", marker = "extra == 'gcp'", specifier = ">=2022.4.0" },
|
||||
@@ -1189,7 +1189,7 @@ requires-dist = [
|
||||
|
||||
[[package]]
|
||||
name = "dlt-runtime"
|
||||
version = "0.20.0a0"
|
||||
version = "0.20.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "attrs" },
|
||||
@@ -1199,22 +1199,22 @@ dependencies = [
|
||||
{ name = "python-jose" },
|
||||
{ name = "tabulate" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/1d/f5/d5c74ba2560507493b9d5d0c98a8482e15f036f338bb2832730065679e22/dlt_runtime-0.20.0a0.tar.gz", hash = "sha256:3e9d5df91f03152c251e5f874e5a13ac1bd66d5fbac0357c11996bf4e8279c8c", size = 48300, upload-time = "2025-12-04T15:51:51.192Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/d0/86/d7f057d8bdf2f3ada28bf1277b7f24a7abbb221d72788bd682176126a75c/dlt_runtime-0.20.0.tar.gz", hash = "sha256:753c7522bc01c92a453459640e482f87b647b14cc5734d754133a91968acc79f", size = 49532, upload-time = "2025-12-09T14:32:33.708Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/21/1f/ebcfea0c69a697d64f836140bfdb0f73d0be2880547dc00ff267eaad7569/dlt_runtime-0.20.0a0-py3-none-any.whl", hash = "sha256:d6498d4078980c833ea9c5cbfc8b7146488beddb77146ae6d3a2e7d7345dfb5a", size = 118233, upload-time = "2025-12-04T15:51:49.67Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/4c/b0/02a6c846d89e3c27a592a929d95526036be1d607e48fca214dcbf3b7bf58/dlt_runtime-0.20.0-py3-none-any.whl", hash = "sha256:0969165672b2b3938a618ddd263e0cf8ec356d289253f58134e325e222753056", size = 119573, upload-time = "2025-12-09T14:32:32.119Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "dlthub"
|
||||
version = "0.20.0a1"
|
||||
version = "0.20.1"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "python-jose" },
|
||||
{ name = "ruamel-yaml" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/13/20/eb0246c88dd80102a4b02ad807b3459c31855ac0c983f69a2b8fe027a88c/dlthub-0.20.0a1.tar.gz", hash = "sha256:bf8e90bd22bad85ed67d497f97d763678e74a41c70bb3c17e3ae2da7897a0bde", size = 158441, upload-time = "2025-12-07T11:26:45.312Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/77/1b/2c079f22243462e914026172094411ed7ef1fc96c8089e0ca66d1a14038a/dlthub-0.20.1.tar.gz", hash = "sha256:7b3a188abc28601fd4bdf8f17e7925ef729d4f91fb67a6b4eb5c5dc5a04ac3a2", size = 158432, upload-time = "2025-12-09T15:18:10.813Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/40/7b/307126b81ecefbf2dd7569e2da81b2a7b9124dbdcfa6039f3e350bd097c3/dlthub-0.20.0a1-py3-none-any.whl", hash = "sha256:de1a20567da74943931e8fd3551babd345c9f7dcb1f3782d200b308a79c660ce", size = 209794, upload-time = "2025-12-07T11:26:43.808Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/21/94/b2a87853102c6aa08606b2708d8f678b1e39855e8227fe111e37c32631b6/dlthub-0.20.1-py3-none-any.whl", hash = "sha256:c4d4e0c4515cd68f316ccd02c9ecc007332c861ae6f92a488f7e961935e7f1a0", size = 209767, upload-time = "2025-12-09T15:18:09.067Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,6 +1,6 @@
|
||||
[project]
|
||||
name = "dlt"
|
||||
version = "1.20.0a1"
|
||||
version = "1.20.0"
|
||||
description = "dlt is an open-source python-first scalable data loading library that does not require any backend to run."
|
||||
authors = [{ name = "dltHub Inc.", email = "services@dlthub.com" }]
|
||||
requires-python = ">=3.9.2, <3.15"
|
||||
@@ -52,7 +52,7 @@ dependencies = [
|
||||
"orjson>=3.11.0 ; python_version > '3.13'",
|
||||
"tenacity>=8.0.2",
|
||||
"jsonpath-ng>=1.5.3",
|
||||
"fsspec>=2025.9.0",
|
||||
"fsspec>=2022.4.0",
|
||||
"packaging>=21.1",
|
||||
"pluggy>=1.3.0",
|
||||
"win-precise-time>=1.4.2 ; os_name == 'nt' and python_version < '3.13'",
|
||||
@@ -189,7 +189,7 @@ workspace = [
|
||||
]
|
||||
hub = [
|
||||
"dlthub>=0.20.0a1,<0.21 ; python_version >= '3.10'",
|
||||
"dlt-runtime>=0.19.0a0,<0.21 ; python_version >= '3.10'",
|
||||
"dlt-runtime>=0.20.0a0,<0.21 ; python_version >= '3.10'",
|
||||
]
|
||||
|
||||
dbml = [
|
||||
|
||||
@@ -174,31 +174,73 @@ def test_context_with_xdg_dir(mocker) -> None:
|
||||
|
||||
def test_ensure_plugin_version_match_same_versions() -> None:
|
||||
"""test that matching versions pass without error."""
|
||||
from packaging.specifiers import SpecifierSet
|
||||
|
||||
# Use explicit specifier to test specific version matching scenarios
|
||||
# PEP 440 ordering: .devN < .aN < .bN < .rcN < final < .postN
|
||||
# So we use .dev0 as lower bound to include all pre-releases
|
||||
specifier_1_19 = SpecifierSet(">=1.19.0.dev0,<1.20.0") # includes all prereleases
|
||||
specifier_2_5 = SpecifierSet(">=2.5.0.dev0,<2.6.0")
|
||||
|
||||
# exact same version
|
||||
ensure_plugin_version_match("dlthub", "1.19.0", "1.19.0", "dlthub", "hub")
|
||||
ensure_plugin_version_match("dlthub", "1.19.5", "1.19.2", "dlthub", "hub")
|
||||
ensure_plugin_version_match(
|
||||
"fake-plugin", "1.19.0", "1.19.0", "fake-plugin", "hub", specifier_1_19
|
||||
)
|
||||
ensure_plugin_version_match(
|
||||
"fake-plugin", "1.19.5", "1.19.2", "fake-plugin", "hub", specifier_1_19
|
||||
)
|
||||
# different patch versions are ok
|
||||
ensure_plugin_version_match("dlthub", "2.5.0", "2.5.10", "dlthub", "hub")
|
||||
# alpha specifiers (e.g. 1.19.0a1)
|
||||
ensure_plugin_version_match("dlthub", "1.19.0a1", "1.19.0a2", "dlthub", "hub")
|
||||
ensure_plugin_version_match("dlthub", "1.19.0a1", "1.19.0", "dlthub", "hub")
|
||||
# dev specifiers (e.g. 1.19.0.dev1)
|
||||
ensure_plugin_version_match("dlthub", "1.19.0.dev1", "1.19.0.dev2", "dlthub", "hub")
|
||||
ensure_plugin_version_match("dlthub", "1.19.0.dev1", "1.19.0", "dlthub", "hub")
|
||||
ensure_plugin_version_match(
|
||||
"fake-plugin", "2.5.0", "2.5.10", "fake-plugin", "hub", specifier_2_5
|
||||
)
|
||||
# alpha specifiers (e.g. 1.19.0a1) - these are LESS than 1.19.0
|
||||
ensure_plugin_version_match(
|
||||
"fake-plugin", "1.19.0a1", "1.19.0a2", "fake-plugin", "hub", specifier_1_19
|
||||
)
|
||||
ensure_plugin_version_match(
|
||||
"fake-plugin", "1.19.0a1", "1.19.0", "fake-plugin", "hub", specifier_1_19
|
||||
)
|
||||
# dev specifiers (e.g. 1.19.0.dev1) - these are LESS than 1.19.0a0
|
||||
ensure_plugin_version_match(
|
||||
"fake-plugin", "1.19.0.dev1", "1.19.0.dev2", "fake-plugin", "hub", specifier_1_19
|
||||
)
|
||||
ensure_plugin_version_match(
|
||||
"fake-plugin", "1.19.0.dev1", "1.19.0", "fake-plugin", "hub", specifier_1_19
|
||||
)
|
||||
# post release specifiers
|
||||
ensure_plugin_version_match("dlthub", "1.19.0.post1", "1.19.0.post2", "dlthub", "hub")
|
||||
ensure_plugin_version_match("dlthub", "1.19.0.post1", "1.19.0", "dlthub", "hub")
|
||||
ensure_plugin_version_match(
|
||||
"fake-plugin", "1.19.0.post1", "1.19.0.post2", "fake-plugin", "hub", specifier_1_19
|
||||
)
|
||||
ensure_plugin_version_match(
|
||||
"fake-plugin", "1.19.0.post1", "1.19.0", "fake-plugin", "hub", specifier_1_19
|
||||
)
|
||||
|
||||
|
||||
def test_ensure_plugin_version_match_alpha_plugin() -> None:
|
||||
"""test that alpha plugins (major=0) match any dlt major version with same minor."""
|
||||
# alpha plugin (0.x.y) should match dlt 1.x.y with same minor
|
||||
ensure_plugin_version_match("dlthub", "1.19.0", "0.19.0", "dlthub", "hub")
|
||||
ensure_plugin_version_match("dlthub", "1.19.5", "0.19.2", "dlthub", "hub")
|
||||
ensure_plugin_version_match("dlthub", "2.19.0", "0.19.0", "dlthub", "hub")
|
||||
"""test that alpha plugins (major=0) match specifier."""
|
||||
from packaging.specifiers import SpecifierSet
|
||||
|
||||
# specifier for 0.19.x versions (including all pre-releases)
|
||||
# PEP 440 ordering: .devN < .aN < .bN < .rcN < final < .postN
|
||||
specifier_0_19 = SpecifierSet(">=0.19.0.dev0,<0.20.0")
|
||||
|
||||
# alpha plugin (0.x.y) should match specifier
|
||||
ensure_plugin_version_match(
|
||||
"fake-plugin", "1.19.0", "0.19.0", "fake-plugin", "hub", specifier_0_19
|
||||
)
|
||||
ensure_plugin_version_match(
|
||||
"fake-plugin", "1.19.5", "0.19.2", "fake-plugin", "hub", specifier_0_19
|
||||
)
|
||||
ensure_plugin_version_match(
|
||||
"fake-plugin", "2.19.0", "0.19.0", "fake-plugin", "hub", specifier_0_19
|
||||
)
|
||||
# alpha plugin with alpha/dev specifiers
|
||||
ensure_plugin_version_match("dlthub", "1.19.0a1", "0.19.0a2", "dlthub", "hub")
|
||||
ensure_plugin_version_match("dlthub", "1.19.0.dev1", "0.19.0.dev2", "dlthub", "hub")
|
||||
ensure_plugin_version_match(
|
||||
"fake-plugin", "1.19.0a1", "0.19.0a2", "fake-plugin", "hub", specifier_0_19
|
||||
)
|
||||
ensure_plugin_version_match(
|
||||
"fake-plugin", "1.19.0.dev1", "0.19.0.dev2", "fake-plugin", "hub", specifier_0_19
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
@@ -217,6 +259,13 @@ def test_ensure_plugin_version_match_alpha_plugin() -> None:
|
||||
)
|
||||
def test_ensure_plugin_version_match_mismatch(dlt_version: str, plugin_version: str) -> None:
|
||||
"""test that mismatched versions raise MissingDependencyException."""
|
||||
from packaging.specifiers import SpecifierSet
|
||||
|
||||
# Use explicit specifier that requires 1.19.x versions
|
||||
specifier = SpecifierSet(">=1.19.0,<1.20.0")
|
||||
|
||||
with pytest.raises(MissingDependencyException) as exc_info:
|
||||
ensure_plugin_version_match("dlthub", dlt_version, plugin_version, "dlthub", "hub")
|
||||
assert "dlthub" in str(exc_info.value)
|
||||
ensure_plugin_version_match(
|
||||
"fake-plugin", dlt_version, plugin_version, "fake-plugin", "hub", specifier
|
||||
)
|
||||
assert "fake-plugin" in str(exc_info.value)
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
import os
|
||||
import pytest
|
||||
from importlib.metadata import PackageNotFoundError
|
||||
from packaging.requirements import Requirement
|
||||
|
||||
from dlt.version import get_installed_requirement_string
|
||||
from dlt.version import get_installed_requirement_string, get_dependency_requirement
|
||||
|
||||
|
||||
def test_installed_requirement_string() -> None:
|
||||
@@ -15,3 +16,24 @@ def test_installed_requirement_string() -> None:
|
||||
# this is not installed
|
||||
with pytest.raises(PackageNotFoundError):
|
||||
get_installed_requirement_string("requests-X")
|
||||
|
||||
|
||||
def test_get_dependency_requirement() -> None:
|
||||
# dlt depends on dlthub, so this should return a Requirement
|
||||
req = get_dependency_requirement("dlthub")
|
||||
assert req is not None
|
||||
assert isinstance(req, Requirement)
|
||||
assert req.name == "dlthub"
|
||||
# click has a version specifier
|
||||
assert str(req.specifier) != ""
|
||||
|
||||
# dlt depends on fsspec with a version constraint
|
||||
req = get_dependency_requirement("fsspec")
|
||||
assert req is not None
|
||||
assert req.name == "fsspec"
|
||||
# verify we can check version satisfaction
|
||||
assert "2022.4.0" in req.specifier
|
||||
|
||||
# non-existent dependency returns None
|
||||
req = get_dependency_requirement("non-existent-package-xyz")
|
||||
assert req is None
|
||||
|
||||
@@ -4463,14 +4463,14 @@ def test_pending_package_exception_warning() -> None:
|
||||
with pytest.raises(PipelineStepFailed) as pip_ex:
|
||||
pipeline.run()
|
||||
|
||||
# none of the jobs passed so we have pending package but not partial
|
||||
# one of the job failed and package is aborted. sometimes the other
|
||||
# job completed, sometimes is still pending so we disable pending test
|
||||
assert pip_ex.value.step == "load"
|
||||
print(str(pip_ex.value))
|
||||
assert "Pending packages" not in str(pip_ex.value)
|
||||
# assert "Pending packages" not in str(pip_ex.value)
|
||||
assert "partially loaded" in str(pip_ex.value)
|
||||
assert pip_ex.value.load_id is not None
|
||||
assert pip_ex.value.is_package_partially_loaded is True
|
||||
assert pip_ex.value.has_pending_data is False
|
||||
# assert pip_ex.value.has_pending_data is False
|
||||
|
||||
|
||||
def test_cleanup() -> None:
|
||||
|
||||
18
uv.lock
generated
18
uv.lock
generated
@@ -2058,7 +2058,7 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "dlt"
|
||||
version = "1.20.0a1"
|
||||
version = "1.20.0"
|
||||
source = { editable = "." }
|
||||
dependencies = [
|
||||
{ name = "click" },
|
||||
@@ -2356,13 +2356,13 @@ requires-dist = [
|
||||
{ name = "db-dtypes", marker = "extra == 'bigquery'", specifier = ">=1.2.0" },
|
||||
{ name = "db-dtypes", marker = "extra == 'gcp'", specifier = ">=1.2.0" },
|
||||
{ name = "deltalake", marker = "extra == 'deltalake'", specifier = ">=0.25.1" },
|
||||
{ name = "dlt-runtime", marker = "python_full_version >= '3.10' and extra == 'hub'", specifier = ">=0.19.0a0,<0.21" },
|
||||
{ name = "dlt-runtime", marker = "python_full_version >= '3.10' and extra == 'hub'", specifier = ">=0.20.0a0,<0.21" },
|
||||
{ name = "dlthub", marker = "python_full_version >= '3.10' and extra == 'hub'", specifier = ">=0.20.0a1,<0.21" },
|
||||
{ name = "duckdb", marker = "extra == 'duckdb'", specifier = ">=0.9" },
|
||||
{ name = "duckdb", marker = "extra == 'ducklake'", specifier = ">=1.2.0" },
|
||||
{ name = "duckdb", marker = "extra == 'motherduck'", specifier = ">=0.9" },
|
||||
{ name = "duckdb", marker = "extra == 'workspace'", specifier = ">=0.9" },
|
||||
{ name = "fsspec", specifier = ">=2025.9.0" },
|
||||
{ name = "fsspec", specifier = ">=2022.4.0" },
|
||||
{ name = "gcsfs", marker = "extra == 'bigquery'", specifier = ">=2022.4.0" },
|
||||
{ name = "gcsfs", marker = "extra == 'clickhouse'", specifier = ">=2022.4.0" },
|
||||
{ name = "gcsfs", marker = "extra == 'gcp'", specifier = ">=2022.4.0" },
|
||||
@@ -2535,7 +2535,7 @@ streamlit = [{ name = "streamlit", marker = "python_full_version >= '3.9' and py
|
||||
|
||||
[[package]]
|
||||
name = "dlt-runtime"
|
||||
version = "0.20.0a0"
|
||||
version = "0.20.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "attrs", version = "25.4.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" },
|
||||
@@ -2545,22 +2545,22 @@ dependencies = [
|
||||
{ name = "python-jose", marker = "python_full_version >= '3.10'" },
|
||||
{ name = "tabulate", marker = "python_full_version >= '3.10'" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/1d/f5/d5c74ba2560507493b9d5d0c98a8482e15f036f338bb2832730065679e22/dlt_runtime-0.20.0a0.tar.gz", hash = "sha256:3e9d5df91f03152c251e5f874e5a13ac1bd66d5fbac0357c11996bf4e8279c8c", size = 48300, upload-time = "2025-12-04T15:51:51.192Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/d0/86/d7f057d8bdf2f3ada28bf1277b7f24a7abbb221d72788bd682176126a75c/dlt_runtime-0.20.0.tar.gz", hash = "sha256:753c7522bc01c92a453459640e482f87b647b14cc5734d754133a91968acc79f", size = 49532, upload-time = "2025-12-09T14:32:33.708Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/21/1f/ebcfea0c69a697d64f836140bfdb0f73d0be2880547dc00ff267eaad7569/dlt_runtime-0.20.0a0-py3-none-any.whl", hash = "sha256:d6498d4078980c833ea9c5cbfc8b7146488beddb77146ae6d3a2e7d7345dfb5a", size = 118233, upload-time = "2025-12-04T15:51:49.67Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/4c/b0/02a6c846d89e3c27a592a929d95526036be1d607e48fca214dcbf3b7bf58/dlt_runtime-0.20.0-py3-none-any.whl", hash = "sha256:0969165672b2b3938a618ddd263e0cf8ec356d289253f58134e325e222753056", size = 119573, upload-time = "2025-12-09T14:32:32.119Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "dlthub"
|
||||
version = "0.20.0a1"
|
||||
version = "0.20.1"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "python-jose", marker = "python_full_version >= '3.10'" },
|
||||
{ name = "ruamel-yaml", marker = "python_full_version >= '3.10'" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/13/20/eb0246c88dd80102a4b02ad807b3459c31855ac0c983f69a2b8fe027a88c/dlthub-0.20.0a1.tar.gz", hash = "sha256:bf8e90bd22bad85ed67d497f97d763678e74a41c70bb3c17e3ae2da7897a0bde", size = 158441, upload-time = "2025-12-07T11:26:45.312Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/77/1b/2c079f22243462e914026172094411ed7ef1fc96c8089e0ca66d1a14038a/dlthub-0.20.1.tar.gz", hash = "sha256:7b3a188abc28601fd4bdf8f17e7925ef729d4f91fb67a6b4eb5c5dc5a04ac3a2", size = 158432, upload-time = "2025-12-09T15:18:10.813Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/40/7b/307126b81ecefbf2dd7569e2da81b2a7b9124dbdcfa6039f3e350bd097c3/dlthub-0.20.0a1-py3-none-any.whl", hash = "sha256:de1a20567da74943931e8fd3551babd345c9f7dcb1f3782d200b308a79c660ce", size = 209794, upload-time = "2025-12-07T11:26:43.808Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/21/94/b2a87853102c6aa08606b2708d8f678b1e39855e8227fe111e37c32631b6/dlthub-0.20.1-py3-none-any.whl", hash = "sha256:c4d4e0c4515cd68f316ccd02c9ecc007332c861ae6f92a488f7e961935e7f1a0", size = 209767, upload-time = "2025-12-09T15:18:09.067Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
||||
Reference in New Issue
Block a user