From 99207237fef625b12656bf15338d01b12a671cfd Mon Sep 17 00:00:00 2001 From: ivasio Date: Tue, 9 Dec 2025 17:30:53 +0100 Subject: [PATCH] docs: add runtime docs to CLI reference (#3445) * bumps to version 1.20.0 * update the hub reference docs, add CI check * use dependency specifier in hub for plugin version check * minimum dlt runtime cli check * rollaback to old fsspec min version * fixes test_hub ci workflow * fixes flaky test * bumps hub extra * updates cli docs linting * fixes docs lock --------- Co-authored-by: Marcin Rudolf Co-authored-by: ivasio --- .github/workflows/test_hub.yml | 11 + deploy/dlt/Dockerfile.minimal | 13 +- dlt/common/runtime/run_context.py | 40 +- dlt/common/storages/fsspec_filesystem.py | 10 +- dlt/version.py | 18 + docs/tools/dlthub_cli/Makefile | 17 +- docs/uv.lock | 18 +- .../docs/hub/command-line-interface.md | 1086 ++++++++++++++++- pyproject.toml | 6 +- tests/common/runtime/test_run_context.py | 89 +- tests/common/test_version.py | 24 +- tests/pipeline/test_pipeline.py | 8 +- uv.lock | 18 +- 13 files changed, 1280 insertions(+), 78 deletions(-) diff --git a/.github/workflows/test_hub.yml b/.github/workflows/test_hub.yml index eb3b4d803..d7b54677d 100644 --- a/.github/workflows/test_hub.yml +++ b/.github/workflows/test_hub.yml @@ -94,6 +94,17 @@ jobs: run: pytest tests/hub # if: matrix.os != 'macos-latest' + - name: Test runtime client + run: | + mkdir .dlt && touch .dlt/.workspace + dlt runtime --help + + # DISABLED: because docs rendering happens in non-deterministic order (of plugin discovery) + # must be fixed + # - name: Check that dlthub cli docs are up to date + # run: cd docs/tools/dlthub_cli && make check-cli-docs + # if: ${{ matrix.python-version == '3.11' && matrix.os == 'ubuntu-latest' }} + matrix_job_required_check: name: hub | dlthub features tests needs: run_hub_features diff --git a/deploy/dlt/Dockerfile.minimal b/deploy/dlt/Dockerfile.minimal index 85080443a..2f0e4c443 100644 --- a/deploy/dlt/Dockerfile.minimal +++ b/deploy/dlt/Dockerfile.minimal @@ -34,7 +34,7 @@ COPY dist/dlt-${IMAGE_VERSION}.tar.gz . RUN mkdir -p /app WORKDIR /app RUN uv venv && uv pip install /tmp/pydlt/dlt-${IMAGE_VERSION}.tar.gz --resolution lowest-direct && uv pip install typing-extensions==4.8.0 -RUN rm -r /tmp/pydlt +# RUN rm -r /tmp/pydlt # make sure dlt can be actually imported RUN uv run python -c 'import dlt;import pendulum;' @@ -50,7 +50,12 @@ RUN uv run dlt pipeline fruit_pipeline info # enable workspace RUN mkdir -p .dlt && touch .dlt/.workspace -# RUN dlt pipeline fruit_pipeline info -RUN uv run dlt workspace info +RUN uv run dlt workspace -v info RUN uv run python minimal_pipeline.py -RUN uv run dlt pipeline fruit_pipeline info \ No newline at end of file +RUN uv run dlt pipeline fruit_pipeline info + +# install hub +RUN uv pip install /tmp/pydlt/dlt-${IMAGE_VERSION}.tar.gz[hub] --resolution lowest-direct && uv pip install typing-extensions==4.8.0 +RUN uv run python minimal_pipeline.py +RUN uv run dlt --non-interactive license issue dlthub.transformation +RUN uv run dlt runtime --help \ No newline at end of file diff --git a/dlt/common/runtime/run_context.py b/dlt/common/runtime/run_context.py index 34a1511b4..1a426e631 100644 --- a/dlt/common/runtime/run_context.py +++ b/dlt/common/runtime/run_context.py @@ -6,7 +6,7 @@ from types import ModuleType from typing import Any, Dict, Iterator, List, Optional from urllib.parse import urlencode -from packaging.version import Version +from packaging.specifiers import SpecifierSet from dlt.common import known_env from dlt.common.configuration.container import Container @@ -246,9 +246,14 @@ def ensure_plugin_version_match( plugin_version: str, plugin_module_name: str, dlt_extra: str, + dlt_version_specifier: Optional[SpecifierSet] = None, ) -> None: - """Ensures that installed dlt version matches plugin version. Plugins are tightly bound to `dlt` - and released together. Both major and minor version must match. For alpha plugins version may be 0. + """Ensures that installed plugin version matches dlt requirements. Plugins are tightly bound + to `dlt` and released together. + + If `dlt_version_specifier` is provided, it is used to check if the plugin version satisfies + the specifier. Otherwise, the specifier is read from dlt's package metadata (Requires-Dist). + If specifier cannot be determined, the function returns without checking. Args: pkg_name: Name of the plugin package (e.g., "dlthub") @@ -256,30 +261,37 @@ def ensure_plugin_version_match( plugin_version: The installed plugin version string plugin_module_name: The module name for MissingDependencyException (e.g., "dlthub") dlt_extra: The dlt extra to install the plugin (e.g., "hub") + dlt_version_specifier: Optional version specifier for the plugin. If not provided, + reads from dlt's package metadata. Raises: MissingDependencyException: If version mismatch is detected """ - installed = Version(plugin_version) - dlt_installed = Version(dlt_version) + # Get specifier from dlt's package metadata if not provided + if dlt_version_specifier is None: + from dlt.version import get_dependency_requirement - # currently packages must match on minor version - if installed.minor != dlt_installed.minor or ( - installed.major != dlt_installed.major and installed.major != 0 - ): + req = get_dependency_requirement(pkg_name) + if req is not None: + dlt_version_specifier = req.specifier + + # If specifier still not available, exit without checking + if dlt_version_specifier is None or len(dlt_version_specifier) == 0: + return + + # Use specifier.contains() for proper version check (allowing prereleases) + if not dlt_version_specifier.contains(plugin_version, prereleases=True): from dlt.common.exceptions import MissingDependencyException custom_msg = ( - f"`{pkg_name}` is a `dlt` plugin and must be installed together with `dlt` with a " - f"matching version. `dlt` {dlt_installed.major}.{dlt_installed.minor}.x requires " - f"`{pkg_name}` 0.{dlt_installed.minor}.x but you have " - f"{plugin_version}. Please install the right version of {pkg_name} with:\n\n" + f"`{pkg_name}` is a `dlt` plugin and must satisfy version requirement " + f"`{dlt_version_specifier}` but you have {plugin_version}. " + f"Please install the right version of {pkg_name} with:\n\n" f'pip install "dlt[{dlt_extra}]=={dlt_version}"\n\n' "or if you are upgrading the plugin:\n\n" f'pip install "dlt[{dlt_extra}]=={dlt_version}" -U {pkg_name}' ) missing_dep_ex = MissingDependencyException(plugin_module_name, []) - # ImportError uses `msg` attribute for __str__, not just args missing_dep_ex.args = (custom_msg,) missing_dep_ex.msg = custom_msg raise missing_dep_ex diff --git a/dlt/common/storages/fsspec_filesystem.py b/dlt/common/storages/fsspec_filesystem.py index 74c3b6b42..ef6f87cbd 100644 --- a/dlt/common/storages/fsspec_filesystem.py +++ b/dlt/common/storages/fsspec_filesystem.py @@ -65,10 +65,16 @@ MTIME_DISPATCH = { "az": lambda f: ensure_pendulum_datetime_utc(f["last_modified"]), "gcs": lambda f: ensure_pendulum_datetime_utc(f["updated"]), "https": lambda f: cast( - pendulum.DateTime, pendulum.parse(f["Last-Modified"], exact=True, strict=False) + pendulum.DateTime, + pendulum.parse( + f.get("Last-Modified", pendulum.now().isoformat()), exact=True, strict=False + ), ), "http": lambda f: cast( - pendulum.DateTime, pendulum.parse(f["Last-Modified"], exact=True, strict=False) + pendulum.DateTime, + pendulum.parse( + f.get("Last-Modified", pendulum.now().isoformat()), exact=True, strict=False + ), ), "file": lambda f: ensure_pendulum_datetime_utc(f["mtime"]), "memory": lambda f: ensure_pendulum_datetime_utc(f["created"]), diff --git a/dlt/version.py b/dlt/version.py index 956902eee..1406d5d5e 100644 --- a/dlt/version.py +++ b/dlt/version.py @@ -1,6 +1,8 @@ from importlib.metadata import version as pkg_version, distribution as pkg_distribution +from typing import Optional from urllib.request import url2pathname from urllib.parse import urlparse +from packaging.requirements import Requirement DLT_IMPORT_NAME = "dlt" PKG_NAME = DLT_PKG_NAME = "dlt" @@ -30,3 +32,19 @@ def get_installed_requirement_string( else: package_requirement = f"{package}{ver_selector}{pkg_version(package)}" return package_requirement + + +def get_dependency_requirement( + dependency_name: str, package: str = DLT_PKG_NAME +) -> Optional[Requirement]: + """Find a specific dependency requirement from package metadata""" + dist = pkg_distribution(package) + + if dist.requires is None: + return None + + for req_str in dist.requires: + req = Requirement(req_str) + if req.name == dependency_name: + return req + return None diff --git a/docs/tools/dlthub_cli/Makefile b/docs/tools/dlthub_cli/Makefile index e1b973ced..64545f8e2 100644 --- a/docs/tools/dlthub_cli/Makefile +++ b/docs/tools/dlthub_cli/Makefile @@ -1,11 +1,14 @@ -.PHONY: install-dlthub, update-cli-docs, check-cli-docs +.PHONY: update-cli-docs, check-cli-docs, dev # this must be run from `dlthub_cli` to see workspace commands +# it will use dlthub and dlt-runtime versions from dlt/docs/pyproject.toml to generate docs -install-dlthub: - uv pip install dlthub +dev: + uv sync -update-cli-docs: install-dlthub - uv run dlt --debug render-docs ../../website/docs/hub/command-line-interface.md --commands license workspace profile +update-cli-docs: dev + # generate as there's no license + RUNTIME__LICENSE="" uv run dlt --debug render-docs ../../website/docs/hub/command-line-interface.md --commands license workspace profile runtime -check-cli-docs: install-dlthub - uv run dlt --debug render-docs ../../website/docs/hub/command-line-interface.md --compare --commands license workspace profile \ No newline at end of file +check-cli-docs: dev + # generate as there's no license + RUNTIME__LICENSE="" uv run dlt --debug render-docs ../../website/docs/hub/command-line-interface.md --compare --commands license workspace profile runtime diff --git a/docs/uv.lock b/docs/uv.lock index 9e7cad15a..f5bde2fae 100644 --- a/docs/uv.lock +++ b/docs/uv.lock @@ -834,7 +834,7 @@ wheels = [ [[package]] name = "dlt" -version = "1.20.0a1" +version = "1.20.0" source = { editable = "../" } dependencies = [ { name = "click" }, @@ -928,13 +928,13 @@ requires-dist = [ { name = "db-dtypes", marker = "extra == 'bigquery'", specifier = ">=1.2.0" }, { name = "db-dtypes", marker = "extra == 'gcp'", specifier = ">=1.2.0" }, { name = "deltalake", marker = "extra == 'deltalake'", specifier = ">=0.25.1" }, - { name = "dlt-runtime", marker = "python_full_version >= '3.10' and extra == 'hub'", specifier = ">=0.19.0a0,<0.21" }, + { name = "dlt-runtime", marker = "python_full_version >= '3.10' and extra == 'hub'", specifier = ">=0.20.0a0,<0.21" }, { name = "dlthub", marker = "python_full_version >= '3.10' and extra == 'hub'", specifier = ">=0.20.0a1,<0.21" }, { name = "duckdb", marker = "extra == 'duckdb'", specifier = ">=0.9" }, { name = "duckdb", marker = "extra == 'ducklake'", specifier = ">=1.2.0" }, { name = "duckdb", marker = "extra == 'motherduck'", specifier = ">=0.9" }, { name = "duckdb", marker = "extra == 'workspace'", specifier = ">=0.9" }, - { name = "fsspec", specifier = ">=2025.9.0" }, + { name = "fsspec", specifier = ">=2022.4.0" }, { name = "gcsfs", marker = "extra == 'bigquery'", specifier = ">=2022.4.0" }, { name = "gcsfs", marker = "extra == 'clickhouse'", specifier = ">=2022.4.0" }, { name = "gcsfs", marker = "extra == 'gcp'", specifier = ">=2022.4.0" }, @@ -1189,7 +1189,7 @@ requires-dist = [ [[package]] name = "dlt-runtime" -version = "0.20.0a0" +version = "0.20.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "attrs" }, @@ -1199,22 +1199,22 @@ dependencies = [ { name = "python-jose" }, { name = "tabulate" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/1d/f5/d5c74ba2560507493b9d5d0c98a8482e15f036f338bb2832730065679e22/dlt_runtime-0.20.0a0.tar.gz", hash = "sha256:3e9d5df91f03152c251e5f874e5a13ac1bd66d5fbac0357c11996bf4e8279c8c", size = 48300, upload-time = "2025-12-04T15:51:51.192Z" } +sdist = { url = "https://files.pythonhosted.org/packages/d0/86/d7f057d8bdf2f3ada28bf1277b7f24a7abbb221d72788bd682176126a75c/dlt_runtime-0.20.0.tar.gz", hash = "sha256:753c7522bc01c92a453459640e482f87b647b14cc5734d754133a91968acc79f", size = 49532, upload-time = "2025-12-09T14:32:33.708Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/21/1f/ebcfea0c69a697d64f836140bfdb0f73d0be2880547dc00ff267eaad7569/dlt_runtime-0.20.0a0-py3-none-any.whl", hash = "sha256:d6498d4078980c833ea9c5cbfc8b7146488beddb77146ae6d3a2e7d7345dfb5a", size = 118233, upload-time = "2025-12-04T15:51:49.67Z" }, + { url = "https://files.pythonhosted.org/packages/4c/b0/02a6c846d89e3c27a592a929d95526036be1d607e48fca214dcbf3b7bf58/dlt_runtime-0.20.0-py3-none-any.whl", hash = "sha256:0969165672b2b3938a618ddd263e0cf8ec356d289253f58134e325e222753056", size = 119573, upload-time = "2025-12-09T14:32:32.119Z" }, ] [[package]] name = "dlthub" -version = "0.20.0a1" +version = "0.20.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "python-jose" }, { name = "ruamel-yaml" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/13/20/eb0246c88dd80102a4b02ad807b3459c31855ac0c983f69a2b8fe027a88c/dlthub-0.20.0a1.tar.gz", hash = "sha256:bf8e90bd22bad85ed67d497f97d763678e74a41c70bb3c17e3ae2da7897a0bde", size = 158441, upload-time = "2025-12-07T11:26:45.312Z" } +sdist = { url = "https://files.pythonhosted.org/packages/77/1b/2c079f22243462e914026172094411ed7ef1fc96c8089e0ca66d1a14038a/dlthub-0.20.1.tar.gz", hash = "sha256:7b3a188abc28601fd4bdf8f17e7925ef729d4f91fb67a6b4eb5c5dc5a04ac3a2", size = 158432, upload-time = "2025-12-09T15:18:10.813Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/40/7b/307126b81ecefbf2dd7569e2da81b2a7b9124dbdcfa6039f3e350bd097c3/dlthub-0.20.0a1-py3-none-any.whl", hash = "sha256:de1a20567da74943931e8fd3551babd345c9f7dcb1f3782d200b308a79c660ce", size = 209794, upload-time = "2025-12-07T11:26:43.808Z" }, + { url = "https://files.pythonhosted.org/packages/21/94/b2a87853102c6aa08606b2708d8f678b1e39855e8227fe111e37c32631b6/dlthub-0.20.1-py3-none-any.whl", hash = "sha256:c4d4e0c4515cd68f316ccd02c9ecc007332c861ae6f92a488f7e961935e7f1a0", size = 209767, upload-time = "2025-12-09T15:18:09.067Z" }, ] [[package]] diff --git a/docs/website/docs/hub/command-line-interface.md b/docs/website/docs/hub/command-line-interface.md index 26ebe0ff4..fcaddc920 100644 --- a/docs/website/docs/hub/command-line-interface.md +++ b/docs/website/docs/hub/command-line-interface.md @@ -5,11 +5,11 @@ keywords: [command line interface, cli, dlt init] --- -# Command line interface reference +# Command Line Interface Reference -This page contains all commands available in the dltHub CLI and is generated +This page contains all commands available in the dlt CLI and is generated automatically from the fully populated python argparse object of dlt. :::note Flags and positional commands are inherited from the parent command. Position within the command string @@ -31,7 +31,7 @@ Creates, adds, inspects and deploys dlt pipelines. Further help is available at ```sh dlt [-h] [--version] [--disable-telemetry] [--enable-telemetry] [--non-interactive] [--debug] [--no-pwd] - {pipeline,workspace,telemetry,schema,profile,init,render-docs,deploy,dashboard,ai,transformation,source,project,license,destination,dbt,dataset,cache} + {workspace,telemetry,schema,profile,pipeline,init,render-docs,deploy,dashboard,ai,license,runtime} ... ``` @@ -52,6 +52,7 @@ dlt [-h] [--version] [--disable-telemetry] [--enable-telemetry] * [`workspace`](#dlt-workspace) - Manage current workspace * [`profile`](#dlt-profile) - Manage workspace built-in profiles * [`license`](#dlt-license) - View dlthub license status +* [`runtime`](#dlt-runtime) - Connect to dlthub runtime and run your code remotely @@ -61,7 +62,7 @@ Manage current Workspace. **Usage** ```sh -dlt workspace [-h] {clean,info,mcp,show} ... +dlt workspace [-h] [--verbose] {clean,info,mcp,show} ... ``` **Description** @@ -77,6 +78,7 @@ Inherits arguments from [`dlt`](#dlt). **Options** * `-h, --help` - Show this help message and exit +* `--verbose, -v` - Provides more information for certain commands. **Available subcommands** * [`clean`](#dlt-workspace-clean) - Cleans local data for the selected profile. locally loaded data will be deleted. pipelines working directories are also deleted by default. data in remote destinations is not affected. @@ -389,7 +391,1081 @@ Issue a new self-signed trial license. Inherits arguments from [`dlt license`](#dlt-license). **Positional arguments** -* `scope` - Scope of the license, a comma separated list of the scopes: ['dlthub.dbt_generator', 'dlthub.sources.mssql', 'dlthub.project', 'dlthub.transformation', 'dlthub.destinations.iceberg', 'dlthub.destinations.snowflake_plus', 'dlthub.runner'] +* `scope` - Scope of the license, a comma separated list of the scopes: ['dlthub.dbt_generator', 'dlthub.sources.mssql', 'dlthub.project', 'dlthub.transformation', 'dlthub.data_quality', 'dlthub.destinations.iceberg', 'dlthub.destinations.snowflake_plus', 'dlthub.runner'] + +**Options** +* `-h, --help` - Show this help message and exit + + + +## `dlt runtime` + +Connect to dltHub Runtime and run your code remotely. + +**Usage** +```sh +dlt runtime [-h] + {login,logout,launch,serve,publish,schedule,logs,cancel,dashboard,deploy,info,deployment,job,jobs,job-run,job-runs,configuration} + ... +``` + +**Description** + +Allows to connect to the dltHub Runtime, deploy and run local workspaces there. Requires dltHub license. + +
+ +Show Arguments and Options + +Inherits arguments from [`dlt`](#dlt). + +**Options** +* `-h, --help` - Show this help message and exit + +**Available subcommands** +* [`login`](#dlt-runtime-login) - Login to dlthub runtime using github oauth and connect current workspace to the remote one +* [`logout`](#dlt-runtime-logout) - Logout from dlthub runtime +* [`launch`](#dlt-runtime-launch) - Deploy code/config and run a script (follow status and logs by default) +* [`serve`](#dlt-runtime-serve) - Deploy and serve an interactive notebook/app (read-only) and follow until ready +* [`publish`](#dlt-runtime-publish) - Generate or revoke a public link for an interactive notebook/app +* [`schedule`](#dlt-runtime-schedule) - Deploy and schedule a script with a cron timetable, or cancel the scheduled script from future runs +* [`logs`](#dlt-runtime-logs) - Show logs for latest or selected job run +* [`cancel`](#dlt-runtime-cancel) - Cancel latest or selected job run +* [`dashboard`](#dlt-runtime-dashboard) - Open the runtime dashboard for this workspace +* [`deploy`](#dlt-runtime-deploy) - Sync code and configuration to runtime without running anything +* [`info`](#dlt-runtime-info) - Show overview of current runtime workspace +* [`deployment`](#dlt-runtime-deployment) - Manipulate deployments in workspace +* [`job`](#dlt-runtime-job) - List, create and inspect jobs +* [`jobs`](#dlt-runtime-jobs) - List, create and inspect jobs +* [`job-run`](#dlt-runtime-job-run) - List, create and inspect job runs +* [`job-runs`](#dlt-runtime-job-runs) - List, create and inspect job runs +* [`configuration`](#dlt-runtime-configuration) - Manipulate configurations in workspace + +
+ +### `dlt runtime login` + +Login to dltHub Runtime using Github OAuth and connect current workspace to the remote one. + +**Usage** +```sh +dlt runtime login [-h] +``` + +**Description** + +Login to dltHub Runtime using Github OAuth. + +
+ +Show Arguments and Options + +Inherits arguments from [`dlt runtime`](#dlt-runtime). + +**Options** +* `-h, --help` - Show this help message and exit + +
+ +### `dlt runtime logout` + +Logout from dltHub Runtime. + +**Usage** +```sh +dlt runtime logout [-h] +``` + +**Description** + +Logout from dltHub Runtime. + +
+ +Show Arguments and Options + +Inherits arguments from [`dlt runtime`](#dlt-runtime). + +**Options** +* `-h, --help` - Show this help message and exit + +
+ +### `dlt runtime launch` + +Deploy code/config and run a script (follow status and logs by default). + +**Usage** +```sh +dlt runtime launch [-h] [-d] script_path +``` + +**Description** + +Deploy current workspace and run a batch script remotely. + +
+ +Show Arguments and Options + +Inherits arguments from [`dlt runtime`](#dlt-runtime). + +**Positional arguments** +* `script_path` - Local path to the script + +**Options** +* `-h, --help` - Show this help message and exit +* `-d, --detach` - Do not follow status changes and logs after starting + +
+ +### `dlt runtime serve` + +Deploy and serve an interactive notebook/app (read-only) and follow until ready. + +**Usage** +```sh +dlt runtime serve [-h] script_path +``` + +**Description** + +Deploy current workspace and run a notebook as a read-only web app. + +
+ +Show Arguments and Options + +Inherits arguments from [`dlt runtime`](#dlt-runtime). + +**Positional arguments** +* `script_path` - Local path to the notebook/app + +**Options** +* `-h, --help` - Show this help message and exit + +
+ +### `dlt runtime publish` + +Generate or revoke a public link for an interactive notebook/app. + +**Usage** +```sh +dlt runtime publish [-h] [--cancel] script_path +``` + +**Description** + +Generate a public link for a notebook/app, or revoke it with --cancel. + +
+ +Show Arguments and Options + +Inherits arguments from [`dlt runtime`](#dlt-runtime). + +**Positional arguments** +* `script_path` - Local path to the notebook/app + +**Options** +* `-h, --help` - Show this help message and exit +* `--cancel` - Revoke the public link for the notebook/app + +
+ +### `dlt runtime schedule` + +Deploy and schedule a script with a cron timetable, or cancel the scheduled script from future runs. + +**Usage** +```sh +dlt runtime schedule [-h] [--current] script_path cron_expr_or_cancel +``` + +**Description** + +Schedule a batch script to run on a cron timetable, or cancel the scheduled script from future runs. + +
+ +Show Arguments and Options + +Inherits arguments from [`dlt runtime`](#dlt-runtime). + +**Positional arguments** +* `script_path` - Local path to the script +* `cron_expr_or_cancel` - Either a cron schedule string if you want to schedule the script, or the literal 'cancel' command if you want to cancel it + +**Options** +* `-h, --help` - Show this help message and exit +* `--current` - When cancelling the schedule, also cancel the currently running instance if any + +
+ +### `dlt runtime logs` + +Show logs for latest or selected job run. + +**Usage** +```sh +dlt runtime logs [-h] [-f] script_path_or_job_name [run_number] +``` + +**Description** + +Show logs for the latest run of a job or a specific run number. + +
+ +Show Arguments and Options + +Inherits arguments from [`dlt runtime`](#dlt-runtime). + +**Positional arguments** +* `script_path_or_job_name` - Local path or job name +* `run_number` - Run number (optional) + +**Options** +* `-h, --help` - Show this help message and exit +* `-f, --follow` - Follow the logs of the run in tailing mode + +
+ +### `dlt runtime cancel` + +Cancel latest or selected job run. + +**Usage** +```sh +dlt runtime cancel [-h] script_path_or_job_name [run_number] +``` + +**Description** + +Cancel the latest run of a job or a specific run number. + +
+ +Show Arguments and Options + +Inherits arguments from [`dlt runtime`](#dlt-runtime). + +**Positional arguments** +* `script_path_or_job_name` - Local path or job name +* `run_number` - Run number (optional) + +**Options** +* `-h, --help` - Show this help message and exit + +
+ +### `dlt runtime dashboard` + +Open the Runtime dashboard for this workspace. + +**Usage** +```sh +dlt runtime dashboard [-h] +``` + +**Description** + +Open link to the Runtime dashboard for current remote workspace. + +
+ +Show Arguments and Options + +Inherits arguments from [`dlt runtime`](#dlt-runtime). + +**Options** +* `-h, --help` - Show this help message and exit + +
+ +### `dlt runtime deploy` + +Sync code and configuration to Runtime without running anything. + +**Usage** +```sh +dlt runtime deploy [-h] +``` + +**Description** + +Upload deployment and configuration if changed. + +
+ +Show Arguments and Options + +Inherits arguments from [`dlt runtime`](#dlt-runtime). + +**Options** +* `-h, --help` - Show this help message and exit + +
+ +### `dlt runtime info` + +Show overview of current Runtime workspace. + +**Usage** +```sh +dlt runtime info [-h] +``` + +**Description** + +Show workspace id and summary of deployments, configurations and jobs. + +
+ +Show Arguments and Options + +Inherits arguments from [`dlt runtime`](#dlt-runtime). + +**Options** +* `-h, --help` - Show this help message and exit + +
+ +### `dlt runtime deployment` + +Manipulate deployments in workspace. + +**Usage** +```sh +dlt runtime deployment [-h] [deployment_version_no] {list,info,sync} ... +``` + +**Description** + +Manipulate deployments in workspace. + +
+ +Show Arguments and Options + +Inherits arguments from [`dlt runtime`](#dlt-runtime). + +**Positional arguments** +* `deployment_version_no` - Deployment version number. only used in the `info` subcommand + +**Options** +* `-h, --help` - Show this help message and exit + +**Available subcommands** +* [`list`](#dlt-runtime-deployment-list) - List all deployments in workspace +* [`info`](#dlt-runtime-deployment-info) - Get detailed information about a deployment +* [`sync`](#dlt-runtime-deployment-sync) - Create new deployment if local workspace content changed + +
+ +### `dlt runtime deployment list` + +List all deployments in workspace. + +**Usage** +```sh +dlt runtime deployment [deployment_version_no] list [-h] +``` + +**Description** + +List all deployments in workspace. + +
+ +Show Arguments and Options + +Inherits arguments from [`dlt runtime deployment`](#dlt-runtime-deployment). + +**Options** +* `-h, --help` - Show this help message and exit + +
+ +### `dlt runtime deployment info` + +Get detailed information about a deployment. + +**Usage** +```sh +dlt runtime deployment [deployment_version_no] info [-h] +``` + +**Description** + +Get detailed information about a deployment. + +
+ +Show Arguments and Options + +Inherits arguments from [`dlt runtime deployment`](#dlt-runtime-deployment). + +**Options** +* `-h, --help` - Show this help message and exit + +
+ +### `dlt runtime deployment sync` + +Create new deployment if local workspace content changed. + +**Usage** +```sh +dlt runtime deployment [deployment_version_no] sync [-h] +``` + +**Description** + +Create new deployment if local workspace content changed. + +
+ +Show Arguments and Options + +Inherits arguments from [`dlt runtime deployment`](#dlt-runtime-deployment). + +**Options** +* `-h, --help` - Show this help message and exit + +
+ +### `dlt runtime job` + +List, create and inspect jobs. + +**Usage** +```sh +dlt runtime job [-h] [script_path_or_job_name] {list,info,create} ... +``` + +**Description** + +List and manipulate jobs registered in the workspace. + +
+ +Show Arguments and Options + +Inherits arguments from [`dlt runtime`](#dlt-runtime). + +**Positional arguments** +* `script_path_or_job_name` - Local script path or job name. required for all commands except `list` + +**Options** +* `-h, --help` - Show this help message and exit + +**Available subcommands** +* [`list`](#dlt-runtime-job-list) - List the jobs registered in the workspace +* [`info`](#dlt-runtime-job-info) - Show job info +* [`create`](#dlt-runtime-job-create) - Create a job without running it + +
+ +### `dlt runtime job list` + +List the jobs registered in the workspace. + +**Usage** +```sh +dlt runtime job [script_path_or_job_name] list [-h] +``` + +**Description** + +List the jobs registered in the workspace. + +
+ +Show Arguments and Options + +Inherits arguments from [`dlt runtime job`](#dlt-runtime-job). + +**Options** +* `-h, --help` - Show this help message and exit + +
+ +### `dlt runtime job info` + +Show job info. + +**Usage** +```sh +dlt runtime job [script_path_or_job_name] info [-h] +``` + +**Description** + +Display detailed information about the job. + +
+ +Show Arguments and Options + +Inherits arguments from [`dlt runtime job`](#dlt-runtime-job). + +**Options** +* `-h, --help` - Show this help message and exit + +
+ +### `dlt runtime job create` + +Create a job without running it. + +**Usage** +```sh +dlt runtime job [script_path_or_job_name] create [-h] [--name [NAME]] + [--schedule [SCHEDULE]] [--interactive] [--description [DESCRIPTION]] +``` + +**Description** + +Manually create the job. + +
+ +Show Arguments and Options + +Inherits arguments from [`dlt runtime job`](#dlt-runtime-job). + +**Options** +* `-h, --help` - Show this help message and exit +* `--name [NAME]` - Job name to create +* `--schedule [SCHEDULE]` - Cron schedule for the job if it's a scheduled one +* `--interactive` - Run the job interactively, e.g. for a notebook +* `--description [DESCRIPTION]` - Job description + +
+ +### `dlt runtime jobs` + +List, create and inspect jobs. + +**Usage** +```sh +dlt runtime jobs [-h] [script_path_or_job_name] {list,info,create} ... +``` + +**Description** + +List and manipulate jobs registered in the workspace. + +
+ +Show Arguments and Options + +Inherits arguments from [`dlt runtime`](#dlt-runtime). + +**Positional arguments** +* `script_path_or_job_name` - Local script path or job name. required for all commands except `list` + +**Options** +* `-h, --help` - Show this help message and exit + +**Available subcommands** +* [`list`](#dlt-runtime-jobs-list) - List the jobs registered in the workspace +* [`info`](#dlt-runtime-jobs-info) - Show job info +* [`create`](#dlt-runtime-jobs-create) - Create a job without running it + +
+ +### `dlt runtime jobs list` + +List the jobs registered in the workspace. + +**Usage** +```sh +dlt runtime jobs [script_path_or_job_name] list [-h] +``` + +**Description** + +List the jobs registered in the workspace. + +
+ +Show Arguments and Options + +Inherits arguments from [`dlt runtime jobs`](#dlt-runtime-jobs). + +**Options** +* `-h, --help` - Show this help message and exit + +
+ +### `dlt runtime jobs info` + +Show job info. + +**Usage** +```sh +dlt runtime jobs [script_path_or_job_name] info [-h] +``` + +**Description** + +Display detailed information about the job. + +
+ +Show Arguments and Options + +Inherits arguments from [`dlt runtime jobs`](#dlt-runtime-jobs). + +**Options** +* `-h, --help` - Show this help message and exit + +
+ +### `dlt runtime jobs create` + +Create a job without running it. + +**Usage** +```sh +dlt runtime jobs [script_path_or_job_name] create [-h] [--name [NAME]] + [--schedule [SCHEDULE]] [--interactive] [--description [DESCRIPTION]] +``` + +**Description** + +Manually create the job. + +
+ +Show Arguments and Options + +Inherits arguments from [`dlt runtime jobs`](#dlt-runtime-jobs). + +**Options** +* `-h, --help` - Show this help message and exit +* `--name [NAME]` - Job name to create +* `--schedule [SCHEDULE]` - Cron schedule for the job if it's a scheduled one +* `--interactive` - Run the job interactively, e.g. for a notebook +* `--description [DESCRIPTION]` - Job description + +
+ +### `dlt runtime job-run` + +List, create and inspect job runs. + +**Usage** +```sh +dlt runtime job-run [-h] [script_path_or_job_name] [run_number] + {list,info,create,logs,cancel} ... +``` + +**Description** + +List and manipulate job runs registered in the workspace. + +
+ +Show Arguments and Options + +Inherits arguments from [`dlt runtime`](#dlt-runtime). + +**Positional arguments** +* `script_path_or_job_name` - Local script path or job name. required for all commands except `list` +* `run_number` - Run number. used in all commands except `list` and `create` as optional argument. if not specified, the latest run of given script be used. + +**Options** +* `-h, --help` - Show this help message and exit + +**Available subcommands** +* [`list`](#dlt-runtime-job-run-list) - List the job runs registered in the workspace +* [`info`](#dlt-runtime-job-run-info) - Show job run info +* [`create`](#dlt-runtime-job-run-create) - Create a job run without running it +* [`logs`](#dlt-runtime-job-run-logs) - Show logs for the latest or selected job run +* [`cancel`](#dlt-runtime-job-run-cancel) - Cancel the latest or selected job run + +
+ +### `dlt runtime job-run list` + +List the job runs registered in the workspace. + +**Usage** +```sh +dlt runtime job-run [script_path_or_job_name] [run_number] list [-h] +``` + +**Description** + +List the job runs registered in the workspace. + +
+ +Show Arguments and Options + +Inherits arguments from [`dlt runtime job-run`](#dlt-runtime-job-run). + +**Options** +* `-h, --help` - Show this help message and exit + +
+ +### `dlt runtime job-run info` + +Show job run info. + +**Usage** +```sh +dlt runtime job-run [script_path_or_job_name] [run_number] info [-h] +``` + +**Description** + +Display detailed information about the job run. + +
+ +Show Arguments and Options + +Inherits arguments from [`dlt runtime job-run`](#dlt-runtime-job-run). + +**Options** +* `-h, --help` - Show this help message and exit + +
+ +### `dlt runtime job-run create` + +Create a job run without running it. + +**Usage** +```sh +dlt runtime job-run [script_path_or_job_name] [run_number] create [-h] +``` + +**Description** + +Manually create the job run. + +
+ +Show Arguments and Options + +Inherits arguments from [`dlt runtime job-run`](#dlt-runtime-job-run). + +**Options** +* `-h, --help` - Show this help message and exit + +
+ +### `dlt runtime job-run logs` + +Show logs for the latest or selected job run. + +**Usage** +```sh +dlt runtime job-run [script_path_or_job_name] [run_number] logs [-h] [-f] +``` + +**Description** + +Show logs for the latest or selected job run. Use --follow to follow the logs in tailing mode. + +
+ +Show Arguments and Options + +Inherits arguments from [`dlt runtime job-run`](#dlt-runtime-job-run). + +**Options** +* `-h, --help` - Show this help message and exit +* `-f, --follow` - Follow the logs of the run in tailing mode + +
+ +### `dlt runtime job-run cancel` + +Cancel the latest or selected job run. + +**Usage** +```sh +dlt runtime job-run [script_path_or_job_name] [run_number] cancel [-h] +``` + +**Description** + +Cancel the latest or selected job run. + +
+ +Show Arguments and Options + +Inherits arguments from [`dlt runtime job-run`](#dlt-runtime-job-run). + +**Options** +* `-h, --help` - Show this help message and exit + +
+ +### `dlt runtime job-runs` + +List, create and inspect job runs. + +**Usage** +```sh +dlt runtime job-runs [-h] [script_path_or_job_name] [run_number] + {list,info,create,logs,cancel} ... +``` + +**Description** + +List and manipulate job runs registered in the workspace. + +
+ +Show Arguments and Options + +Inherits arguments from [`dlt runtime`](#dlt-runtime). + +**Positional arguments** +* `script_path_or_job_name` - Local script path or job name. required for all commands except `list` +* `run_number` - Run number. used in all commands except `list` and `create` as optional argument. if not specified, the latest run of given script be used. + +**Options** +* `-h, --help` - Show this help message and exit + +**Available subcommands** +* [`list`](#dlt-runtime-job-runs-list) - List the job runs registered in the workspace +* [`info`](#dlt-runtime-job-runs-info) - Show job run info +* [`create`](#dlt-runtime-job-runs-create) - Create a job run without running it +* [`logs`](#dlt-runtime-job-runs-logs) - Show logs for the latest or selected job run +* [`cancel`](#dlt-runtime-job-runs-cancel) - Cancel the latest or selected job run + +
+ +### `dlt runtime job-runs list` + +List the job runs registered in the workspace. + +**Usage** +```sh +dlt runtime job-runs [script_path_or_job_name] [run_number] list [-h] +``` + +**Description** + +List the job runs registered in the workspace. + +
+ +Show Arguments and Options + +Inherits arguments from [`dlt runtime job-runs`](#dlt-runtime-job-runs). + +**Options** +* `-h, --help` - Show this help message and exit + +
+ +### `dlt runtime job-runs info` + +Show job run info. + +**Usage** +```sh +dlt runtime job-runs [script_path_or_job_name] [run_number] info [-h] +``` + +**Description** + +Display detailed information about the job run. + +
+ +Show Arguments and Options + +Inherits arguments from [`dlt runtime job-runs`](#dlt-runtime-job-runs). + +**Options** +* `-h, --help` - Show this help message and exit + +
+ +### `dlt runtime job-runs create` + +Create a job run without running it. + +**Usage** +```sh +dlt runtime job-runs [script_path_or_job_name] [run_number] create [-h] +``` + +**Description** + +Manually create the job run. + +
+ +Show Arguments and Options + +Inherits arguments from [`dlt runtime job-runs`](#dlt-runtime-job-runs). + +**Options** +* `-h, --help` - Show this help message and exit + +
+ +### `dlt runtime job-runs logs` + +Show logs for the latest or selected job run. + +**Usage** +```sh +dlt runtime job-runs [script_path_or_job_name] [run_number] logs [-h] [-f] +``` + +**Description** + +Show logs for the latest or selected job run. Use --follow to follow the logs in tailing mode. + +
+ +Show Arguments and Options + +Inherits arguments from [`dlt runtime job-runs`](#dlt-runtime-job-runs). + +**Options** +* `-h, --help` - Show this help message and exit +* `-f, --follow` - Follow the logs of the run in tailing mode + +
+ +### `dlt runtime job-runs cancel` + +Cancel the latest or selected job run. + +**Usage** +```sh +dlt runtime job-runs [script_path_or_job_name] [run_number] cancel [-h] +``` + +**Description** + +Cancel the latest or selected job run. + +
+ +Show Arguments and Options + +Inherits arguments from [`dlt runtime job-runs`](#dlt-runtime-job-runs). + +**Options** +* `-h, --help` - Show this help message and exit + +
+ +### `dlt runtime configuration` + +Manipulate configurations in workspace. + +**Usage** +```sh +dlt runtime configuration [-h] [configuration_version_no] {list,info,sync} ... +``` + +**Description** + +Manipulate configurations in workspace. + +
+ +Show Arguments and Options + +Inherits arguments from [`dlt runtime`](#dlt-runtime). + +**Positional arguments** +* `configuration_version_no` - Configuration version number. only used in the `info` subcommand + +**Options** +* `-h, --help` - Show this help message and exit + +**Available subcommands** +* [`list`](#dlt-runtime-configuration-list) - List all configuration versions +* [`info`](#dlt-runtime-configuration-info) - Get detailed information about a configuration +* [`sync`](#dlt-runtime-configuration-sync) - Create new configuration if local config content changed + +
+ +### `dlt runtime configuration list` + +List all configuration versions. + +**Usage** +```sh +dlt runtime configuration [configuration_version_no] list [-h] +``` + +**Description** + +List all configuration versions. + +
+ +Show Arguments and Options + +Inherits arguments from [`dlt runtime configuration`](#dlt-runtime-configuration). + +**Options** +* `-h, --help` - Show this help message and exit + +
+ +### `dlt runtime configuration info` + +Get detailed information about a configuration. + +**Usage** +```sh +dlt runtime configuration [configuration_version_no] info [-h] +``` + +**Description** + +Get detailed information about a configuration. + +
+ +Show Arguments and Options + +Inherits arguments from [`dlt runtime configuration`](#dlt-runtime-configuration). + +**Options** +* `-h, --help` - Show this help message and exit + +
+ +### `dlt runtime configuration sync` + +Create new configuration if local config content changed. + +**Usage** +```sh +dlt runtime configuration [configuration_version_no] sync [-h] +``` + +**Description** + +Create new configuration if local config content changed. + +
+ +Show Arguments and Options + +Inherits arguments from [`dlt runtime configuration`](#dlt-runtime-configuration). **Options** * `-h, --help` - Show this help message and exit diff --git a/pyproject.toml b/pyproject.toml index fb90d3769..a3f27c37b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "dlt" -version = "1.20.0a1" +version = "1.20.0" description = "dlt is an open-source python-first scalable data loading library that does not require any backend to run." authors = [{ name = "dltHub Inc.", email = "services@dlthub.com" }] requires-python = ">=3.9.2, <3.15" @@ -52,7 +52,7 @@ dependencies = [ "orjson>=3.11.0 ; python_version > '3.13'", "tenacity>=8.0.2", "jsonpath-ng>=1.5.3", - "fsspec>=2025.9.0", + "fsspec>=2022.4.0", "packaging>=21.1", "pluggy>=1.3.0", "win-precise-time>=1.4.2 ; os_name == 'nt' and python_version < '3.13'", @@ -189,7 +189,7 @@ workspace = [ ] hub = [ "dlthub>=0.20.0a1,<0.21 ; python_version >= '3.10'", - "dlt-runtime>=0.19.0a0,<0.21 ; python_version >= '3.10'", + "dlt-runtime>=0.20.0a0,<0.21 ; python_version >= '3.10'", ] dbml = [ diff --git a/tests/common/runtime/test_run_context.py b/tests/common/runtime/test_run_context.py index 0081aec83..890917655 100644 --- a/tests/common/runtime/test_run_context.py +++ b/tests/common/runtime/test_run_context.py @@ -174,31 +174,73 @@ def test_context_with_xdg_dir(mocker) -> None: def test_ensure_plugin_version_match_same_versions() -> None: """test that matching versions pass without error.""" + from packaging.specifiers import SpecifierSet + + # Use explicit specifier to test specific version matching scenarios + # PEP 440 ordering: .devN < .aN < .bN < .rcN < final < .postN + # So we use .dev0 as lower bound to include all pre-releases + specifier_1_19 = SpecifierSet(">=1.19.0.dev0,<1.20.0") # includes all prereleases + specifier_2_5 = SpecifierSet(">=2.5.0.dev0,<2.6.0") + # exact same version - ensure_plugin_version_match("dlthub", "1.19.0", "1.19.0", "dlthub", "hub") - ensure_plugin_version_match("dlthub", "1.19.5", "1.19.2", "dlthub", "hub") + ensure_plugin_version_match( + "fake-plugin", "1.19.0", "1.19.0", "fake-plugin", "hub", specifier_1_19 + ) + ensure_plugin_version_match( + "fake-plugin", "1.19.5", "1.19.2", "fake-plugin", "hub", specifier_1_19 + ) # different patch versions are ok - ensure_plugin_version_match("dlthub", "2.5.0", "2.5.10", "dlthub", "hub") - # alpha specifiers (e.g. 1.19.0a1) - ensure_plugin_version_match("dlthub", "1.19.0a1", "1.19.0a2", "dlthub", "hub") - ensure_plugin_version_match("dlthub", "1.19.0a1", "1.19.0", "dlthub", "hub") - # dev specifiers (e.g. 1.19.0.dev1) - ensure_plugin_version_match("dlthub", "1.19.0.dev1", "1.19.0.dev2", "dlthub", "hub") - ensure_plugin_version_match("dlthub", "1.19.0.dev1", "1.19.0", "dlthub", "hub") + ensure_plugin_version_match( + "fake-plugin", "2.5.0", "2.5.10", "fake-plugin", "hub", specifier_2_5 + ) + # alpha specifiers (e.g. 1.19.0a1) - these are LESS than 1.19.0 + ensure_plugin_version_match( + "fake-plugin", "1.19.0a1", "1.19.0a2", "fake-plugin", "hub", specifier_1_19 + ) + ensure_plugin_version_match( + "fake-plugin", "1.19.0a1", "1.19.0", "fake-plugin", "hub", specifier_1_19 + ) + # dev specifiers (e.g. 1.19.0.dev1) - these are LESS than 1.19.0a0 + ensure_plugin_version_match( + "fake-plugin", "1.19.0.dev1", "1.19.0.dev2", "fake-plugin", "hub", specifier_1_19 + ) + ensure_plugin_version_match( + "fake-plugin", "1.19.0.dev1", "1.19.0", "fake-plugin", "hub", specifier_1_19 + ) # post release specifiers - ensure_plugin_version_match("dlthub", "1.19.0.post1", "1.19.0.post2", "dlthub", "hub") - ensure_plugin_version_match("dlthub", "1.19.0.post1", "1.19.0", "dlthub", "hub") + ensure_plugin_version_match( + "fake-plugin", "1.19.0.post1", "1.19.0.post2", "fake-plugin", "hub", specifier_1_19 + ) + ensure_plugin_version_match( + "fake-plugin", "1.19.0.post1", "1.19.0", "fake-plugin", "hub", specifier_1_19 + ) def test_ensure_plugin_version_match_alpha_plugin() -> None: - """test that alpha plugins (major=0) match any dlt major version with same minor.""" - # alpha plugin (0.x.y) should match dlt 1.x.y with same minor - ensure_plugin_version_match("dlthub", "1.19.0", "0.19.0", "dlthub", "hub") - ensure_plugin_version_match("dlthub", "1.19.5", "0.19.2", "dlthub", "hub") - ensure_plugin_version_match("dlthub", "2.19.0", "0.19.0", "dlthub", "hub") + """test that alpha plugins (major=0) match specifier.""" + from packaging.specifiers import SpecifierSet + + # specifier for 0.19.x versions (including all pre-releases) + # PEP 440 ordering: .devN < .aN < .bN < .rcN < final < .postN + specifier_0_19 = SpecifierSet(">=0.19.0.dev0,<0.20.0") + + # alpha plugin (0.x.y) should match specifier + ensure_plugin_version_match( + "fake-plugin", "1.19.0", "0.19.0", "fake-plugin", "hub", specifier_0_19 + ) + ensure_plugin_version_match( + "fake-plugin", "1.19.5", "0.19.2", "fake-plugin", "hub", specifier_0_19 + ) + ensure_plugin_version_match( + "fake-plugin", "2.19.0", "0.19.0", "fake-plugin", "hub", specifier_0_19 + ) # alpha plugin with alpha/dev specifiers - ensure_plugin_version_match("dlthub", "1.19.0a1", "0.19.0a2", "dlthub", "hub") - ensure_plugin_version_match("dlthub", "1.19.0.dev1", "0.19.0.dev2", "dlthub", "hub") + ensure_plugin_version_match( + "fake-plugin", "1.19.0a1", "0.19.0a2", "fake-plugin", "hub", specifier_0_19 + ) + ensure_plugin_version_match( + "fake-plugin", "1.19.0.dev1", "0.19.0.dev2", "fake-plugin", "hub", specifier_0_19 + ) @pytest.mark.parametrize( @@ -217,6 +259,13 @@ def test_ensure_plugin_version_match_alpha_plugin() -> None: ) def test_ensure_plugin_version_match_mismatch(dlt_version: str, plugin_version: str) -> None: """test that mismatched versions raise MissingDependencyException.""" + from packaging.specifiers import SpecifierSet + + # Use explicit specifier that requires 1.19.x versions + specifier = SpecifierSet(">=1.19.0,<1.20.0") + with pytest.raises(MissingDependencyException) as exc_info: - ensure_plugin_version_match("dlthub", dlt_version, plugin_version, "dlthub", "hub") - assert "dlthub" in str(exc_info.value) + ensure_plugin_version_match( + "fake-plugin", dlt_version, plugin_version, "fake-plugin", "hub", specifier + ) + assert "fake-plugin" in str(exc_info.value) diff --git a/tests/common/test_version.py b/tests/common/test_version.py index 765d690ed..92ca34175 100644 --- a/tests/common/test_version.py +++ b/tests/common/test_version.py @@ -1,8 +1,9 @@ import os import pytest from importlib.metadata import PackageNotFoundError +from packaging.requirements import Requirement -from dlt.version import get_installed_requirement_string +from dlt.version import get_installed_requirement_string, get_dependency_requirement def test_installed_requirement_string() -> None: @@ -15,3 +16,24 @@ def test_installed_requirement_string() -> None: # this is not installed with pytest.raises(PackageNotFoundError): get_installed_requirement_string("requests-X") + + +def test_get_dependency_requirement() -> None: + # dlt depends on dlthub, so this should return a Requirement + req = get_dependency_requirement("dlthub") + assert req is not None + assert isinstance(req, Requirement) + assert req.name == "dlthub" + # click has a version specifier + assert str(req.specifier) != "" + + # dlt depends on fsspec with a version constraint + req = get_dependency_requirement("fsspec") + assert req is not None + assert req.name == "fsspec" + # verify we can check version satisfaction + assert "2022.4.0" in req.specifier + + # non-existent dependency returns None + req = get_dependency_requirement("non-existent-package-xyz") + assert req is None diff --git a/tests/pipeline/test_pipeline.py b/tests/pipeline/test_pipeline.py index bd4c3166c..c4b3e5d03 100644 --- a/tests/pipeline/test_pipeline.py +++ b/tests/pipeline/test_pipeline.py @@ -4463,14 +4463,14 @@ def test_pending_package_exception_warning() -> None: with pytest.raises(PipelineStepFailed) as pip_ex: pipeline.run() - # none of the jobs passed so we have pending package but not partial + # one of the job failed and package is aborted. sometimes the other + # job completed, sometimes is still pending so we disable pending test assert pip_ex.value.step == "load" - print(str(pip_ex.value)) - assert "Pending packages" not in str(pip_ex.value) + # assert "Pending packages" not in str(pip_ex.value) assert "partially loaded" in str(pip_ex.value) assert pip_ex.value.load_id is not None assert pip_ex.value.is_package_partially_loaded is True - assert pip_ex.value.has_pending_data is False + # assert pip_ex.value.has_pending_data is False def test_cleanup() -> None: diff --git a/uv.lock b/uv.lock index ba5dd52f5..cf2b89715 100644 --- a/uv.lock +++ b/uv.lock @@ -2058,7 +2058,7 @@ wheels = [ [[package]] name = "dlt" -version = "1.20.0a1" +version = "1.20.0" source = { editable = "." } dependencies = [ { name = "click" }, @@ -2356,13 +2356,13 @@ requires-dist = [ { name = "db-dtypes", marker = "extra == 'bigquery'", specifier = ">=1.2.0" }, { name = "db-dtypes", marker = "extra == 'gcp'", specifier = ">=1.2.0" }, { name = "deltalake", marker = "extra == 'deltalake'", specifier = ">=0.25.1" }, - { name = "dlt-runtime", marker = "python_full_version >= '3.10' and extra == 'hub'", specifier = ">=0.19.0a0,<0.21" }, + { name = "dlt-runtime", marker = "python_full_version >= '3.10' and extra == 'hub'", specifier = ">=0.20.0a0,<0.21" }, { name = "dlthub", marker = "python_full_version >= '3.10' and extra == 'hub'", specifier = ">=0.20.0a1,<0.21" }, { name = "duckdb", marker = "extra == 'duckdb'", specifier = ">=0.9" }, { name = "duckdb", marker = "extra == 'ducklake'", specifier = ">=1.2.0" }, { name = "duckdb", marker = "extra == 'motherduck'", specifier = ">=0.9" }, { name = "duckdb", marker = "extra == 'workspace'", specifier = ">=0.9" }, - { name = "fsspec", specifier = ">=2025.9.0" }, + { name = "fsspec", specifier = ">=2022.4.0" }, { name = "gcsfs", marker = "extra == 'bigquery'", specifier = ">=2022.4.0" }, { name = "gcsfs", marker = "extra == 'clickhouse'", specifier = ">=2022.4.0" }, { name = "gcsfs", marker = "extra == 'gcp'", specifier = ">=2022.4.0" }, @@ -2535,7 +2535,7 @@ streamlit = [{ name = "streamlit", marker = "python_full_version >= '3.9' and py [[package]] name = "dlt-runtime" -version = "0.20.0a0" +version = "0.20.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "attrs", version = "25.4.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, @@ -2545,22 +2545,22 @@ dependencies = [ { name = "python-jose", marker = "python_full_version >= '3.10'" }, { name = "tabulate", marker = "python_full_version >= '3.10'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/1d/f5/d5c74ba2560507493b9d5d0c98a8482e15f036f338bb2832730065679e22/dlt_runtime-0.20.0a0.tar.gz", hash = "sha256:3e9d5df91f03152c251e5f874e5a13ac1bd66d5fbac0357c11996bf4e8279c8c", size = 48300, upload-time = "2025-12-04T15:51:51.192Z" } +sdist = { url = "https://files.pythonhosted.org/packages/d0/86/d7f057d8bdf2f3ada28bf1277b7f24a7abbb221d72788bd682176126a75c/dlt_runtime-0.20.0.tar.gz", hash = "sha256:753c7522bc01c92a453459640e482f87b647b14cc5734d754133a91968acc79f", size = 49532, upload-time = "2025-12-09T14:32:33.708Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/21/1f/ebcfea0c69a697d64f836140bfdb0f73d0be2880547dc00ff267eaad7569/dlt_runtime-0.20.0a0-py3-none-any.whl", hash = "sha256:d6498d4078980c833ea9c5cbfc8b7146488beddb77146ae6d3a2e7d7345dfb5a", size = 118233, upload-time = "2025-12-04T15:51:49.67Z" }, + { url = "https://files.pythonhosted.org/packages/4c/b0/02a6c846d89e3c27a592a929d95526036be1d607e48fca214dcbf3b7bf58/dlt_runtime-0.20.0-py3-none-any.whl", hash = "sha256:0969165672b2b3938a618ddd263e0cf8ec356d289253f58134e325e222753056", size = 119573, upload-time = "2025-12-09T14:32:32.119Z" }, ] [[package]] name = "dlthub" -version = "0.20.0a1" +version = "0.20.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "python-jose", marker = "python_full_version >= '3.10'" }, { name = "ruamel-yaml", marker = "python_full_version >= '3.10'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/13/20/eb0246c88dd80102a4b02ad807b3459c31855ac0c983f69a2b8fe027a88c/dlthub-0.20.0a1.tar.gz", hash = "sha256:bf8e90bd22bad85ed67d497f97d763678e74a41c70bb3c17e3ae2da7897a0bde", size = 158441, upload-time = "2025-12-07T11:26:45.312Z" } +sdist = { url = "https://files.pythonhosted.org/packages/77/1b/2c079f22243462e914026172094411ed7ef1fc96c8089e0ca66d1a14038a/dlthub-0.20.1.tar.gz", hash = "sha256:7b3a188abc28601fd4bdf8f17e7925ef729d4f91fb67a6b4eb5c5dc5a04ac3a2", size = 158432, upload-time = "2025-12-09T15:18:10.813Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/40/7b/307126b81ecefbf2dd7569e2da81b2a7b9124dbdcfa6039f3e350bd097c3/dlthub-0.20.0a1-py3-none-any.whl", hash = "sha256:de1a20567da74943931e8fd3551babd345c9f7dcb1f3782d200b308a79c660ce", size = 209794, upload-time = "2025-12-07T11:26:43.808Z" }, + { url = "https://files.pythonhosted.org/packages/21/94/b2a87853102c6aa08606b2708d8f678b1e39855e8227fe111e37c32631b6/dlthub-0.20.1-py3-none-any.whl", hash = "sha256:c4d4e0c4515cd68f316ccd02c9ecc007332c861ae6f92a488f7e961935e7f1a0", size = 209767, upload-time = "2025-12-09T15:18:09.067Z" }, ] [[package]]