mirror of
https://github.com/dlt-hub/dlt.git
synced 2025-12-17 19:31:30 +00:00
99207237fe
* bumps to version 1.20.0 * update the hub reference docs, add CI check * use dependency specifier in hub for plugin version check * minimum dlt runtime cli check * rollaback to old fsspec min version * fixes test_hub ci workflow * fixes flaky test * bumps hub extra * updates cli docs linting * fixes docs lock --------- Co-authored-by: Marcin Rudolf <rudolfix@rudolfix.org> Co-authored-by: ivasio <ivan@dlthub.com>
404 lines
10 KiB
TOML
404 lines
10 KiB
TOML
[project]
|
|
name = "dlt"
|
|
version = "1.20.0"
|
|
description = "dlt is an open-source python-first scalable data loading library that does not require any backend to run."
|
|
authors = [{ name = "dltHub Inc.", email = "services@dlthub.com" }]
|
|
requires-python = ">=3.9.2, <3.15"
|
|
readme = "README.md"
|
|
license = "Apache-2.0"
|
|
maintainers = [
|
|
{ name = "Marcin Rudolf", email = "marcin@dlthub.com" },
|
|
{ name = "Adrian Brudaru", email = "adrian@dlthub.com" },
|
|
{ name = "Anton Burnashev", email = "anton@dlthub.com" },
|
|
{ name = "David Scharf", email = "david@dlthub.com" },
|
|
]
|
|
keywords = ["etl"]
|
|
classifiers = [
|
|
"Development Status :: 5 - Production/Stable",
|
|
"Intended Audience :: Developers",
|
|
"License :: OSI Approved :: Apache Software License",
|
|
"Topic :: Software Development :: Libraries",
|
|
"Typing :: Typed",
|
|
"Programming Language :: Python :: 3.9",
|
|
"Programming Language :: Python :: 3.10",
|
|
"Programming Language :: Python :: 3.11",
|
|
"Programming Language :: Python :: 3.12",
|
|
"Programming Language :: Python :: 3.13",
|
|
"Operating System :: MacOS :: MacOS X",
|
|
"Operating System :: POSIX :: Linux",
|
|
"Operating System :: Microsoft :: Windows",
|
|
]
|
|
dependencies = [
|
|
"requests>=2.26.0",
|
|
"pendulum>=2.1.2",
|
|
"pendulum>=3.0.0 ; python_version > '3.13'",
|
|
"simplejson>=3.17.5",
|
|
"PyYAML>=5.4.1",
|
|
"semver>=3.0.0",
|
|
"tzdata>=2022.1",
|
|
"tomlkit>=0.11.3",
|
|
"pathvalidate>=2.5.2",
|
|
"typing-extensions>=4.8.0",
|
|
"click>=7.1",
|
|
"requirements-parser>=0.5.0",
|
|
"setuptools>=65.6.0",
|
|
"humanize>=4.4.0",
|
|
"gitpython>=3.1.29",
|
|
"pytz>=2022.6",
|
|
"giturlparse>=0.10.0",
|
|
"orjson>=3.6.7,<4,!=3.9.11,!=3.9.12,!=3.9.13,!=3.9.14,!=3.10.1 ; platform_python_implementation != 'PyPy' and sys_platform != 'emscripten'",
|
|
# enables pyodide install, mind that 3.10.1 is a buggy version and should be yanked
|
|
"orjson>=3.10.1 ; platform_python_implementation != 'PyPy' and sys_platform != 'emscripten'",
|
|
"orjson>=3.11.0 ; python_version > '3.13'",
|
|
"tenacity>=8.0.2",
|
|
"jsonpath-ng>=1.5.3",
|
|
"fsspec>=2022.4.0",
|
|
"packaging>=21.1",
|
|
"pluggy>=1.3.0",
|
|
"win-precise-time>=1.4.2 ; os_name == 'nt' and python_version < '3.13'",
|
|
"sqlglot>=25.4.0,!=28.1",
|
|
"pywin32>=306 ; sys_platform == 'win32'",
|
|
"rich-argparse>=1.6.0",
|
|
]
|
|
|
|
[project.optional-dependencies]
|
|
gcp = [
|
|
"grpcio>=1.50.0",
|
|
"google-cloud-bigquery>=2.26.0",
|
|
"db-dtypes>=1.2.0",
|
|
"gcsfs>=2022.4.0",
|
|
]
|
|
bigquery = [
|
|
"grpcio>=1.50.0",
|
|
"google-cloud-bigquery>=2.26.0",
|
|
"pyarrow>=16.0.0",
|
|
"gcsfs>=2022.4.0",
|
|
"db-dtypes>=1.2.0",
|
|
]
|
|
postgres = [
|
|
"psycopg2-binary>=2.9.1"
|
|
]
|
|
redshift = [
|
|
"psycopg2-binary>=2.9.1"
|
|
]
|
|
parquet = [
|
|
"pyarrow>=16.0.0",
|
|
]
|
|
duckdb = [
|
|
"duckdb>=0.9"
|
|
]
|
|
ducklake = [
|
|
"duckdb>=1.2.0",
|
|
"pyarrow>=16.0.0",
|
|
]
|
|
filesystem = [
|
|
"s3fs>=2022.4.0",
|
|
"botocore>=1.28",
|
|
]
|
|
s3 = [
|
|
"s3fs>=2022.4.0",
|
|
"botocore>=1.28",
|
|
]
|
|
gs = [
|
|
"gcsfs>=2022.4.0"
|
|
]
|
|
az = [
|
|
"adlfs>=2024.7.0"
|
|
]
|
|
sftp = [
|
|
"paramiko>=3.3.0"
|
|
]
|
|
http = [
|
|
"aiohttp>3.9.0"
|
|
]
|
|
snowflake = ["snowflake-connector-python>=3.5.0"]
|
|
motherduck = [
|
|
"duckdb>=0.9",
|
|
"pyarrow>=16.0.0",
|
|
]
|
|
cli = [
|
|
"pipdeptree>=2.9.3,<2.10",
|
|
"cron-descriptor>=1.2.32",
|
|
"pip>=23.0.0",
|
|
]
|
|
athena = [
|
|
"pyathena>=2.9.6",
|
|
"pyarrow>=16.0.0",
|
|
"s3fs>=2022.4.0",
|
|
"botocore>=1.28",
|
|
]
|
|
weaviate = [
|
|
"weaviate-client>=3.26.7,<4.0.0"
|
|
]
|
|
mssql = [
|
|
"pyodbc>=4.0.39"
|
|
]
|
|
synapse = [
|
|
"pyodbc>=4.0.39",
|
|
"adlfs>=2024.7.0",
|
|
"pyarrow>=16.0.0",
|
|
]
|
|
qdrant = [
|
|
"qdrant-client[fastembed]>=1.8"
|
|
]
|
|
databricks = [
|
|
"databricks-sql-connector>=2.9.3 ; python_version <= '3.12'",
|
|
"databricks-sql-connector>=3.6.0 ; python_version >= '3.13'",
|
|
"databricks-sdk>=0.38.0",
|
|
]
|
|
clickhouse = [
|
|
"clickhouse-driver>=0.2.7",
|
|
"clickhouse-connect>=0.7.7",
|
|
"s3fs>=2022.4.0",
|
|
"gcsfs>=2022.4.0",
|
|
"adlfs>=2024.7.0",
|
|
"pyarrow>=16.0.0",
|
|
]
|
|
dremio = [
|
|
"pyarrow>=16.0.0",
|
|
]
|
|
lancedb = [
|
|
"lancedb>=0.22.0 ; python_version < '3.13'",
|
|
"pyarrow>=16.0.0",
|
|
"tantivy>= 0.22.0",
|
|
]
|
|
deltalake = [
|
|
"deltalake>=0.25.1",
|
|
"pyarrow>=16.0.0",
|
|
]
|
|
sql_database = ["sqlalchemy>=1.4"]
|
|
sqlalchemy = [
|
|
"sqlalchemy>=1.4",
|
|
"alembic>1.10.0",
|
|
]
|
|
pyiceberg = [
|
|
"pyiceberg>=0.9.1",
|
|
"pyarrow>=16.0.0",
|
|
"sqlalchemy>=1.4",
|
|
]
|
|
postgis = [
|
|
"psycopg2-binary>=2.9.1"
|
|
]
|
|
workspace = [
|
|
"duckdb>=0.9",
|
|
"ibis-framework>=10.5.0 ; python_version >= '3.10'",
|
|
"pyarrow>=16.0.0",
|
|
"marimo>=0.14.5",
|
|
"mcp>=1.2.1 ; python_version >= '3.10'",
|
|
"pathspec>=0.11.2",
|
|
]
|
|
hub = [
|
|
"dlthub>=0.20.0a1,<0.21 ; python_version >= '3.10'",
|
|
"dlt-runtime>=0.20.0a0,<0.21 ; python_version >= '3.10'",
|
|
]
|
|
|
|
dbml = [
|
|
"pydbml"
|
|
]
|
|
|
|
[project.urls]
|
|
Homepage = "https://github.com/dlt-hub"
|
|
Repository = "https://github.com/dlt-hub/dlt"
|
|
|
|
[project.scripts]
|
|
dlt = "dlt._workspace.cli._dlt:_main"
|
|
|
|
[dependency-groups]
|
|
# NOTE: add only dependencies used for linting, type checking etc. anything else goes to pipeline group
|
|
# we use those deps in common tests to make sure they work without extra dependencies so if you add them
|
|
# here you'll break everything
|
|
dev = [
|
|
"requests-mock>=1.10.0,<2",
|
|
"types-click>=7.1.8,<8",
|
|
"sqlfluff>=2.3.2,<3",
|
|
"types-deprecated>=1.2.9.2,<2",
|
|
"pytest-console-scripts>=1.4.1,<2",
|
|
"pytest>=7.4.4,<8",
|
|
"mypy>=1.11.0,<1.13.0",
|
|
"flake8>=7.0.0,<8",
|
|
"bandit>=1.7.0,<2",
|
|
"black[jupyter]>=23.7.0,<24",
|
|
"isort>=5.12.0,<6",
|
|
"flake8-bugbear>=22.0.0,<23",
|
|
"pytest-order>=1.0.0",
|
|
"pytest-cases>=3.8.6",
|
|
"pytest-forked>=1.3.0",
|
|
"types-PyYAML>=6.0.7",
|
|
"types-cachetools>=4.2.9",
|
|
"types-protobuf>=3.19.8",
|
|
"types-simplejson>=3.17.0",
|
|
"types-requests>=2.25.6",
|
|
"types-python-dateutil>=2.8.15",
|
|
"flake8-tidy-imports>=4.8.0",
|
|
"flake8-encodings",
|
|
"flake8-builtins>=1.5.3,<2",
|
|
"boto3-stubs[lakeformation,s3]>=1.28.28,<2",
|
|
"types-tqdm>=4.66.0.2,<5",
|
|
"types-psutil>=5.9.5.16,<6",
|
|
"types-psycopg2>=2.9.21.14,<3",
|
|
"pytest-asyncio>=0.23.5,<0.24",
|
|
"types-sqlalchemy>=1.4.53.38,<2",
|
|
"types-pytz>=2024.1.0.20240203",
|
|
"ruff>=0.3.2,<0.4",
|
|
"pyjwt>=2.8.0,<3",
|
|
"pytest-mock>=3.14.0,<4",
|
|
"types-regex>=2024.5.15.20240519,<2025",
|
|
"flake8-print>=5.0.0,<6",
|
|
"pip>=24.0.0",
|
|
"pydoclint>=0.6.5,<0.7",
|
|
"types-paramiko>=3.5.0.20250708",
|
|
"graphviz>=0.21",
|
|
]
|
|
|
|
# NOTE: those dependencies are used to test built in sources
|
|
sources = [
|
|
"connectorx>=0.4.2 ; python_version >= '3.10'",
|
|
"connectorx<0.4.2 ; python_version < '3.10'",
|
|
"pymysql>=1.1.0,<2",
|
|
"openpyxl>=3,<4",
|
|
"mimesis>=7.0.0,<8",
|
|
"sqlalchemy<2",
|
|
]
|
|
|
|
# NOTE: those dependencies are used to test pipelines, do not include what is in extras for destinations
|
|
pipeline = [
|
|
"cffi>=1.16",
|
|
"greenlet>=3.1",
|
|
"regex>=2024.10",
|
|
"sqlalchemy<2",
|
|
"cryptography>=41.0.7,<42",
|
|
"shapely>=2.0.6",
|
|
"tqdm>=4.65.0,<5",
|
|
"enlighten>=1.11.2,<2",
|
|
"alive-progress>=3.1.1,<4",
|
|
"pydantic>=2.10",
|
|
"pyarrow>=16.0.0",
|
|
# this pandas and above has functional numpy dependency
|
|
"pandas>=2.1.4",
|
|
# 1.4 dev fails on azure again
|
|
"duckdb<1.4",
|
|
# 1.3 crashes windows
|
|
"duckdb<1.2.1 ; os_name == 'nt'",
|
|
]
|
|
airflow = ["apache-airflow>=2.8.0,<3 ; python_version < '3.12'"]
|
|
ibis = [
|
|
"ibis-framework[duckdb, postgres, bigquery, snowflake, mssql, clickhouse, databricks]>=10.5.0 ; python_version >= '3.10' and python_version < '3.13'",
|
|
"ibis-framework[duckdb, postgres, bigquery, snowflake, clickhouse, databricks]>=10.5.0 ; python_version >= '3.13'"
|
|
]
|
|
streamlit = ["streamlit>=1.40.0,<2 ; python_version >= '3.9' and python_version != '3.9.7' and python_version < '3.14'"]
|
|
dashboard-tests = [
|
|
"playwright>=1.52.0,<2",
|
|
"pytest-playwright>=0.7.0,<1",
|
|
]
|
|
# NOTE: required to use google secrets config provider
|
|
providers = ["google-api-python-client>=2.86.0,<3"]
|
|
sentry-sdk = ["sentry-sdk>=2.0.0,<3"]
|
|
dbt = [
|
|
"dbt-core>=1.5.0",
|
|
"dbt-redshift>=1.5.0",
|
|
"dbt-bigquery>=1.5.0",
|
|
"dbt-duckdb>=1.5.0",
|
|
"dbt-snowflake>=1.5.0",
|
|
"dbt-athena-community>=1.5.0",
|
|
"dbt-sqlserver>=1.5.0 ; python_version < '3.13'",
|
|
]
|
|
adbc = [
|
|
"dbc>=0.1.0",
|
|
"adbc-driver-manager>=1.8.0",
|
|
]
|
|
|
|
[project.entry-points.dlt]
|
|
dlt = "dlt.__plugins__"
|
|
|
|
[tool.uv]
|
|
|
|
[tool.uv.sources]
|
|
flake8-encodings = { git = "https://github.com/dlt-hub/flake8-encodings.git", branch = "disable_jedi_support" }
|
|
# dlthub = { path = "../dlt-plus/packages/dlthub", editable = true }
|
|
|
|
[tool.hatch.build.targets.sdist]
|
|
packages = ["dlt"]
|
|
# hatchling follows .gitignore and will include all files in gitindex under `dlt`
|
|
include = [
|
|
"LICENSE.txt",
|
|
"README.md",
|
|
]
|
|
|
|
[tool.hatch.build.targets.wheel]
|
|
packages = ["dlt"]
|
|
include = [
|
|
"LICENSE.txt",
|
|
"README.md",
|
|
]
|
|
|
|
[build-system]
|
|
requires = ["hatchling"]
|
|
build-backend = "hatchling.build"
|
|
|
|
|
|
[tool.ruff]
|
|
line-length = 100
|
|
target-version = "py39"
|
|
src = ["dlt"]
|
|
include = [
|
|
"dlt/**/*.py",
|
|
"tests/**/*.py",
|
|
"docs/**/*.py",
|
|
]
|
|
exclude = [
|
|
"docs/website/.dlt-repo",
|
|
"tests/reflection/module_cases/syntax_error.py",
|
|
"docs/website/docs/general-usage/transformations/transformation-snippets.py",
|
|
]
|
|
|
|
[tool.ruff.lint]
|
|
preview = true
|
|
ignore = [
|
|
# from tox.ini
|
|
"E1",
|
|
"E2",
|
|
"E3",
|
|
"E4",
|
|
"F401",
|
|
"W391",
|
|
"W292",
|
|
"E501",
|
|
# additional ignores to appease ruff
|
|
"E713",
|
|
"E721",
|
|
"E731",
|
|
"E741",
|
|
"E999",
|
|
"F601",
|
|
"F821",
|
|
"F841",
|
|
"F811",
|
|
]
|
|
|
|
[tool.black]
|
|
line-length = 100
|
|
preview = true
|
|
|
|
[tool.isort]
|
|
color_output = true
|
|
line_length = 100
|
|
profile = "black"
|
|
src_paths = ["dlt"]
|
|
multi_line_output = 3
|
|
|
|
[tool.pytest.ini_options]
|
|
pythonpath = ["dlt"]
|
|
norecursedirs = [".direnv", ".eggs", "build", "dist"]
|
|
addopts = "--showlocals --durations 10 -m 'not rfam'"
|
|
xfail_strict = true
|
|
log_cli_level = "INFO"
|
|
console_output_style = "count"
|
|
python_files = ["test_*.py", "*_test.py"]
|
|
python_functions = ["*_test", "test_*"]
|
|
filterwarnings = ["ignore::DeprecationWarning"]
|
|
markers = [
|
|
"essential: marks all essential tests",
|
|
"no_load: marks tests that do not load anything",
|
|
"rfam: marks tests that use rfam db"
|
|
]
|