mirror of
https://github.com/dlt-hub/dlt.git
synced 2025-12-17 19:31:30 +00:00
3ebbfa1f9e
* move pyproject.toml and makefile from old branch and add inbetween changes
* update workflow files to use uv
* run new version of formatter
* fix building of images with uv
* possibly fix docs linting
* downgrade lancedb dependency to fix tests
* fix gcs compat mode for s3 for newest boto
* fix docstrings in examples
* add some uv constraints
* update readme.md and contributing.md and some other places
* allow duckdb 0.8 in range
* add link-mode copy to uv venv on windows
* remove poetry lockfile and unneeded lockfile checker
* fix chess api related failures
* sleep after dremio start..
* set correct package in pyproject
* Revert "add some uv constraints"
This reverts commit d611e9ecce.
# Conflicts:
# pyproject.toml
# uv.lock
* add missing databricks sql connector version bounds
341 lines
9.0 KiB
TOML
341 lines
9.0 KiB
TOML
[project]
|
|
name = "dlt"
|
|
version = "1.12.0"
|
|
description = "dlt is an open-source python-first scalable data loading library that does not require any backend to run."
|
|
authors = [{ name = "dltHub Inc.", email = "services@dlthub.com" }]
|
|
requires-python = ">=3.9.2, <3.14, !=3.9.7"
|
|
readme = "README.md"
|
|
license = "Apache-2.0"
|
|
maintainers = [
|
|
{ name = "Marcin Rudolf", email = "marcin@dlthub.com" },
|
|
{ name = "Adrian Brudaru", email = "adrian@dlthub.com" },
|
|
{ name = "Anton Burnashev", email = "anton@dlthub.com" },
|
|
{ name = "David Scharf", email = "david@dlthub.com" },
|
|
]
|
|
keywords = ["etl"]
|
|
classifiers = [
|
|
"Development Status :: 5 - Production/Stable",
|
|
"Intended Audience :: Developers",
|
|
"License :: OSI Approved :: Apache Software License",
|
|
"Topic :: Software Development :: Libraries",
|
|
"Typing :: Typed",
|
|
"Programming Language :: Python :: 3.9",
|
|
"Programming Language :: Python :: 3.10",
|
|
"Programming Language :: Python :: 3.11",
|
|
"Programming Language :: Python :: 3.12",
|
|
"Programming Language :: Python :: 3.13",
|
|
"Operating System :: MacOS :: MacOS X",
|
|
"Operating System :: POSIX :: Linux",
|
|
"Operating System :: Microsoft :: Windows",
|
|
]
|
|
dependencies = [
|
|
"requests>=2.26.0",
|
|
"pendulum>=2.1.2",
|
|
"simplejson>=3.17.5",
|
|
"PyYAML>=5.4.1",
|
|
"semver>=3.0.0",
|
|
"hexbytes>=0.2.2",
|
|
"tzdata>=2022.1",
|
|
"tomlkit>=0.11.3",
|
|
"pathvalidate>=2.5.2",
|
|
"typing-extensions>=4.8.0",
|
|
"click>=7.1",
|
|
"requirements-parser>=0.5.0",
|
|
"setuptools>=65.6.0",
|
|
"humanize>=4.4.0",
|
|
"gitpython>=3.1.29",
|
|
"pytz>=2022.6",
|
|
"giturlparse>=0.10.0",
|
|
"orjson>=3.6.7,<4,!=3.9.11,!=3.9.12,!=3.9.13,!=3.9.14,!=3.10.1 ; platform_python_implementation != 'PyPy'",
|
|
"tenacity>=8.0.2",
|
|
"jsonpath-ng>=1.5.3",
|
|
"fsspec>=2022.4.0",
|
|
"packaging>=21.1",
|
|
"pluggy>=1.3.0",
|
|
"win-precise-time>=1.4.2 ; os_name == 'nt' and python_version < '3.13'",
|
|
"sqlglot>=23.0.0",
|
|
"pywin32>=306 ; sys_platform == 'win32'",
|
|
"rich-argparse>=1.6.0",
|
|
]
|
|
|
|
[project.optional-dependencies]
|
|
gcp = [
|
|
"grpcio>=1.50.0",
|
|
"google-cloud-bigquery>=2.26.0",
|
|
"db-dtypes>=1.2.0",
|
|
"gcsfs>=2022.4.0",
|
|
]
|
|
bigquery = [
|
|
"grpcio>=1.50.0",
|
|
"google-cloud-bigquery>=2.26.0",
|
|
"pyarrow>=14.0.0; python_version < '3.13'",
|
|
"pyarrow>=18.0.0; python_version >= '3.13'",
|
|
"gcsfs>=2022.4.0",
|
|
"db-dtypes>=1.2.0",
|
|
]
|
|
postgres = [
|
|
"psycopg2-binary>=2.9.1"
|
|
]
|
|
redshift = [
|
|
"psycopg2-binary>=2.9.1"
|
|
]
|
|
parquet = [
|
|
"pyarrow>=14.0.0; python_version < '3.13'",
|
|
"pyarrow>=18.0.0 ; python_version >= '3.13'",
|
|
]
|
|
duckdb = [
|
|
"duckdb>=0.9"
|
|
]
|
|
filesystem = [
|
|
"s3fs>=2022.4.0",
|
|
"botocore>=1.28",
|
|
]
|
|
s3 = [
|
|
"s3fs>=2022.4.0",
|
|
"botocore>=1.28",
|
|
]
|
|
gs = [
|
|
"gcsfs>=2022.4.0"
|
|
]
|
|
az = [
|
|
"adlfs>=2024.7.0"
|
|
]
|
|
sftp = [
|
|
"paramiko>=3.3.0"
|
|
]
|
|
snowflake = ["snowflake-connector-python>=3.5.0"]
|
|
motherduck = [
|
|
"duckdb>=0.9",
|
|
"pyarrow>=14.0.0 ; python_version < '3.13'",
|
|
"pyarrow>=18.0.0 ; python_version >= '3.13'",
|
|
]
|
|
cli = [
|
|
"pipdeptree>=2.9.0,<2.10",
|
|
"cron-descriptor>=1.2.32",
|
|
"pip>=23.0.0",
|
|
]
|
|
athena = [
|
|
"pyathena>=2.9.6",
|
|
"pyarrow>=14.0.0 ; python_version < '3.13'",
|
|
"pyarrow>=18.0.0 ; python_version >= '3.13'",
|
|
"s3fs>=2022.4.0",
|
|
"botocore>=1.28",
|
|
]
|
|
weaviate = [
|
|
"weaviate-client>=3.22"
|
|
]
|
|
mssql = [
|
|
"pyodbc>=4.0.39"
|
|
]
|
|
synapse = [
|
|
"pyodbc>=4.0.39",
|
|
"adlfs>=2024.7.0",
|
|
"pyarrow>=14.0.0 ; python_version < '3.13'",
|
|
"pyarrow>=18.0.0 ; python_version >= '3.13'",
|
|
]
|
|
qdrant = [
|
|
"qdrant-client[fastembed]>=1.8"
|
|
]
|
|
databricks = [
|
|
"databricks-sql-connector>=2.9.3,<4 ; python_version <= '3.12'",
|
|
"databricks-sql-connector>=3.6.0 ; python_version >= '3.13'",
|
|
"databricks-sdk>=0.38.0",
|
|
]
|
|
clickhouse = [
|
|
"clickhouse-driver>=0.2.7",
|
|
"clickhouse-connect>=0.7.7",
|
|
"s3fs>=2022.4.0",
|
|
"gcsfs>=2022.4.0",
|
|
"adlfs>=2024.7.0",
|
|
"pyarrow>=14.0.0 ; python_version < '3.13'",
|
|
"pyarrow>=18.0.0 ; python_version >= '3.13'",
|
|
]
|
|
dremio = [
|
|
"pyarrow>=14.0.0 ; python_version < '3.13'",
|
|
"pyarrow>=18.0.0 ; python_version >= '3.13'",
|
|
]
|
|
lancedb = [
|
|
"lancedb>=0.8.2 ; python_version < '3.13'",
|
|
"pyarrow>=14.0.0 ; python_version < '3.13'",
|
|
"pyarrow>=18.0.0 ; python_version >= '3.13'",
|
|
"tantivy>= 0.22.0",
|
|
]
|
|
deltalake = [
|
|
"deltalake>=0.25.1",
|
|
"pyarrow>=14.0.0; python_version < '3.13'",
|
|
"pyarrow>=18.0.0 ; python_version >= '3.13'",
|
|
]
|
|
sql_database = ["sqlalchemy>=1.4"]
|
|
sqlalchemy = [
|
|
"sqlalchemy>=1.4",
|
|
"alembic>1.10.0",
|
|
]
|
|
pyiceberg = [
|
|
"pyiceberg>=0.9.1",
|
|
"pyarrow>=14.0.0 ; python_version < '3.13'",
|
|
"pyarrow>=18.0.0 ; python_version >= '3.13'",
|
|
"sqlalchemy>=1.4",
|
|
]
|
|
postgis = [
|
|
"psycopg2-binary>=2.9.1"
|
|
]
|
|
|
|
[project.urls]
|
|
Homepage = "https://github.com/dlt-hub"
|
|
Repository = "https://github.com/dlt-hub/dlt"
|
|
|
|
[project.scripts]
|
|
dlt = "dlt.cli._dlt:_main"
|
|
|
|
[dependency-groups]
|
|
dev = [
|
|
"cffi>=1.16",
|
|
"greenlet>=3.1",
|
|
"regex>=2024.10",
|
|
"sqlalchemy<2",
|
|
"requests-mock>=1.10.0,<2",
|
|
"types-click>=7.1.8,<8",
|
|
"sqlfluff>=2.3.2,<3",
|
|
"types-deprecated>=1.2.9.2,<2",
|
|
"pytest-console-scripts>=1.4.1,<2",
|
|
"pytest>=7.0.0,<8",
|
|
"mypy>=1.11.0,<1.13.0",
|
|
"flake8>=7.0.0,<8",
|
|
"bandit>=1.7.0,<2",
|
|
"black>=23.7.0,<24",
|
|
"isort>=5.12.0,<6",
|
|
"flake8-bugbear>=22.0.0,<23",
|
|
"pytest-order>=1.0.0",
|
|
"pytest-cases>=3.6.9",
|
|
"pytest-forked>=1.3.0",
|
|
"types-PyYAML>=6.0.7",
|
|
"types-cachetools>=4.2.9",
|
|
"types-protobuf>=3.19.8",
|
|
"types-simplejson>=3.17.0",
|
|
"types-requests>=2.25.6",
|
|
"types-python-dateutil>=2.8.15",
|
|
"flake8-tidy-imports>=4.8.0",
|
|
"flake8-encodings",
|
|
"flake8-builtins>=1.5.3,<2",
|
|
"boto3-stubs>=1.28.28,<2",
|
|
"types-tqdm>=4.66.0.2,<5",
|
|
"types-psutil>=5.9.5.16,<6",
|
|
"types-psycopg2>=2.9.21.14,<3",
|
|
"cryptography>=41.0.7,<42",
|
|
"google-api-python-client>=1.7.11",
|
|
"pytest-asyncio>=0.23.5,<0.24",
|
|
"types-sqlalchemy>=1.4.53.38,<2",
|
|
"types-pytz>=2024.1.0.20240203",
|
|
"ruff>=0.3.2,<0.4",
|
|
"pyjwt>=2.8.0,<3",
|
|
"pytest-mock>=3.14.0,<4",
|
|
"types-regex>=2024.5.15.20240519,<2025",
|
|
"flake8-print>=5.0.0,<6",
|
|
"mimesis>=7.0.0,<8",
|
|
"shapely>=2.0.6",
|
|
"pip>=24.0.0",
|
|
# keep last duckdb version that works with azure and iceberg correctly
|
|
"duckdb<1.2.1",
|
|
"pydoclint>=0.6.5,<0.7",
|
|
# limit the pyarrow version not to test on too new one
|
|
"pyarrow>=14.0.0,<19.0.0",
|
|
]
|
|
sources = [
|
|
"connectorx>=0.3.3 ; python_version >= '3.9'",
|
|
"connectorx>=0.4.0,<0.4.2 ; python_version >= '3.10'",
|
|
"pymysql>=1.1.0,<2",
|
|
"openpyxl>=3,<4",
|
|
]
|
|
pipeline = [
|
|
"google-auth-oauthlib>=1.0.0,<2",
|
|
"tqdm>=4.65.0,<5",
|
|
"enlighten>=1.11.2,<2",
|
|
"alive-progress>=3.1.1,<4",
|
|
"pydantic>=2.10",
|
|
"numpy>=1.21,<2.0 ; python_version >= '3.9' and python_version < '3.12'",
|
|
"numpy>=1.26,<2.0 ; python_version >= '3.12' and python_version < '3.13'",
|
|
"numpy>=2.0.0 ; python_version >= '3.13'",
|
|
"pandas>2.1 ; python_version >= '3.12'",
|
|
"pandas<2.1 ; python_version < '3.12'",
|
|
]
|
|
airflow = ["apache-airflow>=2.8.0,<3 ; python_version < '3.12'"]
|
|
ibis = ["ibis-framework[duckdb, postgres, bigquery, snowflake, mssql, clickhouse, databricks]>=10.3.0 ; python_version >= '3.10'"]
|
|
streamlit = ["streamlit>=1.40.0,<2 ; python_version >= '3.9' and python_version != '3.9.7' and python_version < '3.14'"]
|
|
marimo = [
|
|
"marimo>=0.13.6",
|
|
"playwright>=1.52.0,<2",
|
|
"pytest-playwright>=0.7.0,<1",
|
|
"pyarrow>=14.0.0; python_version < '3.13'",
|
|
"pyarrow>=18.0.0 ; python_version >= '3.13'"
|
|
]
|
|
providers = ["google-api-python-client>=2.86.0,<3"]
|
|
sentry-sdk = ["sentry-sdk>=2.0.0,<3"]
|
|
dbt = [
|
|
"dbt-core>=1.5.0",
|
|
"dbt-redshift>=1.5.0",
|
|
"dbt-bigquery>=1.5.0",
|
|
"dbt-duckdb>=1.5.0",
|
|
"dbt-snowflake>=1.5.0",
|
|
"dbt-athena-community>=1.5.0",
|
|
"dbt-sqlserver>=1.5.0",
|
|
]
|
|
adbc = [
|
|
"adbc-driver-postgresql>=1.6.0"
|
|
]
|
|
docs = [
|
|
"SQLAlchemy>=1.4.0",
|
|
"pymysql>=1.1.0,<2",
|
|
"pypdf2>=3.0.1,<4",
|
|
"pydoc-markdown>=4.8.2,<5",
|
|
"dbt-core>=1.5.0",
|
|
"dbt-duckdb>=1.5.0",
|
|
"pymongo>=4.3.3",
|
|
"pandas>2",
|
|
"alive-progress>=3.0.1",
|
|
"pyarrow>=14.0.0; python_version < '3.13'",
|
|
"pyarrow>=18.0.0 ; python_version >= '3.13'",
|
|
"psycopg2-binary>=2.9",
|
|
"lancedb>=0.8.2 ; python_version < '3.13'",
|
|
"openai>=1.45",
|
|
"connectorx>=0.3.2,<0.4.2",
|
|
"modal>=0.64.170",
|
|
]
|
|
|
|
[tool.uv]
|
|
|
|
[tool.uv.sources]
|
|
flake8-encodings = { git = "https://github.com/dlt-hub/flake8-encodings.git", branch = "disable_jedi_support" }
|
|
|
|
[tool.hatch.build.targets.sdist]
|
|
packages = ["dlt"]
|
|
include = [
|
|
"LICENSE.txt",
|
|
"README.md",
|
|
"dlt/sources/pipeline_templates/.gitignore",
|
|
"dlt/sources/pipeline_templates/.dlt/config.toml",
|
|
]
|
|
|
|
[tool.hatch.build.targets.wheel]
|
|
packages = ["dlt"]
|
|
include = [
|
|
"LICENSE.txt",
|
|
"README.md",
|
|
"dlt/sources/pipeline_templates/.gitignore",
|
|
"dlt/sources/pipeline_templates/.dlt/config.toml",
|
|
]
|
|
|
|
[build-system]
|
|
requires = ["hatchling"]
|
|
build-backend = "hatchling.build"
|
|
|
|
[tool.black]
|
|
line-length = 100
|
|
preview = true
|
|
|
|
[tool.isort]
|
|
color_output = true
|
|
line_length = 100
|
|
profile = "black"
|
|
src_paths = ["dlt"]
|
|
multi_line_output = 3
|