dlt/pyproject.toml

[project]
name = "dlt"
version = "1.20.0"
description = "dlt is an open-source python-first scalable data loading library that does not require any backend to run."
authors = [{ name = "dltHub Inc.", email = "services@dlthub.com" }]
requires-python = ">=3.9.2, <3.15"
readme = "README.md"
license = "Apache-2.0"
maintainers = [
    { name = "Marcin Rudolf", email = "marcin@dlthub.com" },
    { name = "Adrian Brudaru", email = "adrian@dlthub.com" },
    { name = "Anton Burnashev", email = "anton@dlthub.com" },
    { name = "David Scharf", email = "david@dlthub.com" },
]
keywords = ["etl"]
classifiers = [
    "Development Status :: 5 - Production/Stable",
    "Intended Audience :: Developers",
    "License :: OSI Approved :: Apache Software License",
    "Topic :: Software Development :: Libraries",
    "Typing :: Typed",
    "Programming Language :: Python :: 3.9",
    "Programming Language :: Python :: 3.10",
    "Programming Language :: Python :: 3.11",
    "Programming Language :: Python :: 3.12",
    "Programming Language :: Python :: 3.13",
    "Operating System :: MacOS :: MacOS X",
    "Operating System :: POSIX :: Linux",
    "Operating System :: Microsoft :: Windows",
]
dependencies = [
    "requests>=2.26.0",
    "pendulum>=2.1.2",
    "pendulum>=3.0.0 ; python_version > '3.13'",
    "simplejson>=3.17.5",
    "PyYAML>=5.4.1",
    "semver>=3.0.0",
    "tzdata>=2022.1",
    "tomlkit>=0.11.3",
    "pathvalidate>=2.5.2",
    "typing-extensions>=4.8.0",
    "click>=7.1",
    "requirements-parser>=0.5.0",
    "setuptools>=65.6.0",
    "humanize>=4.4.0",
    "gitpython>=3.1.29",
    "pytz>=2022.6",
    "giturlparse>=0.10.0",
    "orjson>=3.6.7,<4,!=3.9.11,!=3.9.12,!=3.9.13,!=3.9.14,!=3.10.1 ; platform_python_implementation != 'PyPy' and sys_platform != 'emscripten'",
    # enables pyodide install, mind that 3.10.1 is a buggy version and should be yanked
    "orjson>=3.10.1 ; platform_python_implementation != 'PyPy' and sys_platform != 'emscripten'",
    "orjson>=3.11.0 ; python_version > '3.13'",
    "tenacity>=8.0.2",
    "jsonpath-ng>=1.5.3",
    "fsspec>=2022.4.0",
    "packaging>=21.1",
    "pluggy>=1.3.0",
    "win-precise-time>=1.4.2 ; os_name == 'nt' and python_version < '3.13'",
    "sqlglot>=25.4.0,!=28.1",
    "pywin32>=306 ; sys_platform == 'win32'",
    "rich-argparse>=1.6.0",
]

[project.optional-dependencies]
gcp = [
    "grpcio>=1.50.0",
    "google-cloud-bigquery>=2.26.0",
    "db-dtypes>=1.2.0",
    "gcsfs>=2022.4.0",
]
bigquery = [
    "grpcio>=1.50.0",
    "google-cloud-bigquery>=2.26.0",
    "pyarrow>=16.0.0",
    "gcsfs>=2022.4.0",
    "db-dtypes>=1.2.0",
]
postgres = [
    "psycopg2-binary>=2.9.1"
]
redshift = [
    "psycopg2-binary>=2.9.1"
]
parquet = [
    "pyarrow>=16.0.0",
]
duckdb = [
    "duckdb>=0.9"
    ]
ducklake = [
    "duckdb>=1.2.0",
    "pyarrow>=16.0.0",
    ]
filesystem = [
    "s3fs>=2022.4.0",
    "botocore>=1.28",
]
s3 = [
    "s3fs>=2022.4.0",
    "botocore>=1.28",
]
gs = [
    "gcsfs>=2022.4.0"
]
az = [
    "adlfs>=2024.7.0"
]
sftp = [
    "paramiko>=3.3.0"
]
http = [
    "aiohttp>3.9.0"
]
snowflake = ["snowflake-connector-python>=3.5.0"]
motherduck = [
    "duckdb>=0.9",
    "pyarrow>=16.0.0",
]
cli = [
    "pipdeptree>=2.9.3,<2.10",
    "cron-descriptor>=1.2.32",
    "pip>=23.0.0",
]
athena = [
    "pyathena>=2.9.6",
    "pyarrow>=16.0.0",
    "s3fs>=2022.4.0",
    "botocore>=1.28",
]
weaviate = [
    "weaviate-client>=3.26.7,<4.0.0"
]
mssql = [
    "pyodbc>=4.0.39"
]
synapse = [
    "pyodbc>=4.0.39",
    "adlfs>=2024.7.0",
    "pyarrow>=16.0.0",
]
qdrant = [
    "qdrant-client[fastembed]>=1.8"
]
databricks = [
    "databricks-sql-connector>=2.9.3 ; python_version <= '3.12'",
    "databricks-sql-connector>=3.6.0 ; python_version >= '3.13'",
    "databricks-sdk>=0.38.0",
]
clickhouse = [
   "clickhouse-driver>=0.2.7",
    "clickhouse-connect>=0.7.7",
    "s3fs>=2022.4.0",
    "gcsfs>=2022.4.0",
    "adlfs>=2024.7.0",
    "pyarrow>=16.0.0",
]
dremio = [
    "pyarrow>=16.0.0",
]
lancedb = [
    "lancedb>=0.22.0 ; python_version < '3.13'",
    "pyarrow>=16.0.0",
    "tantivy>= 0.22.0",
]
deltalake = [
    "deltalake>=0.25.1",
    "pyarrow>=16.0.0",
]
sql_database = ["sqlalchemy>=1.4"]
sqlalchemy = [
    "sqlalchemy>=1.4",
    "alembic>1.10.0",
]
pyiceberg = [
    "pyiceberg>=0.9.1",
    "pyiceberg-core>=0.6.0",
    "pyarrow>=16.0.0",
    "sqlalchemy>=1.4",
]
postgis = [
    "psycopg2-binary>=2.9.1"
]
workspace = [
    "duckdb>=0.9",
    "ibis-framework>=10.5.0 ; python_version >= '3.10'",
    "pyarrow>=16.0.0",
    "marimo>=0.14.5",
    "mcp>=1.2.1 ; python_version >= '3.10'",
    "pathspec>=0.11.2",
]
hub = [
    "dlthub>=0.20.0a1,<0.21 ; python_version >= '3.10'",
    "dlt-runtime>=0.20.0a0,<0.21 ; python_version >= '3.10'",
]

dbml = [
    "pydbml"
]

[project.urls]
Homepage = "https://github.com/dlt-hub"
Repository = "https://github.com/dlt-hub/dlt"

[project.scripts]
dlt = "dlt._workspace.cli._dlt:_main"

[dependency-groups]
# NOTE: add only dependencies used for linting, type checking etc. anything else goes to pipeline group
# we use those deps in common tests to make sure they work without extra dependencies so if you add them
# here you'll break everything
dev = [
    "requests-mock>=1.10.0,<2",
    "types-click>=7.1.8,<8",
    "sqlfluff>=2.3.2,<3",
    "types-deprecated>=1.2.9.2,<2",
    "pytest-console-scripts>=1.4.1,<2",
    "pytest>=7.4.4,<8",
    "mypy>=1.11.0,<1.13.0",
    "flake8>=7.0.0,<8",
    "bandit>=1.7.0,<2",
    "black[jupyter]>=23.7.0,<24",
    "isort>=5.12.0,<6",
    "flake8-bugbear>=22.0.0,<23",
    "pytest-order>=1.0.0",
    "pytest-cases>=3.8.6",
    "pytest-forked>=1.3.0",
    "types-PyYAML>=6.0.7",
    "types-cachetools>=4.2.9",
    "types-protobuf>=3.19.8",
    "types-simplejson>=3.17.0",
    "types-requests>=2.25.6",
    "types-python-dateutil>=2.8.15",
    "flake8-tidy-imports>=4.8.0",
    "flake8-encodings",
    "flake8-builtins>=1.5.3,<2",
    "boto3-stubs[lakeformation,s3]>=1.28.28,<2",
    "types-tqdm>=4.66.0.2,<5",
    "types-psutil>=5.9.5.16,<6",
    "types-psycopg2>=2.9.21.14,<3",
    "pytest-asyncio>=0.23.5,<0.24",
    "types-sqlalchemy>=1.4.53.38,<2",
    "types-pytz>=2024.1.0.20240203",
    "ruff>=0.3.2,<0.4",
    "pyjwt>=2.8.0,<3",
    "pytest-mock>=3.14.0,<4",
    "types-regex>=2024.5.15.20240519,<2025",
    "flake8-print>=5.0.0,<6",
    "pip>=24.0.0",
    "pydoclint>=0.6.5,<0.7",
    "types-paramiko>=3.5.0.20250708",
    "graphviz>=0.21",
    # limits sqlglot - remove when #3489 is fixed
    "sqlglot<28.1",
]

# NOTE: those dependencies are used to test built in sources
sources = [
    "connectorx>=0.4.2 ; python_version >= '3.10'",
    "connectorx<0.4.2 ; python_version < '3.10'",
    "pymysql>=1.1.0,<2",
    "openpyxl>=3,<4",
    "mimesis>=7.0.0,<8",
    "sqlalchemy<2",
]

# NOTE: those dependencies are used to test pipelines, do not include what is in extras for destinations
pipeline = [
    "cffi>=1.16",
    "greenlet>=3.1",
    "regex>=2024.10",
    "sqlalchemy<2",
    "cryptography>=41.0.7,<42",
    "shapely>=2.0.6",
    "tqdm>=4.65.0,<5",
    "enlighten>=1.11.2,<2",
    "alive-progress>=3.1.1,<4",
    "pydantic>=2.10",
    "pyarrow>=16.0.0",
    # this pandas and above has functional numpy dependency
    "pandas>=2.1.4",
    # 1.4 dev fails on azure again
    "duckdb<1.4",
    # 1.3 crashes windows
    "duckdb<1.2.1 ; os_name == 'nt'",
]
airflow = ["apache-airflow>=2.8.0,<3 ; python_version < '3.12'"]
ibis = [
    "ibis-framework[duckdb, postgres, bigquery, snowflake, mssql, clickhouse, databricks]>=10.5.0 ; python_version >= '3.10' and python_version < '3.13'",
    "ibis-framework[duckdb, postgres, bigquery, snowflake, clickhouse, databricks]>=10.5.0 ; python_version >= '3.13'"
]
streamlit = ["streamlit>=1.40.0,<2 ; python_version >= '3.9' and python_version != '3.9.7' and python_version < '3.14'"]
dashboard-tests = [
    "playwright>=1.52.0,<2",
    "pytest-playwright>=0.7.0,<1",
    ]
# NOTE: required to use google secrets config provider
providers = ["google-api-python-client>=2.86.0,<3"]
sentry-sdk = ["sentry-sdk>=2.0.0,<3"]
dbt = [
    "dbt-core>=1.5.0",
    "dbt-redshift>=1.5.0",
    "dbt-bigquery>=1.5.0",
    "dbt-duckdb>=1.5.0",
    "dbt-snowflake>=1.5.0",
    "dbt-athena-community>=1.5.0",
    "dbt-sqlserver>=1.5.0 ; python_version < '3.13'",
]
adbc = [
    "dbc>=0.1.0",
    "adbc-driver-manager>=1.8.0",
]

[project.entry-points.dlt]
dlt = "dlt.__plugins__"

[tool.uv]

[tool.uv.sources]
flake8-encodings = { git = "https://github.com/dlt-hub/flake8-encodings.git", branch = "disable_jedi_support" }
# dlthub = { path = "../dlt-plus/packages/dlthub", editable = true }

[tool.hatch.build.targets.sdist]
packages = ["dlt"]
# hatchling follows .gitignore and will include all files in gitindex under `dlt`
include = [
    "LICENSE.txt",
    "README.md",
]

[tool.hatch.build.targets.wheel]
packages = ["dlt"]
include = [
    "LICENSE.txt",
    "README.md",
]

[build-system]
requires = ["hatchling"]
build-backend = "hatchling.build"


[tool.ruff]
line-length = 100
target-version = "py39"
src = ["dlt"]
include = [
    "dlt/**/*.py",
    "tests/**/*.py",
    "docs/**/*.py",
]
exclude = [
    "docs/website/.dlt-repo",
    "tests/reflection/module_cases/syntax_error.py",
    "docs/website/docs/general-usage/transformations/transformation-snippets.py",
]

[tool.ruff.lint]
preview = true
ignore = [
    # from tox.ini
    "E1",
    "E2",
    "E3",
    "E4",
    "F401",
    "W391",
    "W292",
    "E501",
    # additional ignores to appease ruff
    "E713",
    "E721",
    "E731",
    "E741",
    "E999",
    "F601",
    "F821",
    "F841",
    "F811",
]

[tool.black]
line-length = 100
preview = true

[tool.isort]
color_output = true
line_length = 100
profile = "black"
src_paths = ["dlt"]
multi_line_output = 3

[tool.pytest.ini_options]
pythonpath = ["dlt"]
norecursedirs = [".direnv", ".eggs", "build", "dist"]
addopts = "--showlocals --durations 10 -m 'not rfam'"
xfail_strict = true
log_cli_level = "INFO"
console_output_style = "count"
python_files = ["test_*.py", "*_test.py"]
python_functions = ["*_test", "test_*"]
filterwarnings = ["ignore::DeprecationWarning"]
markers = [
    "essential: marks all essential tests",
    "no_load: marks tests that do not load anything",
    "rfam: marks tests that use rfam db"
]