From 7e10fc72d59e77e687c211b92d483a6b13779048 Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Thu, 6 Nov 2025 14:08:00 +0000 Subject: [PATCH] Move from setup.py to pyproject.toml (#12129) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * convert setup.py to pyproject.toml * move dev requirements into pyproject.toml * with setup.py gone we can install from root * lint cleanrly state intention to remove * convert precommit to use dev deps * consolidate version to pyproject.toml * editable req get rid of editable-req * docs updates * tweak configs for builds * fix script * changelog * fixes to build * revert unnecesary changes more simplification revert linting more simplification fix don’t need it --- .bumpversion.cfg | 6 +- .../Under the Hood-20251029-133450.yaml | 6 + ARCHITECTURE.md | 2 +- core/dbt/version.py | 21 +++- core/pyproject.toml | 110 +++++++++++++++++ core/setup.py | 115 +++--------------- pyproject.toml | 4 + scripts/build-dist.sh | 17 ++- scripts/update_dev_packages.sh | 8 +- 9 files changed, 181 insertions(+), 108 deletions(-) create mode 100644 .changes/unreleased/Under the Hood-20251029-133450.yaml create mode 100644 core/pyproject.toml diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 5739043fc..681310f4f 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -32,6 +32,6 @@ first_value = 1 [bumpversion:part:nightly] -[bumpversion:file:core/setup.py] - -[bumpversion:file:core/dbt/version.py] +[bumpversion:file:core/pyproject.toml] +search = version = "{current_version}" +replace = version = "{new_version}" diff --git a/.changes/unreleased/Under the Hood-20251029-133450.yaml b/.changes/unreleased/Under the Hood-20251029-133450.yaml new file mode 100644 index 000000000..bb80f8a39 --- /dev/null +++ b/.changes/unreleased/Under the Hood-20251029-133450.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: Move from setup.py to pyproject.toml +time: 2025-10-29T13:34:50.106244-04:00 +custom: + Author: emmyoop + Issue: "5696" diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 55261b8da..aae6ee9c3 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -39,7 +39,7 @@ Each adapter plugin is a standalone python package that includes: - `dbt/include/[name]`: A "sub-global" dbt project, of YAML and SQL files, that reimplements Jinja macros to use the adapter's supported SQL syntax - `dbt/adapters/[name]`: Python modules that inherit, and optionally reimplement, the base adapter classes defined in dbt-core -- `setup.py` +- `pyproject.toml` The Postgres adapter code is the most central, and many of its implementations are used as the default defined in the dbt-core global project. The greater the distance of a data technology from Postgres, the more its adapter plugin may need to reimplement. diff --git a/core/dbt/version.py b/core/dbt/version.py index 73cb8b023..5ac124e54 100644 --- a/core/dbt/version.py +++ b/core/dbt/version.py @@ -3,6 +3,9 @@ import importlib import importlib.util import json import os +import re +from importlib import metadata as importlib_metadata +from pathlib import Path from typing import Iterator, List, Optional, Tuple import requests @@ -226,5 +229,21 @@ def _get_adapter_plugin_names() -> Iterator[str]: yield plugin_name -__version__ = "1.11.0b4" +def _resolve_version() -> str: + try: + return importlib_metadata.version("dbt-core") + except importlib_metadata.PackageNotFoundError: + pyproject_path = Path(__file__).resolve().parents[1] / "pyproject.toml" + if not pyproject_path.exists(): + raise RuntimeError("Unable to locate pyproject.toml to determine dbt-core version") + + text = pyproject_path.read_text(encoding="utf-8") + match = re.search(r'^version\s*=\s*"(?P[^"]+)"', text, re.MULTILINE) + if match: + return match.group("version") + + raise RuntimeError("Unable to determine dbt-core version from pyproject.toml") + + +__version__ = _resolve_version() installed = get_installed_version() diff --git a/core/pyproject.toml b/core/pyproject.toml new file mode 100644 index 000000000..6e2bf21db --- /dev/null +++ b/core/pyproject.toml @@ -0,0 +1,110 @@ +[tool.setuptools] +package-dir = {"" = "."} +include-package-data = true +zip-safe = false + +[tool.setuptools.packages.find] +where = ["."] +include = [ + "dbt", + "dbt.*", +] + +# this needs to match MANIFEST.in for the wheels +[tool.setuptools.package-data] +"dbt" = [ + "include/**/*.py", + "include/**/*.sql", + "include/**/*.yml", + "include/**/*.html", + "include/**/*.md", + "include/**/.gitkeep", + "include/**/.gitignore", + "task/docs/**/*.html", + "jsonschemas/**/*.json", + "py.typed", +] + +[project] +name = "dbt-core" +version = "1.11.0b4" +description = "With dbt, data analysts and engineers can build analytics the way engineers build applications." +readme = "README.md" +requires-python = ">=3.10" +license = "Apache-2.0" +license-files = ["License.md"] # License.md copied to core/ by build script even though it lives at the root by convention +keywords = [] +authors = [ + { name = "dbt Labs", email = "info@dbtlabs.com" }, +] +maintainers = [ + { name = "dbt Labs", email = "info@dbtlabs.com" }, +] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Operating System :: Microsoft :: Windows", + "Operating System :: MacOS :: MacOS X", + "Operating System :: POSIX :: Linux", + "Programming Language :: Python", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: PyPy", +] +dependencies = [ + # ---- + # dbt-core uses these packages deeply, throughout the codebase, and there have been breaking changes in past patch releases (even though these are major-version-one). + # Pin to the patch or minor version, and bump in each new minor version of dbt-core. + "agate>=1.7.0,<1.10", + "Jinja2>=3.1.3,<4", + "mashumaro[msgpack]>=3.9,<3.15", + # ---- + # dbt-core uses these packages in standard ways. Pin to the major version, and check compatibility + # with major versions in each new minor version of dbt-core. + "click>=8.0.2,<9.0", + "jsonschema>=4.19.1,<5.0", + "networkx>=2.3,<4.0", + "protobuf>=6.0,<7.0", + "requests<3.0.0", # should match dbt-common + "snowplow-tracker>=1.0.2,<2.0", + # ---- + # These packages are major-version-0. Keep upper bounds on upcoming minor versions (which could have breaking changes) + # and check compatibility / bump in each new minor version of dbt-core. + "pathspec>=0.9,<0.13", + "sqlparse>=0.5.0,<0.6.0", + # ---- + # These are major-version-0 packages also maintained by dbt-labs. + # Accept patches but avoid automatically updating past a set minor version range. + "dbt-extractor>=0.5.0,<=0.6", + "dbt-semantic-interfaces>=0.9.0,<0.10", + # Minor versions for these are expected to be backwards-compatible + "dbt-common>=1.27.0,<2.0", + "dbt-adapters>=1.15.5,<2.0", + "dbt-protos>=1.0.375,<2.0", + "pydantic<3", + # ---- + # Expect compatibility with all new versions of these packages, so lower bounds only. + "packaging>20.9", + "pytz>=2015.7", + "pyyaml>=6.0", + "daff>=1.3.46", + "typing-extensions>=4.4", +] + +[project.urls] +Homepage = "https://github.com/dbt-labs/dbt-core" +Repository = "https://github.com/dbt-labs/dbt-core.git" +Issues = "https://github.com/dbt-labs/dbt-core/issues" +Changelog = "https://github.com/dbt-labs/dbt-core/blob/main/CHANGELOG.md" + +[project.scripts] +dbt = "dbt.cli.main:cli" + +[build-system] +requires = [ + "setuptools>=61", + "wheel", +] +build-backend = "setuptools.build_meta" diff --git a/core/setup.py b/core/setup.py index 29ddf5f4d..3c2321f5a 100644 --- a/core/setup.py +++ b/core/setup.py @@ -1,101 +1,26 @@ #!/usr/bin/env python -import os -import sys - -if sys.version_info < (3, 10): - print("Error: dbt does not support this version of Python.") - print("Please upgrade to Python 3.10 or higher.") - sys.exit(1) +"""Legacy setuptools shim retained for compatibility with existing workflows. Will be removed in a future version.""" from setuptools import setup -try: - from setuptools import find_namespace_packages -except ImportError: - # the user has a downlevel version of setuptools. - print("Error: dbt requires setuptools v40.1.0 or higher.") - print('Please upgrade setuptools with "pip install --upgrade setuptools" ' "and try again") - sys.exit(1) +# the user has a downlevel version of setuptools. +# ---- +# dbt-core uses these packages deeply, throughout the codebase, and there have been breaking changes in past patch releases (even though these are major-version-one). +# Pin to the patch or minor version, and bump in each new minor version of dbt-core. +# ---- +# dbt-core uses these packages in standard ways. Pin to the major version, and check compatibility +# with major versions in each new minor version of dbt-core. +# ---- +# These packages are major-version-0. Keep upper bounds on upcoming minor versions (which could have breaking changes) +# and check compatibility / bump in each new minor version of dbt-core. +# ---- +# These are major-version-0 packages also maintained by dbt-labs. +# Accept patches but avoid automatically updating past a set minor version range. +# Minor versions for these are expected to be backwards-compatible +# ---- +# Expect compatibility with all new versions of these packages, so lower bounds only. +# ---- - -this_directory = os.path.abspath(os.path.dirname(__file__)) -with open(os.path.join(this_directory, "README.md")) as f: - long_description = f.read() - - -package_name = "dbt-core" -package_version = "1.11.0b4" -description = """With dbt, data analysts and engineers can build analytics \ -the way engineers build applications.""" - - -setup( - name=package_name, - version=package_version, - description=description, - long_description=long_description, - long_description_content_type="text/markdown", - author="dbt Labs", - author_email="info@dbtlabs.com", - url="https://github.com/dbt-labs/dbt-core", - packages=find_namespace_packages(include=["dbt", "dbt.*"]), - include_package_data=True, - test_suite="test", - entry_points={ - "console_scripts": ["dbt = dbt.cli.main:cli"], - }, - install_requires=[ - # ---- - # dbt-core uses these packages deeply, throughout the codebase, and there have been breaking changes in past patch releases (even though these are major-version-one). - # Pin to the patch or minor version, and bump in each new minor version of dbt-core. - "agate>=1.7.0,<1.10", - "Jinja2>=3.1.3,<4", - "mashumaro[msgpack]>=3.9,<3.15", - # ---- - # dbt-core uses these packages in standard ways. Pin to the major version, and check compatibility - # with major versions in each new minor version of dbt-core. - "click>=8.0.2,<9.0", - "jsonschema>=4.19.1,<5.0", - "networkx>=2.3,<4.0", - "protobuf>=6.0,<7.0", - "requests<3.0.0", # should match dbt-common - "snowplow-tracker>=1.0.2,<2.0", - # ---- - # These packages are major-version-0. Keep upper bounds on upcoming minor versions (which could have breaking changes) - # and check compatibility / bump in each new minor version of dbt-core. - "pathspec>=0.9,<0.13", - "sqlparse>=0.5.0,<0.6.0", - # ---- - # These are major-version-0 packages also maintained by dbt-labs. - # Accept patches but avoid automatically updating past a set minor version range. - "dbt-extractor>=0.5.0,<=0.6", - "dbt-semantic-interfaces>=0.9.0,<0.10", - # Minor versions for these are expected to be backwards-compatible - "dbt-common>=1.27.0,<2.0", - "dbt-adapters>=1.15.5,<2.0", - "dbt-protos>=1.0.375,<2.0", - "pydantic<3", - # ---- - # Expect compatibility with all new versions of these packages, so lower bounds only. - "packaging>20.9", - "pytz>=2015.7", - "pyyaml>=6.0", - "daff>=1.3.46", - "typing-extensions>=4.4", - # ---- - ], - zip_safe=False, - classifiers=[ - "Development Status :: 5 - Production/Stable", - "License :: OSI Approved :: Apache Software License", - "Operating System :: Microsoft :: Windows", - "Operating System :: MacOS :: MacOS X", - "Operating System :: POSIX :: Linux", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: 3.12", - "Programming Language :: Python :: 3.13", - ], - python_requires=">=3.10", -) +if __name__ == "__main__": + setup() diff --git a/pyproject.toml b/pyproject.toml index 89f288c93..b050ddc0c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,7 @@ +# Root-level pyproject.toml for tool configurations +# Packaging configuration is in core/pyproject.toml +# This file exists so tools like mypy and black can find their config when run from root + [tool.mypy] # TODO: widen range of files as we fix issues files = 'core/dbt' diff --git a/scripts/build-dist.sh b/scripts/build-dist.sh index 5dd0fa17f..c36321e39 100755 --- a/scripts/build-dist.sh +++ b/scripts/build-dist.sh @@ -12,13 +12,20 @@ set -x rm -rf "$DBT_PATH"/dist rm -rf "$DBT_PATH"/build +rm -rf "$DBT_PATH"/core/dist +rm -rf "$DBT_PATH"/core/build + mkdir -p "$DBT_PATH"/dist -rm -rf "$DBT_PATH"/core/dist -rm -rf "$DBT_PATH"core/build -cd "$DBT_PATH"/core -$PYTHON_BIN setup.py sdist bdist_wheel -cp -r "$DBT_PATH"/"core"/dist/* "$DBT_PATH"/dist/ +# Copy License.md to core/ for inclusion in distribution (required by Apache 2.0) +# The license-files in pyproject.toml references it relative to core/ +cp "$DBT_PATH"/License.md "$DBT_PATH"/core/License.md +cd "$DBT_PATH"/core +$PYTHON_BIN -m pip install --upgrade build +$PYTHON_BIN -m build --outdir "$DBT_PATH/dist" + +# Clean up License.md that was copied to core/ for build +rm -f "$DBT_PATH/core/License.md" set +x diff --git a/scripts/update_dev_packages.sh b/scripts/update_dev_packages.sh index 3f140f04b..be5349843 100755 --- a/scripts/update_dev_packages.sh +++ b/scripts/update_dev_packages.sh @@ -1,14 +1,16 @@ #!/bin/bash -e set -e +# this is used in dbt-common for CI + repo=$1 ref=$2 target_req_file="dev-requirements.txt" req_sed_pattern="s|${repo}.git@main|${repo}.git@${ref}|g" if [[ "$OSTYPE" == darwin* ]]; then - # mac ships with a different version of sed that requires a delimiter arg - sed -i "" "$req_sed_pattern" $target_req_file + # mac ships with a different version of sed that requires a delimiter arg + sed -i "" "$req_sed_pattern" "$target_req_file" else - sed -i "$req_sed_pattern" $target_req_file + sed -i "$req_sed_pattern" "$target_req_file" fi