chore/moves cli to _workspace module (#3215)

* adds selective required context, checks profile support in switch_profile

* creates and tests hub module

* adds plugin version to telemetry

* renames imports in docs

* renames ci workflows

* fixes lint

* tests deploy command on duckdb

* moves cli module to workspace

* moves cli tests to workspace module

* renames fixtures, rewrites fixture to patch run context to _storage

* allows to patch global dir in workspace context

* when finding git repo, does not look up if GIT_CEILING_DIRECTORIES is set

* imports git utils only when need to clone package in dbt runner

* runs workspace tests as part of common

* fixes tests, config tests sideeffects

* moves dashboards to workspace

* fixes pipeline trace test

* moves dashboard helper tests

* excludes additional secret files and pinned profile from gitignore

* cleansup hatchling files in pyproject

* fixes dashboard running tests in ci

* moves git module to libs

* diff fix

* fixes fixture names

.github/workflows/test_common.yml

@@ -127,7 +127,7 @@ jobs:
 
       - name: Run workspace tests
         run: |
-          pytest tests/workspace tests/cli/common ${{ matrix.pytest_args }}
+          pytest tests/workspace ${{ matrix.pytest_args }}
         if: matrix.python-version != '3.14.0-beta.4'
 
       - name: Install pipeline and sources dependencies

.github/workflows/test_destinations_local.yml

@@ -33,13 +33,11 @@ jobs:
             post_install_commands: "uv run pip install sqlalchemy==2.0.18" # minimum version required by `pyiceberg`
 
           # TODO: also test ducklake in remote mode with a buckets and remote postgres
-          - name: postgres, duckdb, ducklake, and dummy with cli commands
+          - name: postgres, duckdb, ducklake, and dummy
             destinations: "[\"postgres\", \"duckdb\", \"ducklake\", \"dummy\"]"
             filesystem_drivers: "[\"memory\", \"file\"]"
             extras: "--group adbc --extra postgres --extra postgis --extra parquet --extra duckdb --extra cli --extra filesystem"
             needs_postgres: true
-            additional_tests: "pytest tests/cli"
-
 
           # Clickhouse OSS (TODO: test with minio s3)
           - name: clickhouse

.github/workflows/test_tools_dashboard.yml

@@ -102,18 +102,18 @@ jobs:
       # Run pipeline dashboard unit tests
       - name: Run pipeline dashboard unit tests
         run: |
-          pytest tests/helpers/dashboard
+          pytest tests/workspace/helpers/dashboard
 
       # Run pipeline dashboard e2e tests (does not pass with python 3.9, does not pass on windows (playwright does not work somehow), does not pass on python 3.13 (ibis not available))
       # Mac is also disabled for the time being
       - name: Run dashboard e2e
         run: |
-          marimo run --headless dlt/helpers/dashboard/dlt_dashboard.py -- -- --pipelines-dir _storage/.dlt/pipelines/ --with_test_identifiers true & pytest --browser chromium tests/e2e
+          marimo run --headless dlt/_workspace/helpers/dashboard/dlt_dashboard.py -- -- --pipelines-dir _storage/.dlt/pipelines/ --with_test_identifiers true & pytest --browser chromium tests/e2e
         if: matrix.python-version != '3.9' && matrix.python-version != '3.14.0-beta.4' && matrix.os != 'windows-latest'
 
       - name: Run dashboard e2e windows
         run: |
-          start marimo run --headless dlt/helpers/dashboard/dlt_dashboard.py -- -- --pipelines-dir _storage\.dlt\pipelines\ --with_test_identifiers true
+          start marimo run --headless dlt/_workspace/helpers/dashboard/dlt_dashboard.py -- -- --pipelines-dir _storage\.dlt\pipelines\ --with_test_identifiers true
           pytest --browser chromium tests/e2e
         if: matrix.python-version != '3.9' && matrix.python-version != '3.14.0-beta.4' && matrix.os == 'windows-latest'
   

Makefile

@@ -123,7 +123,7 @@ test-load-local-postgres:
 	DESTINATION__POSTGRES__CREDENTIALS=postgresql://loader:loader@localhost:5432/dlt_data ACTIVE_DESTINATIONS='["postgres"]' ALL_FILESYSTEM_DRIVERS='["memory"]'  uv run pytest tests/load
 
 test-common:
-	uv run pytest tests/common tests/normalize tests/extract tests/pipeline tests/reflection tests/sources tests/cli/common tests/load/test_dummy_client.py tests/libs tests/destinations
+	uv run pytest tests/common tests/normalize tests/extract tests/pipeline tests/reflection tests/sources tests/workspace tests/load/test_dummy_client.py tests/libs tests/destinations
 
 reset-test-storage:
 	-rm -r _storage
@@ -180,8 +180,8 @@ test-e2e-dashboard-headed:
 	uv run pytest --headed --browser chromium tests/e2e
 
 start-dlt-dashboard-e2e:
-	uv run marimo run --headless dlt/helpers/dashboard/dlt_dashboard.py -- -- --pipelines-dir _storage/.dlt/pipelines --with_test_identifiers true
+	uv run marimo run --headless dlt/_workspace/helpers/dashboard/dlt_dashboard.py -- -- --pipelines-dir _storage/.dlt/pipelines --with_test_identifiers true
 
 # creates the dashboard test pipelines globally for manual testing of the dashboard app and cli
 create-test-pipelines:
-	uv run python tests/helpers/dashboard/example_pipelines.py
+	uv run python tests/workspace/helpers/dashboard/example_pipelines.py

dlt/__main__.py

@@ -1,4 +1,4 @@
-from dlt.cli._dlt import main
+from dlt._workspace.cli._dlt import main
 
 if __name__ == "__main__":
     main()

dlt/_workspace/_templates/_single_file_templates/.gitignore

@@ -1,5 +1,8 @@
 # ignore secrets, virtual environments and typical python compilation artifacts
 secrets.toml
+*.secrets.toml
+# ignore pinned profile name
+.dlt/profile-name
 # ignore basic python artifacts
 .env
 **/__pycache__/

dlt/_workspace/_workspace_context.py

@@ -41,6 +41,7 @@ class WorkspaceRunContext(ProfilesRunContext):
         )
         # TODO: if local_dir == run_dir and profile "dev" profile prefixing for local_dir for OSS compat
         self._local_dir = default_working_dir(self.run_dir, name, profile, DEFAULT_LOCAL_FOLDER)
+        self._global_dir = global_dir()
 
     @property
     def name(self) -> str:
@@ -50,7 +51,7 @@ class WorkspaceRunContext(ProfilesRunContext):
     @property
     def global_dir(self) -> str:
         """Directory in which global settings are stored ie ~/.dlt/"""
-        return global_dir()
+        return self._global_dir
 
     @property
     def uri(self) -> str:

dlt/_workspace/cli/__init__.py

@@ -0,0 +1,14 @@
+from dlt.common.configuration.plugins import SupportsCliCommand
+
+from dlt._workspace.cli.exceptions import CliCommandException
+
+DEFAULT_VERIFIED_SOURCES_REPO = "https://github.com/dlt-hub/verified-sources.git"
+DEFAULT_VIBE_SOURCES_REPO = "https://github.com/dlt-hub/vibe-hub.git"
+
+
+__all__ = [
+    "SupportsCliCommand",
+    "CliCommandException",
+    "DEFAULT_VERIFIED_SOURCES_REPO",
+    "DEFAULT_VIBE_SOURCES_REPO",
+]

dlt/_workspace/cli/_ai_command.py

@@ -3,8 +3,8 @@ import shutil
 from pathlib import Path
 from typing import List, Tuple, get_args, Literal, Union, cast
 
-from dlt.cli import echo as fmt
-from dlt.common import git
+from dlt._workspace.cli import echo as fmt
+from dlt.common.libs import git
 from dlt.common.pipeline import get_dlt_repos_dir
 from dlt.common.runtime import run_context
 

dlt/_workspace/cli/_command_wrappers.py

@@ -8,35 +8,30 @@ from dlt.common.json import json
 from dlt.common.schema import Schema
 from dlt.common.typing import DictStrAny
 
-import dlt.cli.echo as fmt
-from dlt.cli import utils
 from dlt.pipeline.exceptions import CannotRestorePipelineException
-from dlt.cli.exceptions import CliCommandException
 
-from dlt.cli.init_command import (
+import dlt._workspace.cli.echo as fmt
+from dlt._workspace.cli import utils
+from dlt._workspace.cli.exceptions import CliCommandException, PipelineWasNotRun
+
+from dlt._workspace.cli._init_command import (
     init_command,
     list_sources_command,
     list_destinations_command,
-    DLT_INIT_DOCS_URL,
 )
-from dlt.cli.pipeline_command import pipeline_command, DLT_PIPELINE_COMMAND_DOCS_URL
-from dlt.cli.telemetry_command import (
-    DLT_TELEMETRY_DOCS_URL,
+from dlt._workspace.cli._pipeline_command import pipeline_command
+from dlt._workspace.cli._telemetry_command import (
     change_telemetry_status_command,
     telemetry_status_command,
+    DLT_TELEMETRY_DOCS_URL,
 )
-from dlt.cli.ai_command import ai_setup_command, TSupportedIde
+from dlt._workspace.cli._ai_command import ai_setup_command, TSupportedIde
 
 try:
-    from dlt.cli import deploy_command
-    from dlt.cli.deploy_command import (
-        PipelineWasNotRun,
-    )
+    from dlt._workspace.cli import _deploy_command
 except ModuleNotFoundError:
     pass
 
-DLT_DEPLOY_DOCS_URL = "https://dlthub.com/docs/walkthroughs/deploy-a-pipeline"
-
 
 @utils.track_command("init", False, "source_name", "destination_type")
 def init_command_wrapper(
@@ -96,7 +91,7 @@ def deploy_command_wrapper(
     from git import InvalidGitRepositoryError, NoSuchPathError
 
     try:
-        deploy_command.deploy_command(
+        _deploy_command.deploy_command(
             pipeline_script_path=pipeline_script_path,
             deployment_method=deployment_method,
             repo_location=repo_location,
@@ -124,7 +119,10 @@ def deploy_command_wrapper(
                 "https://docs.github.com/en/get-started/importing-your-projects-to-github/importing-source-code-to-github/adding-locally-hosted-code-to-github"
             )
         )
-        fmt.note("Please refer to %s for further assistance" % fmt.bold(DLT_DEPLOY_DOCS_URL))
+        fmt.note(
+            "Please refer to %s for further assistance"
+            % fmt.bold(_deploy_command.DLT_DEPLOY_DOCS_URL)
+        )
         raise CliCommandException(error_code=-4)
     except NoSuchPathError as path_ex:
         click.secho("The pipeline script does not exist\n%s" % str(path_ex), err=True, fg="red")
@@ -155,7 +153,7 @@ def schema_command_wrapper(file_path: str, format_: str, remove_defaults: bool)
 
 @utils.track_command("dashboard", True)
 def dashboard_command_wrapper(pipelines_dir: Optional[str], edit: bool) -> None:
-    from dlt.helpers.dashboard.runner import run_dashboard
+    from dlt._workspace.helpers.dashboard.runner import run_dashboard
 
     run_dashboard(pipelines_dir=pipelines_dir, edit=edit)
 

dlt/_workspace/cli/_deploy_command.py

@@ -4,14 +4,15 @@ import yaml
 from enum import Enum
 from importlib.metadata import version as pkg_version
 
+from dlt.version import DLT_PKG_NAME
 from dlt.common.configuration.providers import SECRETS_TOML, SECRETS_TOML_KEY
 from dlt.common.configuration.utils import serialize_value
-from dlt.common.git import is_dirty
+from dlt.common.destination.reference import Destination
+from dlt.common.libs.git import is_dirty
 
-from dlt.cli import utils
-from dlt.cli import echo as fmt
-from dlt.cli.deploy_command_helpers import (
-    PipelineWasNotRun,
+from dlt._workspace.cli import utils
+from dlt._workspace.cli import echo as fmt
+from dlt._workspace.cli._deploy_command_helpers import (
     BaseDeployment,
     ask_files_overwrite,
     generate_pip_freeze,
@@ -21,10 +22,6 @@ from dlt.cli.deploy_command_helpers import (
     get_schedule_description,
 )
 
-from dlt.version import DLT_PKG_NAME
-
-from dlt.common.destination.reference import Destination
-
 REQUIREMENTS_GITHUB_ACTION = "requirements_github_action.txt"
 DLT_AIRFLOW_GCP_DOCS_URL = (
     "https://dlthub.com/docs/walkthroughs/deploy-a-pipeline/deploy-with-airflow-composer"
@@ -34,6 +31,7 @@ AIRFLOW_DAG_TEMPLATE_SCRIPT = "dag_template.py"
 AIRFLOW_CLOUDBUILD_YAML = "cloudbuild.yaml"
 COMMAND_REPO_LOCATION = "https://github.com/dlt-hub/dlt-%s-template.git"
 COMMAND_DEPLOY_REPO_LOCATION = COMMAND_REPO_LOCATION % "deploy"
+DLT_DEPLOY_DOCS_URL = "https://dlthub.com/docs/walkthroughs/deploy-a-pipeline"
 
 
 class DeploymentMethods(Enum):

dlt/_workspace/cli/_deploy_command_helpers.py

@@ -21,14 +21,14 @@ except ImportError:
 
 import dlt
 
-from dlt.common import git
+from dlt.common.libs import git
 from dlt.common.configuration.exceptions import LookupTrace, ConfigFieldMissingException
 from dlt.common.configuration.providers import (
     CONFIG_TOML,
     EnvironProvider,
     StringTomlProvider,
 )
-from dlt.common.git import get_origin, get_repo, Repo
+from dlt.common.libs.git import get_origin, get_repo, Repo
 from dlt.common.configuration.specs.runtime_configuration import get_default_pipeline_name
 from dlt.common.typing import StrAny
 from dlt.common.reflection.utils import evaluate_node_literal
@@ -41,9 +41,9 @@ from dlt.pipeline.trace import PipelineTrace
 from dlt.reflection import names as n
 from dlt.reflection.script_visitor import PipelineScriptVisitor
 
-from dlt.cli import utils
-from dlt.cli import echo as fmt
-from dlt.cli.exceptions import CliCommandInnerException
+from dlt._workspace.cli import utils
+from dlt._workspace.cli import echo as fmt
+from dlt._workspace.cli.exceptions import CliCommandInnerException, PipelineWasNotRun
 
 GITHUB_URL = "https://github.com/"
 
@@ -447,8 +447,3 @@ def ask_files_overwrite(files: Sequence[str]) -> None:
         fmt.echo("Following files will be overwritten: %s" % fmt.bold(str(existing)))
         if not fmt.confirm("Do you want to continue?", default=False):
             raise CliCommandInnerException("init", "Aborted")
-
-
-class PipelineWasNotRun(CliCommandInnerException):
-    def __init__(self, msg: str) -> None:
-        super().__init__("deploy", msg, None)

dlt/_workspace/cli/_dlt.py

@@ -6,17 +6,15 @@ from rich.markdown import Markdown
 
 from dlt.version import __version__
 from dlt.common.runners import Venv
-from dlt.cli import SupportsCliCommand
+from dlt._workspace.cli import SupportsCliCommand
 
-import dlt.cli.echo as fmt
-from dlt.cli.exceptions import CliCommandException
-
-from dlt.cli.command_wrappers import (
+import dlt._workspace.cli.echo as fmt
+from dlt._workspace.cli.exceptions import CliCommandException
+from dlt._workspace.cli._command_wrappers import (
     telemetry_change_status_command_wrapper,
 )
-from dlt.cli import debug
-from dlt.cli.echo import maybe_no_stdin
-
+from dlt._workspace.cli import _debug
+from dlt._workspace.cli.echo import maybe_no_stdin
 from dlt._workspace.cli.utils import display_run_context_info
 
 ACTION_EXECUTED = False
@@ -99,7 +97,7 @@ class DebugAction(argparse.Action):
         option_string: str = None,
     ) -> None:
         # will show stack traces (and maybe more debug things)
-        debug.enable_debug()
+        _debug.enable_debug()
 
 
 def _create_parser() -> Tuple[argparse.ArgumentParser, Dict[str, SupportsCliCommand]]:
@@ -211,7 +209,7 @@ def main() -> int:
                 click.secho(str(ex), err=True, fg="red")
 
             fmt.note("Please refer to our docs at '%s' for further assistance." % docs_url)
-            if debug.is_debug_enabled() and raiseable_exception:
+            if _debug.is_debug_enabled() and raiseable_exception:
                 raise raiseable_exception
 
             return error_code

dlt/_workspace/cli/_docs_command.py

@@ -5,7 +5,7 @@ import textwrap
 import os
 import re
 
-import dlt.cli.echo as fmt
+import dlt._workspace.cli.echo as fmt
 
 HEADER = """---
 title: Command Line Interface

dlt/_workspace/cli/_init_command.py

@@ -6,7 +6,7 @@ from pathlib import Path
 
 
 import dlt.destinations
-from dlt.common import git
+from dlt.common.libs import git
 from dlt.common.configuration.specs import known_sections
 from dlt.common.configuration.providers import (
     SECRETS_TOML,
@@ -27,12 +27,9 @@ from dlt.sources import SourceReference
 import dlt.reflection.names as n
 from dlt.reflection.script_inspector import import_pipeline_script
 
-from dlt.cli import echo as fmt, pipeline_files as files_ops, source_detection, utils
-
-# keep it for backward compat
-from dlt.cli import DEFAULT_VERIFIED_SOURCES_REPO
-from dlt.cli.config_toml_writer import WritableConfigValue, write_values
-from dlt.cli.pipeline_files import (
+from dlt._workspace.cli import echo as fmt, _pipeline_files as files_ops, source_detection, utils
+from dlt._workspace.cli.config_toml_writer import WritableConfigValue, write_values
+from dlt._workspace.cli._pipeline_files import (
     TEMPLATE_FILES,
     SOURCES_MODULE_NAME,
     SINGLE_FILE_TEMPLATE_MODULE_NAME,
@@ -40,8 +37,8 @@ from dlt.cli.pipeline_files import (
     TVerifiedSourceFileEntry,
     TVerifiedSourceFileIndex,
 )
-from dlt.cli.exceptions import CliCommandInnerException
-from dlt.cli.ai_command import SUPPORTED_IDES, TSupportedIde
+from dlt._workspace.cli.exceptions import CliCommandInnerException
+from dlt._workspace.cli._ai_command import SUPPORTED_IDES, TSupportedIde
 
 
 DLT_INIT_DOCS_URL = "https://dlthub.com/docs/reference/command-line-interface#dlt-init"
@@ -118,8 +115,8 @@ def init_command(
         sources_dir,
     )
     if is_dlthub_source and copied_files is not None and selected_ide:
-        from dlt.cli import DEFAULT_VIBE_SOURCES_REPO, DEFAULT_VERIFIED_SOURCES_REPO
-        from dlt.cli.ai_command import ai_setup_command, vibe_source_setup
+        from dlt._workspace.cli import DEFAULT_VIBE_SOURCES_REPO, DEFAULT_VERIFIED_SOURCES_REPO
+        from dlt._workspace.cli._ai_command import ai_setup_command, vibe_source_setup
 
         fmt.echo()
         fmt.echo()
@@ -187,8 +184,8 @@ def init_pipeline_at_destination(
         destination_spec = destination_reference.spec
 
     # lookup core storages
-    core_sources_storage = _get_core_sources_storage()
-    templates_storage = _get_templates_storage()
+    core_sources_storage = files_ops.get_core_sources_storage()
+    templates_storage = files_ops.get_single_file_templates_storage()
 
     # discover type of source
     source_type: files_ops.TSourceType = "template"
@@ -348,12 +345,20 @@ def init_pipeline_at_destination(
         source_configuration.src_pipeline_script,
     )
 
-    # inspect the script
-    import_pipeline_script(
-        source_configuration.storage.storage_path,
-        source_configuration.storage.to_relative_path(source_configuration.src_pipeline_script),
-        ignore_missing_imports=True,
-    )
+    # inspect the script to populate source references
+    if source_configuration.source_type != "core":
+        import_pipeline_script(
+            source_configuration.storage.storage_path,
+            source_configuration.storage.to_relative_path(source_configuration.src_pipeline_script),
+            ignore_missing_imports=True,
+        )
+    else:
+        # core sources are imported directly from the pipeline script which is in the _workspace module
+        import_pipeline_script(
+            os.path.dirname(source_configuration.src_pipeline_script),
+            os.path.basename(source_configuration.src_pipeline_script),
+            ignore_missing_imports=True,
+        )
 
     # detect all the required secrets and configs that should go into tomls files
     if source_configuration.source_type == "template":
@@ -588,23 +593,6 @@ def _get_source_display_name(source_name: str) -> Tuple[bool, str, str]:
     return is_vibe_source, display_source_name, source_name
 
 
-def _get_core_sources_storage() -> FileStorage:
-    """Get FileStorage for core sources"""
-    local_path = Path(os.path.dirname(os.path.realpath(__file__))).parent / SOURCES_MODULE_NAME
-    return FileStorage(str(local_path))
-
-
-def _get_templates_storage() -> FileStorage:
-    """Get FileStorage for single file templates"""
-    # look up init storage in core
-    init_path = (
-        Path(os.path.dirname(os.path.realpath(__file__))).parent
-        / SOURCES_MODULE_NAME
-        / SINGLE_FILE_TEMPLATE_MODULE_NAME
-    )
-    return FileStorage(str(init_path))
-
-
 def _clone_and_get_verified_sources_storage(repo_location: str, branch: str = None) -> FileStorage:
     """Clone and get FileStorage for verified sources templates"""
 
@@ -772,7 +760,7 @@ def _get_dependency_system(dest_storage: FileStorage) -> str:
 
 
 def _list_template_sources() -> Dict[str, SourceConfiguration]:
-    template_storage = _get_templates_storage()
+    template_storage = files_ops.get_single_file_templates_storage()
     sources: Dict[str, SourceConfiguration] = {}
     for source_name in files_ops.get_sources_names(template_storage, source_type="template"):
         sources[source_name] = files_ops.get_template_configuration(
@@ -782,7 +770,7 @@ def _list_template_sources() -> Dict[str, SourceConfiguration]:
 
 
 def _list_core_sources() -> Dict[str, SourceConfiguration]:
-    core_sources_storage = _get_core_sources_storage()
+    core_sources_storage = files_ops.get_core_sources_storage()
 
     sources: Dict[str, SourceConfiguration] = {}
     for source_name in files_ops.get_sources_names(core_sources_storage, source_type="core"):

dlt/_workspace/cli/_pipeline_command.py

@@ -2,7 +2,7 @@ import os
 import yaml
 from typing import Any, Sequence, Tuple
 import dlt
-from dlt.cli.exceptions import CliCommandInnerException
+from dlt._workspace.cli.exceptions import CliCommandInnerException
 
 from dlt.common.json import json
 from dlt.common.pipeline import get_dlt_pipelines_dir, TSourceState
@@ -22,7 +22,7 @@ from dlt.extract.state import resource_state
 from dlt.pipeline.helpers import pipeline_drop
 from dlt.pipeline.exceptions import CannotRestorePipelineException
 
-from dlt.cli import echo as fmt
+from dlt._workspace.cli import echo as fmt
 
 
 DLT_PIPELINE_COMMAND_DOCS_URL = (
@@ -57,7 +57,7 @@ def pipeline_command(
 
     # we may open the dashboard for a pipeline without checking if it exists
     if operation == "show" and not command_kwargs.get("streamlit"):
-        from dlt.helpers.dashboard.runner import run_dashboard
+        from dlt._workspace.helpers.dashboard.runner import run_dashboard
 
         run_dashboard(pipeline_name, edit=command_kwargs.get("edit"), pipelines_dir=pipelines_dir)
         # return so streamlit does not run
@@ -147,7 +147,13 @@ def pipeline_command(
             streamlit_cmd = [
                 "streamlit",
                 "run",
-                os.path.join(os.path.dirname(dlt.__file__), "helpers", "streamlit_app", "index.py"),
+                os.path.join(
+                    os.path.dirname(dlt.__file__),
+                    "_workspace",
+                    "helpers",
+                    "streamlit_app",
+                    "index.py",
+                ),
                 "--client.showSidebarNavigation",
                 "false",
             ]

dlt/_workspace/cli/_pipeline_files.py

@@ -5,22 +5,23 @@ import yaml
 import posixpath
 from pathlib import Path
 from typing import Dict, NamedTuple, Sequence, Tuple, List, Literal
-from dlt.cli.exceptions import VerifiedSourceRepoError
+from dlt._workspace.cli.exceptions import VerifiedSourceRepoError
 
-from dlt.common import git
+from dlt.common.libs import git
 from dlt.common.storages import FileStorage
 from dlt.common.typing import TypedDict
 
 from dlt.common.reflection.utils import get_module_docstring
 
-from dlt.cli import utils
-from dlt.cli.requirements import SourceRequirements
+from dlt._workspace.cli import utils
+from dlt._workspace.cli.requirements import SourceRequirements
 
 TSourceType = Literal["core", "verified", "template", "vibe"]
 
 SOURCES_INIT_INFO_ENGINE_VERSION = 1
 
 SOURCES_MODULE_NAME = "sources"
+TEMPLATES_MODULE_NAME = "_templates"
 CORE_SOURCE_TEMPLATE_MODULE_NAME = "_core_source_templates"
 SINGLE_FILE_TEMPLATE_MODULE_NAME = "_single_file_templates"
 
@@ -256,7 +257,11 @@ def _get_source_files(sources_storage: FileStorage, source_name: str) -> List[st
 def get_core_source_configuration(
     sources_storage: FileStorage, source_name: str, eject_source: bool
 ) -> SourceConfiguration:
-    src_pipeline_file = CORE_SOURCE_TEMPLATE_MODULE_NAME + "/" + source_name + PIPELINE_FILE_SUFFIX
+    src_pipeline_file_root = Path(os.path.dirname(os.path.realpath(__file__))).parent
+
+    src_pipeline_file = src_pipeline_file_root.joinpath(
+        TEMPLATES_MODULE_NAME, CORE_SOURCE_TEMPLATE_MODULE_NAME, source_name + PIPELINE_FILE_SUFFIX
+    )
     dest_pipeline_file = source_name + PIPELINE_FILE_SUFFIX
     files: List[str] = _get_source_files(sources_storage, source_name) if eject_source else []
 
@@ -264,7 +269,7 @@ def get_core_source_configuration(
         "core",
         "dlt.sources." + source_name,
         sources_storage,
-        src_pipeline_file,
+        str(src_pipeline_file),
         dest_pipeline_file,
         files,
         SourceRequirements([]),
@@ -308,6 +313,25 @@ def get_verified_source_configuration(
     )
 
 
+def get_core_sources_storage() -> FileStorage:
+    """Get FileStorage for core sources"""
+    local_path = (
+        Path(os.path.dirname(os.path.realpath(__file__))).parent.parent / SOURCES_MODULE_NAME
+    )
+    return FileStorage(str(local_path))
+
+
+def get_single_file_templates_storage() -> FileStorage:
+    """Get FileStorage for single file templates"""
+    # look up init storage in core
+    init_path = (
+        Path(os.path.dirname(os.path.realpath(__file__))).parent
+        / TEMPLATES_MODULE_NAME
+        / SINGLE_FILE_TEMPLATE_MODULE_NAME
+    )
+    return FileStorage(str(init_path))
+
+
 def gen_index_diff(
     local_index: TVerifiedSourceFileIndex, remote_index: TVerifiedSourceFileIndex
 ) -> Tuple[

dlt/_workspace/cli/_plugins.py

@@ -20,56 +20,56 @@ __all__ = [
 
 @plugins.hookimpl(specname="plug_cli")
 def plug_cli_init() -> Type[plugins.SupportsCliCommand]:
-    from dlt.cli.commands import InitCommand
+    from dlt._workspace.cli.commands import InitCommand
 
     return InitCommand
 
 
 @plugins.hookimpl(specname="plug_cli")
 def plug_cli_pipeline() -> Type[plugins.SupportsCliCommand]:
-    from dlt.cli.commands import PipelineCommand
+    from dlt._workspace.cli.commands import PipelineCommand
 
     return PipelineCommand
 
 
 @plugins.hookimpl(specname="plug_cli")
 def plug_cli_schema() -> Type[plugins.SupportsCliCommand]:
-    from dlt.cli.commands import SchemaCommand
+    from dlt._workspace.cli.commands import SchemaCommand
 
     return SchemaCommand
 
 
 @plugins.hookimpl(specname="plug_cli")
 def plug_cli_dashboard() -> Type[plugins.SupportsCliCommand]:
-    from dlt.cli.commands import DashboardCommand
+    from dlt._workspace.cli.commands import DashboardCommand
 
     return DashboardCommand
 
 
 @plugins.hookimpl(specname="plug_cli")
 def plug_cli_telemetry() -> Type[plugins.SupportsCliCommand]:
-    from dlt.cli.commands import TelemetryCommand
+    from dlt._workspace.cli.commands import TelemetryCommand
 
     return TelemetryCommand
 
 
 @plugins.hookimpl(specname="plug_cli")
 def plug_cli_deploy() -> Type[plugins.SupportsCliCommand]:
-    from dlt.cli.commands import DeployCommand
+    from dlt._workspace.cli.commands import DeployCommand
 
     return DeployCommand
 
 
 @plugins.hookimpl(specname="plug_cli")
 def plug_cli_docs() -> Type[plugins.SupportsCliCommand]:
-    from dlt.cli.commands import CliDocsCommand
+    from dlt._workspace.cli.commands import CliDocsCommand
 
     return CliDocsCommand
 
 
 @plugins.hookimpl(specname="plug_cli")
 def plug_cli_ai() -> Type[plugins.SupportsCliCommand]:
-    from dlt.cli.commands import AiCommand
+    from dlt._workspace.cli.commands import AiCommand
 
     return AiCommand
 

dlt/_workspace/cli/_profile_command.py

@@ -8,7 +8,7 @@ from dlt._workspace.profile import (
     read_profile_pin,
     save_profile_pin,
 )
-from dlt.cli import SupportsCliCommand, echo as fmt
+from dlt._workspace.cli import SupportsCliCommand, echo as fmt
 
 
 class ProfileCommand(SupportsCliCommand):

dlt/_workspace/cli/_telemetry_command.py

@@ -4,9 +4,9 @@ from dlt.common.configuration.container import Container
 from dlt.common.configuration.providers.toml import ConfigTomlProvider
 from dlt.common.configuration.specs import RuntimeConfiguration
 
-from dlt.cli import echo as fmt
-from dlt.cli.utils import get_telemetry_status
-from dlt.cli.config_toml_writer import WritableConfigValue, write_values
+from dlt._workspace.cli import echo as fmt
+from dlt._workspace.cli.utils import get_telemetry_status
+from dlt._workspace.cli.config_toml_writer import WritableConfigValue, write_values
 from dlt.common.configuration.specs import PluggableRunContext
 from dlt.common.runtime.anon_tracker import get_anonymous_id
 

dlt/_workspace/cli/_workspace_command.py

@@ -1,14 +1,12 @@
 import argparse
 
-from dlt.common import logger
 from dlt.common.configuration.container import Container
 from dlt.common.configuration.specs.pluggable_run_context import (
     PluggableRunContext,
     RunContextBase,
 )
 
-from dlt.cli import SupportsCliCommand, echo as fmt, utils
-
+from dlt._workspace.cli import SupportsCliCommand, echo as fmt, utils
 from dlt._workspace._workspace_context import WorkspaceRunContext, active
 from dlt._workspace.cli.utils import add_mcp_arg_parser, delete_local_data
 from dlt._workspace.profile import read_profile_pin

dlt/_workspace/cli/commands.py

@@ -3,10 +3,10 @@ import argparse
 
 from dlt.common.configuration import plugins
 
-import dlt.cli.echo as fmt
-from dlt.cli import SupportsCliCommand, DEFAULT_VERIFIED_SOURCES_REPO
-from dlt.cli.exceptions import CliCommandException
-from dlt.cli.command_wrappers import (
+import dlt._workspace.cli.echo as fmt
+from dlt._workspace.cli import SupportsCliCommand, DEFAULT_VERIFIED_SOURCES_REPO
+from dlt._workspace.cli.exceptions import CliCommandException
+from dlt._workspace.cli._command_wrappers import (
     init_command_wrapper,
     list_sources_command_wrapper,
     list_destinations_command_wrapper,
@@ -17,20 +17,17 @@ from dlt.cli.command_wrappers import (
     ai_setup_command_wrapper,
     dashboard_command_wrapper,
 )
-from dlt.cli.ai_command import SUPPORTED_IDES
-from dlt.cli.docs_command import render_argparse_markdown
-from dlt.cli.command_wrappers import (
-    DLT_PIPELINE_COMMAND_DOCS_URL,
-    DLT_INIT_DOCS_URL,
-    DLT_TELEMETRY_DOCS_URL,
-    DLT_DEPLOY_DOCS_URL,
-)
+from dlt._workspace.cli._ai_command import SUPPORTED_IDES
+from dlt._workspace.cli._docs_command import render_argparse_markdown
+from dlt._workspace.cli._pipeline_command import DLT_PIPELINE_COMMAND_DOCS_URL
+from dlt._workspace.cli._init_command import DLT_INIT_DOCS_URL
+from dlt._workspace.cli._telemetry_command import DLT_TELEMETRY_DOCS_URL
 from dlt._workspace.cli.utils import add_mcp_arg_parser
-
-from dlt.cli.deploy_command import (
+from dlt._workspace.cli._deploy_command import (
     DeploymentMethods,
     COMMAND_DEPLOY_REPO_LOCATION,
     SecretFormats,
+    DLT_DEPLOY_DOCS_URL,
 )
 from dlt.common.storages.configuration import SCHEMA_FILES_EXTENSIONS
 
@@ -669,7 +666,7 @@ If you are reading this on the docs website, you are looking at the rendered ver
         )
 
     def execute(self, args: argparse.Namespace) -> None:
-        from dlt.cli._dlt import _create_parser
+        from dlt._workspace.cli._dlt import _create_parser
 
         parser, _ = _create_parser()
 

dlt/_workspace/cli/config_toml_writer.py

@@ -2,7 +2,6 @@ from typing import Any, NamedTuple, Tuple, Iterable, Mapping
 import tomlkit
 from tomlkit.items import Table as TOMLTable
 from tomlkit.container import Container as TOMLContainer
-from collections.abc import Sequence as C_Sequence
 
 from dlt.common.configuration.specs.base_configuration import is_hint_not_resolvable
 from dlt.common.configuration.const import TYPE_EXAMPLES

dlt/_workspace/cli/exceptions.py

@@ -26,3 +26,8 @@ class VerifiedSourceRepoError(DltException):
     def __init__(self, msg: str, source_name: str) -> None:
         self.source_name = source_name
         super().__init__(msg)
+
+
+class PipelineWasNotRun(CliCommandInnerException):
+    def __init__(self, msg: str) -> None:
+        super().__init__("deploy", msg, None)

dlt/_workspace/cli/source_detection.py

@@ -8,8 +8,8 @@ from dlt.common.reflection.utils import creates_func_def_name_node
 from dlt.common.typing import is_optional_type
 
 from dlt.sources import SourceReference
-from dlt.cli.config_toml_writer import WritableConfigValue
-from dlt.cli.exceptions import CliCommandInnerException
+from dlt._workspace.cli.config_toml_writer import WritableConfigValue
+from dlt._workspace.cli.exceptions import CliCommandInnerException
 from dlt.reflection.script_visitor import PipelineScriptVisitor
 
 

dlt/_workspace/cli/utils.py

@@ -1,17 +1,34 @@
+import ast
 import os
 import shutil
-from typing import Any
+from typing import Any, Callable
 
 import dlt
-from dlt._workspace.profile import LOCAL_PROFILES
-from dlt.cli.exceptions import CliCommandException
+from dlt.common.typing import TFun
+from dlt.common.configuration.resolve import resolve_configuration
 from dlt.common.configuration.specs.pluggable_run_context import (
     RunContextBase,
     ProfilesRunContext,
 )
+from dlt.common.configuration.specs.runtime_configuration import RuntimeConfiguration
+from dlt.common.reflection.utils import set_ast_parents
+from dlt.common.runtime import run_context
+from dlt.common.runtime.telemetry import with_telemetry
 from dlt.common.storages.file_storage import FileStorage
 
-from dlt.cli import echo as fmt
+from dlt._workspace.profile import LOCAL_PROFILES
+from dlt._workspace.cli.exceptions import CliCommandException, CliCommandInnerException
+from dlt._workspace.cli import echo as fmt
+
+from dlt.reflection.script_visitor import PipelineScriptVisitor
+
+REQUIREMENTS_TXT = "requirements.txt"
+PYPROJECT_TOML = "pyproject.toml"
+GITHUB_WORKFLOWS_DIR = os.path.join(".github", "workflows")
+AIRFLOW_DAGS_FOLDER = os.path.join("dags")
+AIRFLOW_BUILD_FOLDER = os.path.join("build")
+LOCAL_COMMAND_REPO_FOLDER = "repos"
+MODULE_INIT = "__init__.py"
 
 
 def display_run_context_info() -> None:
@@ -119,3 +136,52 @@ def delete_local_data(
             "Will delete pipeline working folders & other entities data %s",
             recreate_dirs,
         )
+
+
+def parse_init_script(
+    command: str, script_source: str, init_script_name: str
+) -> PipelineScriptVisitor:
+    # parse the script first
+    tree = ast.parse(source=script_source)
+    set_ast_parents(tree)
+    visitor = PipelineScriptVisitor(script_source)
+    visitor.visit_passes(tree)
+    if len(visitor.mod_aliases) == 0:
+        raise CliCommandInnerException(
+            command,
+            f"The pipeline script {init_script_name} does not import dlt and does not seem to run"
+            " any pipelines",
+        )
+
+    return visitor
+
+
+def ensure_git_command(command: str) -> None:
+    try:
+        import git
+    except ImportError as imp_ex:
+        if "Bad git executable" not in str(imp_ex):
+            raise
+        raise CliCommandInnerException(
+            command,
+            "'git' command is not available. Install and setup git with the following the guide %s"
+            % "https://docs.github.com/en/get-started/quickstart/set-up-git",
+            imp_ex,
+        ) from imp_ex
+
+
+def track_command(command: str, track_before: bool, *args: str) -> Callable[[TFun], TFun]:
+    return with_telemetry("command", command, track_before, *args)
+
+
+def get_telemetry_status() -> bool:
+    c = resolve_configuration(RuntimeConfiguration())
+    return c.dlthub_telemetry
+
+
+def make_dlt_settings_path(path: str = None) -> str:
+    """Returns path to file in dlt settings folder. Returns settings folder if path not specified."""
+    ctx = run_context.active()
+    if not path:
+        return ctx.settings_dir
+    return ctx.get_setting(path)

dlt/_workspace/helpers/dashboard/dlt_dashboard.py

@@ -15,8 +15,8 @@ with app.setup:
 
     import dlt
     import pyarrow
-    from dlt.helpers.dashboard import strings, utils, ui_elements as ui
-    from dlt.helpers.dashboard.config import DashboardConfiguration
+    from dlt._workspace.helpers.dashboard import strings, utils, ui_elements as ui
+    from dlt._workspace.helpers.dashboard.config import DashboardConfiguration
 
 
 @app.cell(hide_code=True)

dlt/_workspace/helpers/dashboard/runner.py

@@ -40,7 +40,7 @@ def run_dashboard(
     port: int = None,
     host: str = None,
 ) -> None:
-    from dlt.helpers.dashboard import dlt_dashboard
+    from dlt._workspace.helpers.dashboard import dlt_dashboard
 
     ejected_app_path = os.path.join(os.getcwd(), EJECTED_APP_FILE_NAME)
     ejected_css_path = os.path.join(os.getcwd(), STYLE_FILE_NAME)
@@ -52,7 +52,7 @@ def run_dashboard(
             app_code = f.read()
         with open(ejected_app_path, "w", encoding="utf-8") as f:
             f.write(app_code)
-        css_file_path = Path(files("dlt.helpers.dashboard") / STYLE_FILE_NAME)  # type: ignore
+        css_file_path = Path(files("dlt._workspace.helpers.dashboard") / STYLE_FILE_NAME)  # type: ignore
         with open(css_file_path, "r", encoding="utf-8") as f:
             css_content = f.read()
         with open(os.path.join(os.getcwd(), ejected_css_path), "w", encoding="utf-8") as f:

dlt/_workspace/helpers/dashboard/utils.py

@@ -25,12 +25,11 @@ from dlt.common.schema.typing import TTableSchema
 from dlt.common.storages import FileStorage
 from dlt.common.destination.client import DestinationClientConfiguration
 from dlt.common.configuration.exceptions import ConfigFieldMissingException
-from dlt.destinations.dataset import ReadableDBAPIDataset, ReadableDBAPIRelation
 from dlt.common.typing import DictStrAny
 from dlt.common.utils import map_nested_keys_in_place
 
-from dlt.helpers.dashboard import ui_elements as ui
-from dlt.helpers.dashboard.config import DashboardConfiguration
+from dlt._workspace.helpers.dashboard import ui_elements as ui
+from dlt._workspace.helpers.dashboard.config import DashboardConfiguration
 from dlt.destinations.exceptions import DatabaseUndefinedRelation, DestinationUndefinedEntity
 from dlt.pipeline.exceptions import PipelineConfigMissing
 from dlt.pipeline.exceptions import CannotRestorePipelineException

dlt/_workspace/helpers/streamlit_app/blocks/load_info.py

@@ -3,8 +3,8 @@ import humanize
 import streamlit as st
 
 from dlt.common.pendulum import pendulum
-from dlt.helpers.streamlit_app.utils import query_data_live
-from dlt.helpers.streamlit_app.widgets import stat
+from dlt._workspace.helpers.streamlit_app.utils import query_data_live
+from dlt._workspace.helpers.streamlit_app.widgets import stat
 
 
 def last_load_info(pipeline: dlt.Pipeline) -> None:

dlt/_workspace/helpers/streamlit_app/blocks/menu.py

@@ -1,9 +1,9 @@
 import dlt
 import streamlit as st
 
-from dlt.helpers.streamlit_app.utils import HERE
-from dlt.helpers.streamlit_app.widgets import mode_selector
-from dlt.helpers.streamlit_app.widgets import pipeline_summary
+from dlt._workspace.helpers.streamlit_app.utils import HERE
+from dlt._workspace.helpers.streamlit_app.widgets import mode_selector
+from dlt._workspace.helpers.streamlit_app.widgets import pipeline_summary
 
 
 def menu(pipeline: dlt.Pipeline) -> None:

dlt/_workspace/helpers/streamlit_app/blocks/query.py

@@ -3,7 +3,7 @@ import dlt
 import streamlit as st
 
 from dlt.common.exceptions import MissingDependencyException
-from dlt.helpers.streamlit_app.utils import query_data
+from dlt._workspace.helpers.streamlit_app.utils import query_data
 
 
 def maybe_run_query(

dlt/_workspace/helpers/streamlit_app/blocks/show_data.py

@@ -1,7 +1,7 @@
 import dlt
 import streamlit as st
 
-from dlt.helpers.streamlit_app.utils import query_data
+from dlt._workspace.helpers.streamlit_app.utils import query_data
 
 
 def show_data_button(pipeline: dlt.Pipeline, table_name: str) -> None:

dlt/_workspace/helpers/streamlit_app/blocks/table_hints.py

@@ -5,8 +5,8 @@ import streamlit as st
 
 from dlt.common.schema.typing import TTableSchema, TColumnSchema
 from dlt.common.utils import flatten_list_or_items
-from dlt.helpers.streamlit_app.blocks.resource_state import resource_state_info
-from dlt.helpers.streamlit_app.blocks.show_data import show_data_button
+from dlt._workspace.helpers.streamlit_app.blocks.resource_state import resource_state_info
+from dlt._workspace.helpers.streamlit_app.blocks.show_data import show_data_button
 
 
 def list_table_hints(pipeline: dlt.Pipeline, tables: List[TTableSchema]) -> None:

dlt/_workspace/helpers/streamlit_app/index.py

@@ -1,6 +1,6 @@
 import streamlit as st
 
-from dlt.helpers.streamlit_app.utils import HERE
+from dlt._workspace.helpers.streamlit_app.utils import HERE
 
 if __name__ == "__main__":
     st.switch_page(f"{HERE}/pages/dashboard.py")

dlt/_workspace/helpers/streamlit_app/pages/dashboard.py

@@ -1,11 +1,11 @@
 import dlt
 import streamlit as st
 
-from dlt.helpers.streamlit_app.blocks.query import maybe_run_query
-from dlt.helpers.streamlit_app.blocks.table_hints import list_table_hints
-from dlt.helpers.streamlit_app.blocks.menu import menu
-from dlt.helpers.streamlit_app.utils import render_with_pipeline
-from dlt.helpers.streamlit_app.widgets import schema_picker
+from dlt._workspace.helpers.streamlit_app.blocks.query import maybe_run_query
+from dlt._workspace.helpers.streamlit_app.blocks.table_hints import list_table_hints
+from dlt._workspace.helpers.streamlit_app.blocks.menu import menu
+from dlt._workspace.helpers.streamlit_app.utils import render_with_pipeline
+from dlt._workspace.helpers.streamlit_app.widgets import schema_picker
 from dlt.pipeline import Pipeline
 
 

dlt/_workspace/helpers/streamlit_app/pages/load_info.py

@@ -3,10 +3,10 @@ import streamlit as st
 
 from dlt.common.configuration.exceptions import ConfigFieldMissingException
 from dlt.common.destination.client import WithStateSync
-from dlt.helpers.streamlit_app.blocks.load_info import last_load_info
-from dlt.helpers.streamlit_app.blocks.menu import menu
-from dlt.helpers.streamlit_app.widgets import stat
-from dlt.helpers.streamlit_app.utils import (
+from dlt._workspace.helpers.streamlit_app.blocks.load_info import last_load_info
+from dlt._workspace.helpers.streamlit_app.blocks.menu import menu
+from dlt._workspace.helpers.streamlit_app.widgets import stat
+from dlt._workspace.helpers.streamlit_app.utils import (
     query_data,
     query_data_live,
     render_with_pipeline,

dlt/_workspace/helpers/streamlit_app/utils.py

@@ -8,7 +8,7 @@ import dlt
 import pandas as pd
 import streamlit as st
 
-from dlt.cli import echo as fmt
+from dlt._workspace.cli import echo as fmt
 from dlt.common.destination.exceptions import SqlClientNotAvailable
 
 HERE = Path(__file__).absolute().parent

dlt/_workspace/helpers/streamlit_app/widgets/__init__.py

@@ -0,0 +1,5 @@
+from dlt._workspace.helpers.streamlit_app.widgets.stats import stat
+from dlt._workspace.helpers.streamlit_app.widgets.summary import pipeline_summary
+from dlt._workspace.helpers.streamlit_app.widgets.tags import tag
+from dlt._workspace.helpers.streamlit_app.widgets.schema import schema_picker
+from dlt._workspace.helpers.streamlit_app.widgets.color_mode_selector import mode_selector

dlt/_workspace/helpers/streamlit_app/widgets/color_mode_selector.py

@@ -2,7 +2,7 @@ import streamlit as st
 
 from typing_extensions import Callable, Literal
 
-from dlt.helpers.streamlit_app.theme import dark_theme, light_theme
+from dlt._workspace.helpers.streamlit_app.theme import dark_theme, light_theme
 
 ColorMode = Literal["light", "dark"]
 

dlt/_workspace/mcp/tools/helpers.py

@@ -2,11 +2,6 @@ import pyarrow as pa
 import pyarrow.csv as pacsv
 import pyarrow.types as patypes
 import unicodedata
-from dlt.cli.init_command import (
-    _list_verified_sources,
-    DEFAULT_VERIFIED_SOURCES_REPO,
-)
-from dlt.cli.echo import suppress_echo
 
 
 def format_csv(table: pa.Table, info: str = "") -> str:
@@ -60,14 +55,3 @@ def format_csv(table: pa.Table, info: str = "") -> str:
     pacsv.write_csv(cleaned_table, sink, write_options=write_options)
     csv_text = sink.getvalue().to_pybytes().decode("utf-8")
     return str(info + csv_text)
-
-
-def get_verified_sources() -> dict[str, str]:
-    """List all available verified sources, cloning from dlt-verified-sources"""
-    sources = {}
-    with suppress_echo():
-        for source_name, source_config in _list_verified_sources(
-            repo_location=DEFAULT_VERIFIED_SOURCES_REPO
-        ).items():
-            sources[source_name] = source_config.doc
-        return sources

dlt/cli/__init__.py

@@ -1,14 +0,0 @@
-from dlt.common.configuration.plugins import SupportsCliCommand
-
-from dlt.cli.exceptions import CliCommandException
-
-DEFAULT_VERIFIED_SOURCES_REPO = "https://github.com/dlt-hub/verified-sources.git"
-DEFAULT_VIBE_SOURCES_REPO = "https://github.com/dlt-hub/vibe-hub.git"
-
-
-__all__ = [
-    "SupportsCliCommand",
-    "CliCommandException",
-    "DEFAULT_VERIFIED_SOURCES_REPO",
-    "DEFAULT_VIBE_SOURCES_REPO",
-]

dlt/cli/utils.py

@@ -1,72 +0,0 @@
-import ast
-import os
-from typing import Callable, Literal, Optional
-
-from dlt.common.reflection.utils import set_ast_parents
-from dlt.common.typing import TFun
-from dlt.common.configuration import resolve_configuration
-from dlt.common.configuration.specs import RuntimeConfiguration
-from dlt.common.runtime.telemetry import with_telemetry
-from dlt.common.runtime import run_context
-
-from dlt.reflection.script_visitor import PipelineScriptVisitor
-
-from dlt.cli.exceptions import CliCommandInnerException
-
-
-REQUIREMENTS_TXT = "requirements.txt"
-PYPROJECT_TOML = "pyproject.toml"
-GITHUB_WORKFLOWS_DIR = os.path.join(".github", "workflows")
-AIRFLOW_DAGS_FOLDER = os.path.join("dags")
-AIRFLOW_BUILD_FOLDER = os.path.join("build")
-LOCAL_COMMAND_REPO_FOLDER = "repos"
-MODULE_INIT = "__init__.py"
-
-
-def parse_init_script(
-    command: str, script_source: str, init_script_name: str
-) -> PipelineScriptVisitor:
-    # parse the script first
-    tree = ast.parse(source=script_source)
-    set_ast_parents(tree)
-    visitor = PipelineScriptVisitor(script_source)
-    visitor.visit_passes(tree)
-    if len(visitor.mod_aliases) == 0:
-        raise CliCommandInnerException(
-            command,
-            f"The pipeline script {init_script_name} does not import dlt and does not seem to run"
-            " any pipelines",
-        )
-
-    return visitor
-
-
-def ensure_git_command(command: str) -> None:
-    try:
-        import git
-    except ImportError as imp_ex:
-        if "Bad git executable" not in str(imp_ex):
-            raise
-        raise CliCommandInnerException(
-            command,
-            "'git' command is not available. Install and setup git with the following the guide %s"
-            % "https://docs.github.com/en/get-started/quickstart/set-up-git",
-            imp_ex,
-        ) from imp_ex
-
-
-def track_command(command: str, track_before: bool, *args: str) -> Callable[[TFun], TFun]:
-    return with_telemetry("command", command, track_before, *args)
-
-
-def get_telemetry_status() -> bool:
-    c = resolve_configuration(RuntimeConfiguration())
-    return c.dlthub_telemetry
-
-
-def make_dlt_settings_path(path: str = None) -> str:
-    """Returns path to file in dlt settings folder. Returns settings folder if path not specified."""
-    ctx = run_context.active()
-    if not path:
-        return ctx.settings_dir
-    return ctx.get_setting(path)

dlt/common/libs/git.py

@@ -4,6 +4,7 @@ import giturlparse
 from typing import Iterator, Optional, TYPE_CHECKING
 from contextlib import contextmanager
 
+from dlt.common.exceptions import MissingDependencyException
 from dlt.common.storages import FileStorage
 from dlt.common.utils import uniq_id
 from dlt.common.typing import Any
@@ -16,6 +17,17 @@ else:
     Repo = Any
 
 
+def _import_git() -> None:
+    try:
+        import git
+    except ModuleNotFoundError:
+        raise MissingDependencyException(
+            "git repository helpers",
+            ["gitpython>=3.1.29"],
+            "Install PythonGit to work with git repositories.",
+        )
+
+
 @contextmanager
 def git_custom_key_command(private_key: Optional[str]) -> Iterator[str]:
     if private_key:
@@ -50,13 +62,6 @@ def is_dirty(repo: Repo) -> bool:
     return len(status.strip()) > 0
 
 
-# def is_dirty(repo: Repo) -> bool:
-#     # get branch status
-#     status: str = repo.git.status("--short", "--branch")
-#     # we expect first status line ## main...origin/main
-#     return len(status.splitlines()) > 1
-
-
 def get_default_branch(repo: Repo) -> str:
     origin = repo.remotes.origin
     # Get the remote's HEAD reference (default branch)
@@ -88,6 +93,8 @@ def clone_repo(
     branch: Optional[str] = None,
     with_git_command: Optional[str] = None,
 ) -> Repo:
+    _import_git()
+
     from git import Repo
 
     repo = Repo.clone_from(repository_url, clone_path, env=dict(GIT_SSH_COMMAND=with_git_command))
@@ -152,9 +159,16 @@ def get_fresh_repo_files(
 
 
 def get_repo(path: str) -> Repo:
+    _import_git()
+
     from git import Repo
 
-    repo = Repo(path, search_parent_directories=True)
+    # if GIT_CEILING_DIRECTORIES is set then do not look up for repositories in parent dirs
+    search_parent_directories: bool = True
+    if os.getenv("GIT_CEILING_DIRECTORIES"):
+        search_parent_directories = False
+
+    repo = Repo(path, search_parent_directories=search_parent_directories)
     return repo
 
 

dlt/helpers/dbt/runner.py

@@ -14,7 +14,6 @@ from dlt.common.runners.stdout import iter_stdout_with_result
 from dlt.common.typing import StrAny, TSecretStrValue
 from dlt.common.logger import is_json_logging
 from dlt.common.storages import FileStorage
-from dlt.common.git import git_custom_key_command, ensure_remote_head, force_clone_repo
 from dlt.common.utils import with_custom_environ
 
 from dlt.helpers.dbt.configuration import DBTRunnerConfiguration
@@ -97,6 +96,7 @@ class DBTPackageRunner:
 
     def ensure_newest_package(self) -> None:
         """Clones or brings the dbt package at `package_location` up to date."""
+        from dlt.common.libs.git import git_custom_key_command, ensure_remote_head, force_clone_repo
         from git import GitError
 
         with git_custom_key_command(self.config.package_repository_ssh_key) as ssh_command:

dlt/helpers/streamlit_app/widgets/__init__.py

@@ -1,5 +0,0 @@
-from dlt.helpers.streamlit_app.widgets.stats import stat
-from dlt.helpers.streamlit_app.widgets.summary import pipeline_summary
-from dlt.helpers.streamlit_app.widgets.tags import tag
-from dlt.helpers.streamlit_app.widgets.schema import schema_picker
-from dlt.helpers.streamlit_app.widgets.color_mode_selector import mode_selector

docs/examples/conftest.py

@@ -1,7 +1,7 @@
 import os
 
 from tests.utils import (
-    patch_home_dir,
+    auto_test_run_context,
     autouse_test_storage,
     preserve_environ,
     deactivate_pipeline,

docs/tools/check_embedded_snippets.py

@@ -13,7 +13,7 @@ from typing import List, Dict
 
 import tomlkit
 import yaml
-import dlt.cli.echo as fmt
+import dlt._workspace.cli.echo as fmt
 
 from dlt.common import json
 

docs/tools/fix_grammar_gpt.py

@@ -11,7 +11,7 @@ from typing import List
 from openai import OpenAI
 from dotenv import load_dotenv
 
-import dlt.cli.echo as fmt
+import dlt._workspace.cli.echo as fmt
 
 from utils import collect_markdown_files
 

docs/tools/prepare_examples_tests.py

@@ -5,7 +5,7 @@ import os
 import argparse
 from typing import List
 
-import dlt.cli.echo as fmt
+import dlt._workspace.cli.echo as fmt
 
 EXAMPLES_DIR = "../examples"
 

docs/tools/utils.py

@@ -2,7 +2,7 @@ from typing import List
 import os
 import glob
 
-import dlt.cli.echo as fmt
+import dlt._workspace.cli.echo as fmt
 
 
 DOCS_DIR = "../website/docs"

docs/website/docs/conftest.py

@@ -1,7 +1,7 @@
 import os
 
 from tests.utils import (
-    patch_home_dir,
+    auto_test_run_context,
     autouse_test_storage,
     preserve_environ,
     deactivate_pipeline,

docs/website/docs/general-usage/dataset-access/dataset_snippets/dataset_snippets.py

@@ -2,7 +2,7 @@
 import dlt
 import pytest
 
-from dlt.sources._single_file_templates.fruitshop_pipeline import (
+from dlt._workspace._templates._single_file_templates.fruitshop_pipeline import (
     fruitshop as fruitshop_source,
 )
 

docs/website/docs/hub/features/transformations/transformation-snippets.py

@@ -14,7 +14,7 @@ def fruitshop_pipeline() -> dlt.Pipeline:
     # @@@DLT_SNIPPET_START quick_start_example
 
     from dlt.destinations import duckdb
-    from dlt.sources._single_file_templates.fruitshop_pipeline import (
+    from dlt._workspace._templates._single_file_templates.fruitshop_pipeline import (
         fruitshop as fruitshop_source,
     )
 

docs/website/pydoc-markdown.yml

@@ -4,7 +4,7 @@ loaders:
     packages: ["dlt"]
 processors:
   - type: filter
-    expression: not name.startswith("dlt.cli") and not name.startswith("dlt.normalize") and not name.startswith("dlt.load") and not name.startswith("dlt.reflection") and default()
+    expression: not name.startswith("dlt._workspace.cli") and not name.startswith("dlt.normalize") and not name.startswith("dlt.load") and not name.startswith("dlt.reflection") and default()
   - type: pydoc_markdown_dlt.DltProcessor
 renderer:
   type: docusaurus

mypy.ini

@@ -10,14 +10,13 @@ warn_return_any=true
 namespace_packages=true
 warn_unused_ignores=true
 show_error_codes=true
-#exclude=reflection/module_cases/*
 exclude=docs/examples/archive/*|tests/reflection/module_cases/*|tests/common/reflection/cases/modules/*|dlt/helpers/marimo/_widgets/*
 
 [mypy-tests.*]
 disallow_untyped_defs=false
 warn_return_any=false
 
-[mypy-dlt.sources._single_file_templates.*]
+[mypy-dlt._workspace._templates._single_file_templates.*]
 disallow_untyped_defs=false
 
 [mypy-docs.*]

pyproject.toml

@@ -115,7 +115,7 @@ motherduck = [
     "pyarrow>=16.0.0",
 ]
 cli = [
-    "pipdeptree>=2.9.0,<2.10",
+    "pipdeptree>=2.9.3,<2.10",
     "cron-descriptor>=1.2.32",
     "pip>=23.0.0",
 ]
@@ -193,7 +193,7 @@ Homepage = "https://github.com/dlt-hub"
 Repository = "https://github.com/dlt-hub/dlt"
 
 [project.scripts]
-dlt = "dlt.cli._dlt:_main"
+dlt = "dlt._workspace.cli._dlt:_main"
 
 [dependency-groups]
 # NOTE: add only dependencies used for linting, type checking etc. anything else goes to pipeline group
@@ -330,12 +330,10 @@ flake8-encodings = { git = "https://github.com/dlt-hub/flake8-encodings.git", br
 
 [tool.hatch.build.targets.sdist]
 packages = ["dlt"]
+# hatchling follows .gitignore and will include all files in gitindex under `dlt`
 include = [
     "LICENSE.txt",
     "README.md",
-    "dlt/sources/pipeline_templates/.gitignore",
-    "dlt/sources/pipeline_templates/.dlt/config.toml",
-    "dlt/helpers/dashboard/dlt_dashboard_styles.css"
 ]
 
 [tool.hatch.build.targets.wheel]
@@ -343,8 +341,6 @@ packages = ["dlt"]
 include = [
     "LICENSE.txt",
     "README.md",
-    "dlt/sources/pipeline_templates/.gitignore",
-    "dlt/sources/pipeline_templates/.dlt/config.toml",
 ]
 
 [build-system]

tests/cli/cases/deploy_pipeline/.dlt/secrets.toml

@@ -1,9 +0,0 @@
-api_key = "api_key_9x3ehash"
-
-[destination.postgres.credentials]
-database = "dlt_data"
-password = "wrong"  # keep a wrong password here
-username = "loader"
-host = "localhost"
-port = 5432
-connect_timeout = 15

tests/cli/cases/deploy_pipeline/.dlt/secrets.toml.ci

@@ -1,2 +0,0 @@
-api_key = "api_key_9x3ehash"
-

tests/cli/cases/deploy_pipeline/README.md

@@ -1,14 +0,0 @@
-# How this repo works
-1. `dlt init <source> <destination>` clones this repo, it uses the version of the dlt as a git tag to clone
-2. if the source is one of the variants ie. `chess` then `chess.py` is used as pipeline template, if not the `pipeline.py` will be used (after renaming to the <source>)
-2. id `--generic` options is passed the `pipeline_generic.py` template is used
-3. it modifies the script by importing the right destination and using it in the pipeline
-4. it copies the .gitignore, the pipeline script created above, the README and requirements to the folder in which dlt was called
-5. it will create the `secrets.toml` and `config.toml` for the source and destination in the script
-6. it will add the right dlt extra to `requirements.txt`
-
-
-
-# How to customize and deploy this pipeline?
-
-TODO: write some starting instructions

tests/cli/conftest.py

@@ -1,7 +0,0 @@
-from tests.utils import (
-    preserve_environ,
-    autouse_test_storage,
-    unload_modules,
-    deactivate_pipeline,
-    patch_home_dir,
-)

tests/cli/test_deploy_command.py

@@ -1,184 +0,0 @@
-import os
-import io
-import contextlib
-import shutil
-import tempfile
-from subprocess import CalledProcessError
-from git import InvalidGitRepositoryError, NoSuchPathError
-import pytest
-
-import dlt
-from dlt.common.runners import Venv
-from dlt.common.storages.file_storage import FileStorage
-from dlt.common.typing import StrAny
-from dlt.common.utils import set_working_dir
-
-from dlt.cli import deploy_command, echo, command_wrappers
-from dlt.cli.exceptions import CliCommandInnerException
-from dlt.pipeline.exceptions import CannotRestorePipelineException
-from dlt.cli.deploy_command_helpers import get_schedule_description
-from dlt.cli.exceptions import CliCommandException
-
-from tests.utils import TEST_STORAGE_ROOT, reset_providers, test_storage, LOCAL_POSTGRES_CREDENTIALS
-
-
-DEPLOY_PARAMS = [
-    ("github-action", {"schedule": "*/30 * * * *", "run_on_push": True, "run_manually": True}),
-    ("airflow-composer", {"secrets_format": "toml"}),
-    ("airflow-composer", {"secrets_format": "env"}),
-]
-
-
-@pytest.mark.parametrize("deployment_method,deployment_args", DEPLOY_PARAMS)
-def test_deploy_command_no_repo(
-    test_storage: FileStorage, deployment_method: str, deployment_args: StrAny
-) -> None:
-    pipeline_wf = tempfile.mkdtemp()
-    shutil.copytree("tests/cli/cases/deploy_pipeline", pipeline_wf, dirs_exist_ok=True)
-
-    with set_working_dir(pipeline_wf):
-        # we do not have repo
-        with pytest.raises(InvalidGitRepositoryError):
-            deploy_command.deploy_command(
-                "debug_pipeline.py",
-                deployment_method,
-                deploy_command.COMMAND_DEPLOY_REPO_LOCATION,
-                **deployment_args,
-            )
-
-        # test wrapper
-        with pytest.raises(CliCommandException) as ex:
-            command_wrappers.deploy_command_wrapper(
-                "debug_pipeline.py",
-                deployment_method,
-                deploy_command.COMMAND_DEPLOY_REPO_LOCATION,
-                **deployment_args,
-            )
-        assert ex._excinfo[1].error_code == -4
-
-
-@pytest.mark.parametrize("deployment_method,deployment_args", DEPLOY_PARAMS)
-def test_deploy_command(
-    test_storage: FileStorage, deployment_method: str, deployment_args: StrAny
-) -> None:
-    # drop pipeline
-    p = dlt.pipeline(pipeline_name="debug_pipeline")
-    p._wipe_working_folder()
-
-    shutil.copytree("tests/cli/cases/deploy_pipeline", TEST_STORAGE_ROOT, dirs_exist_ok=True)
-
-    with set_working_dir(TEST_STORAGE_ROOT):
-        from git import Repo, Remote
-
-        # we have a repo without git origin
-        with Repo.init(".") as repo:
-            # test no origin
-            with pytest.raises(CliCommandInnerException) as py_ex:
-                deploy_command.deploy_command(
-                    "debug_pipeline.py",
-                    deployment_method,
-                    deploy_command.COMMAND_DEPLOY_REPO_LOCATION,
-                    **deployment_args,
-                )
-            assert "Your current repository has no origin set" in py_ex.value.args[0]
-            with pytest.raises(CliCommandInnerException):
-                command_wrappers.deploy_command_wrapper(
-                    "debug_pipeline.py",
-                    deployment_method,
-                    deploy_command.COMMAND_DEPLOY_REPO_LOCATION,
-                    **deployment_args,
-                )
-
-            # we have a repo that was never run
-            Remote.create(repo, "origin", "git@github.com:rudolfix/dlt-cmd-test-2.git")
-            with pytest.raises(CannotRestorePipelineException):
-                deploy_command.deploy_command(
-                    "debug_pipeline.py",
-                    deployment_method,
-                    deploy_command.COMMAND_DEPLOY_REPO_LOCATION,
-                    **deployment_args,
-                )
-            with pytest.raises(CliCommandException) as ex:
-                command_wrappers.deploy_command_wrapper(
-                    "debug_pipeline.py",
-                    deployment_method,
-                    deploy_command.COMMAND_DEPLOY_REPO_LOCATION,
-                    **deployment_args,
-                )
-            assert ex._excinfo[1].error_code == -3
-
-            # run the script with wrong credentials (it is postgres there)
-            venv = Venv.restore_current()
-            # mod environ so wrong password is passed to override secrets.toml
-            os.environ["DESTINATION__POSTGRES__CREDENTIALS__PASSWORD"] = "password"
-            with pytest.raises(CalledProcessError):
-                venv.run_script("debug_pipeline.py")
-            # print(py_ex.value.output)
-            with pytest.raises(deploy_command.PipelineWasNotRun) as py_ex2:
-                deploy_command.deploy_command(
-                    "debug_pipeline.py",
-                    deployment_method,
-                    deploy_command.COMMAND_DEPLOY_REPO_LOCATION,
-                    **deployment_args,
-                )
-            assert "The last pipeline run ended with error" in py_ex2.value.args[0]
-            with pytest.raises(CliCommandException) as ex:
-                command_wrappers.deploy_command_wrapper(
-                    "debug_pipeline.py",
-                    deployment_method,
-                    deploy_command.COMMAND_DEPLOY_REPO_LOCATION,
-                    **deployment_args,
-                )
-            assert ex._excinfo[1].error_code == -3
-
-            os.environ["DESTINATION__POSTGRES__CREDENTIALS"] = LOCAL_POSTGRES_CREDENTIALS
-            # also delete secrets so credentials are not mixed up on CI
-            test_storage.delete(".dlt/secrets.toml")
-            test_storage.atomic_rename(".dlt/secrets.toml.ci", ".dlt/secrets.toml")
-
-            # reset toml providers to (1) where secrets exist (2) non existing dir so API_KEY is not found
-            for settings_dir, api_key in [
-                (os.path.join(test_storage.storage_path, ".dlt"), "api_key_9x3ehash"),
-                (".", "please set me up!"),
-            ]:
-                with reset_providers(settings_dir=settings_dir):
-                    # this time script will run
-                    venv.run_script("debug_pipeline.py")
-                    with echo.always_choose(False, always_choose_value=True):
-                        with io.StringIO() as buf, contextlib.redirect_stdout(buf):
-                            deploy_command.deploy_command(
-                                "debug_pipeline.py",
-                                deployment_method,
-                                deploy_command.COMMAND_DEPLOY_REPO_LOCATION,
-                                **deployment_args,
-                            )
-                            _out = buf.getvalue()
-                        print(_out)
-                        # make sure our secret and config values are all present
-                        assert api_key in _out
-                        assert "dlt_data" in _out
-                        if "schedule" in deployment_args:
-                            assert get_schedule_description(deployment_args["schedule"])
-                        secrets_format = deployment_args.get("secrets_format", "env")
-                        if secrets_format == "env":
-                            assert "API_KEY" in _out
-                        else:
-                            assert "api_key = " in _out
-
-            # non existing script name
-            with pytest.raises(NoSuchPathError):
-                deploy_command.deploy_command(
-                    "no_pipeline.py",
-                    deployment_method,
-                    deploy_command.COMMAND_DEPLOY_REPO_LOCATION,
-                    **deployment_args,
-                )
-            with echo.always_choose(False, always_choose_value=True):
-                with pytest.raises(CliCommandException) as ex:
-                    command_wrappers.deploy_command_wrapper(
-                        "no_pipeline.py",
-                        deployment_method,
-                        deploy_command.COMMAND_DEPLOY_REPO_LOCATION,
-                        **deployment_args,
-                    )
-                assert ex._excinfo[1].error_code == -5

tests/common/configuration/test_accessors.py

@@ -6,11 +6,7 @@ import dlt
 from dlt.common import json
 from dlt.common.configuration.exceptions import ConfigFieldMissingException
 
-from dlt.common.configuration.providers import (
-    EnvironProvider,
-    ConfigTomlProvider,
-    SecretsTomlProvider,
-)
+from dlt.common.configuration.providers import EnvironProvider
 from dlt.common.configuration.providers.toml import (
     CONFIG_TOML,
     SECRETS_TOML,

tests/common/configuration/test_configuration.py

@@ -78,7 +78,7 @@ from dlt.common.configuration.utils import (
     add_config_to_env,
 )
 from dlt.common.pipeline import TRefreshMode, PipelineContext
-from dlt.cli.config_toml_writer import TYPE_EXAMPLES
+from dlt._workspace.cli.config_toml_writer import TYPE_EXAMPLES
 
 from dlt.destinations.impl.postgres.configuration import PostgresCredentials
 from tests.utils import preserve_environ, TEST_STORAGE_ROOT
@@ -96,7 +96,7 @@ from tests.common.configuration.utils import (
     toml_providers,
     mock_provider,
     env_provider,
-    reset_resolved_traces,
+    auto_reset_resolved_traces,
 )
 import dlt
 
@@ -1425,36 +1425,39 @@ def test_resolved_trace(environment: Any) -> None:
 @pytest.mark.parametrize("enable_logging", (True, False))
 def test_unresolved_trace(environment: Any, enable_logging: bool) -> None:
     tracer = get_resolved_traces()
-    tracer.logging_enabled = enable_logging
+    try:
+        tracer.logging_enabled = enable_logging
 
-    @configspec
-    class OptEmbeddedConfiguration(BaseConfiguration):
-        default: Optional[str] = None
-        instrumented: InstrumentedConfiguration = None
-        sectioned: SectionedConfiguration = None
+        @configspec
+        class OptEmbeddedConfiguration(BaseConfiguration):
+            default: Optional[str] = None
+            instrumented: InstrumentedConfiguration = None
+            sectioned: SectionedConfiguration = None
 
-    with custom_environ(
-        {
-            "INSTRUMENTED__HEAD": "h",
-            "INSTRUMENTED__TUBE": '["tu", "u", "be"]',
-            "INSTRUMENTED__HEELS": "xhe",
-        }
-    ):
-        resolve.resolve_configuration(
-            OptEmbeddedConfiguration(default="_DEFF"),
-            sections=("wrapper", "spec"),
-            explicit_value={"default": None, "sectioned": {"password": "$pwd"}},
-        )
+        with custom_environ(
+            {
+                "INSTRUMENTED__HEAD": "h",
+                "INSTRUMENTED__TUBE": '["tu", "u", "be"]',
+                "INSTRUMENTED__HEELS": "xhe",
+            }
+        ):
+            resolve.resolve_configuration(
+                OptEmbeddedConfiguration(default="_DEFF"),
+                sections=("wrapper", "spec"),
+                explicit_value={"default": None, "sectioned": {"password": "$pwd"}},
+            )
 
-    if enable_logging:
-        # we try in ("wrapper", "spec") so there are 3 read attempts per resolved value
-        assert len(tracer.all_traces) == 3 * len(tracer.resolved_traces)
-        # there are 3 resolved values, explicit values are not included
-        assert len(tracer.resolved_traces) == 3
-        # first resolved value sections are full depth
-        assert tracer.all_traces[0].sections == ["wrapper", "spec", "instrumented"]
-    else:
-        assert len(tracer.all_traces) == len(tracer.resolved_traces) == 0
+        if enable_logging:
+            # we try in ("wrapper", "spec") so there are 3 read attempts per resolved value
+            assert len(tracer.all_traces) == 3 * len(tracer.resolved_traces)
+            # there are 3 resolved values, explicit values are not included
+            assert len(tracer.resolved_traces) == 3
+            # first resolved value sections are full depth
+            assert tracer.all_traces[0].sections == ["wrapper", "spec", "instrumented"]
+        else:
+            assert len(tracer.all_traces) == len(tracer.resolved_traces) == 0
+    finally:
+        tracer.logging_enabled = True
 
 
 def test_extract_inner_hint() -> None:

tests/common/configuration/test_toml_provider.py

@@ -34,7 +34,7 @@ from dlt.common.configuration.specs import (
 from dlt.common.runners.configuration import PoolRunnerConfiguration
 from dlt.common.typing import TSecretValue
 
-from tests.utils import preserve_environ, unload_modules
+from tests.utils import preserve_environ, auto_unload_modules
 from tests.common.configuration.utils import (
     ConnectionStringCompatCredentials,
     SecretCredentials,

tests/common/configuration/utils.py

@@ -14,19 +14,17 @@ from typing import (
     MutableMapping,
     Optional,
     Sequence,
-    TYPE_CHECKING,
 )
 
 from dlt.common import Decimal, pendulum
 from dlt.common.configuration import configspec
 from dlt.common.configuration.specs import BaseConfiguration, CredentialsConfiguration
-from dlt.common.configuration.container import Container
 from dlt.common.configuration.providers import ConfigProvider, EnvironProvider
 from dlt.common.configuration.specs.connection_string_credentials import ConnectionStringCredentials
 from dlt.common.configuration.utils import get_resolved_traces
 from dlt.common.configuration.specs.config_providers_context import ConfigProvidersContainer
 from dlt.common.typing import TSecretValue, StrAny
-from tests.utils import _inject_providers, _reset_providers, inject_providers
+from tests.utils import _reset_providers, inject_providers
 
 
 @configspec
@@ -118,7 +116,7 @@ def environment() -> Any:
 
 
 @pytest.fixture(autouse=True)
-def reset_resolved_traces() -> Iterator[None]:
+def auto_reset_resolved_traces() -> Iterator[None]:
     log = get_resolved_traces()
     try:
         log.clear()

tests/common/reflection/test_ref_ops.py

@@ -4,7 +4,7 @@ import sys
 
 from dlt.common.reflection.ref import object_from_ref, callable_typechecker
 from dlt.extract.reference import SourceFactory, SourceReference
-from tests.utils import unload_modules
+from tests.utils import auto_unload_modules
 
 
 @pytest.fixture(autouse=True)

tests/common/runtime/test_run_context_data_dir.py

@@ -4,7 +4,7 @@ import dlt
 
 # import auto fixture that sets global and data dir to TEST_STORAGE
 from dlt.common.runtime.run_context import DOT_DLT
-from tests.utils import TEST_STORAGE_ROOT, patch_home_dir
+from tests.utils import TEST_STORAGE_ROOT, auto_test_run_context
 
 
 def test_data_dir_test_storage() -> None: