Fix: reset config in PluggableRunContext.reload_providers (#3409)

* implement RunContext.reset_config, call it in PluggableRunContext.reload_providers

* fix _config access

* reiinitialize RunContext._runtime_config on access

* adjust the test to .runtime_config being always available

* fixes dlthub tests

---------

Co-authored-by: ivasio <ivan@dlthub.com>
Co-authored-by: Marcin Rudolf <rudolfix@rudolfix.org>

.github/workflows/test_hub.yml

@@ -81,7 +81,7 @@ jobs:
         run: make dev
 
       - name: Install dlthub
-        run: uv run pip install --upgrade --force-reinstall --no-cache-dir ${{ matrix.dlthub_dep }}
+        run: uv run pip install --upgrade --force-reinstall --pre --no-cache-dir ${{ matrix.dlthub_dep }}
 
       - name: Run tests
         run: pytest tests/hub

dlt/_workspace/_workspace_context.py

@@ -100,7 +100,7 @@ class WorkspaceRunContext(ProfilesRunContext):
 
     @property
     def runtime_config(self) -> WorkspaceRuntimeConfiguration:
-        return self._config.runtime
+        return self.config.runtime
 
     @property
     def config(self) -> WorkspaceConfiguration:
@@ -162,6 +162,11 @@ class WorkspaceRunContext(ProfilesRunContext):
     def unplug(self) -> None:
         pass
 
+    def reset_config(self) -> None:
+        # Drop resolved configuration to force re-resolve with refreshed providers
+        self._config = None
+        # no need to initialize the _config anew as it's done in .config property
+
     # SupportsProfilesOnContext
 
     @property

dlt/common/configuration/specs/pluggable_run_context.py

@@ -137,6 +137,10 @@ class RunContextBase(ABC):
                 f"`{run_dir=:}` doesn't belong to module `{m_.__file__}` which seems unrelated."
             )
 
+    @abstractmethod
+    def reset_config(self) -> None:
+        """Hook for contexts that store resolved configuration to reset it"""
+
 
 class ProfilesRunContext(RunContextBase):
     """Adds profile support on run context. Note: runtime checkable protocols are slow on isinstance"""
@@ -206,7 +210,10 @@ class PluggableRunContext(ContainerInjectableContext):
 
     def reload_providers(self) -> None:
         self.providers = ConfigProvidersContainer(self.context.initial_providers())
+        # Re-add extras and re-initialize runtime so changes take effect
         self.providers.add_extras()
+        # Invalidate any cached configuration on the context so it re-resolves using new providers
+        self.context.reset_config()
 
     def after_add(self) -> None:
         super().after_add()

dlt/common/runtime/run_context.py

@@ -81,6 +81,8 @@ class RunContext(RunContextBase):
 
     @property
     def runtime_config(self) -> RuntimeConfiguration:
+        if self._runtime_config is None:
+            self.initialize_runtime()
         return self._runtime_config
 
     @property
@@ -114,6 +116,9 @@ class RunContext(RunContextBase):
     def unplug(self) -> None:
         pass
 
+    def reset_config(self) -> None:
+        self._runtime_config = None
+
     @property
     def name(self) -> str:
         return "dlt"

dlt/helpers/ibis.py

@@ -11,7 +11,6 @@ from dlt.destinations.impl.athena.configuration import AthenaClientConfiguration
 from dlt.destinations.impl.duckdb.configuration import DuckDbClientConfiguration
 from dlt.destinations.impl.databricks.configuration import DatabricksClientConfiguration
 from dlt.destinations.impl.ducklake.configuration import DuckLakeClientConfiguration
-from dlt.destinations.impl.ducklake.ducklake import DuckLakeClient
 from dlt.destinations.impl.motherduck.configuration import MotherDuckClientConfiguration
 from dlt.destinations.impl.postgres.configuration import PostgresClientConfiguration
 from dlt.destinations.impl.redshift.configuration import RedshiftClientConfiguration
@@ -74,6 +73,8 @@ def create_ibis_backend(
             # move main connection ownership to ibis
             con = ibis.duckdb.from_connection(client.config.credentials.conn_pool.move_conn())
     elif issubclass(destination.spec, DuckLakeClientConfiguration):
+        from dlt.destinations.impl.ducklake.ducklake import DuckLakeClient
+
         assert isinstance(client, DuckLakeClient)
         # open connection but do not close it, ducklake always creates a separate connection
         # and will not close it in destructor

tests/common/runtime/test_run_context.py

@@ -46,11 +46,6 @@ def test_run_context() -> None:
     # check config providers
     assert len(run_context.initial_providers()) == 3
 
-    assert ctx.context.runtime_config is None
-    ctx.add_extras()
-    # still not applied - must be in container
-    assert ctx.context.runtime_config is None
-
     with Container().injectable_context(ctx):
         ctx.initialize_runtime()
     assert ctx.context.runtime_config is not None

tests/load/test_lineage.py

@@ -137,7 +137,10 @@ def test_various_queries(destination_config: DestinationTestConfiguration, examp
         0
     ] == {
         "name": {"name": "name", "data_type": "text", "x-annotation-pii": True},
-        "total_amount": {"name": "total_amount", "data_type": "double"},  # , "data_type": "double"},
+        "total_amount": {
+            "name": "total_amount",
+            "data_type": "double",
+        },  # , "data_type": "double"},
     }
 
     # test WITH clause