Force use of typeddict from typingexentions, pin poetry in tests, simple disable python 3.8 (#2185)

* import correct typeddict version for use in pydantic, disallow use of usual python typeddict imports

* add test

* update import for examples

* fixed some imports

* remove python 3.8 lint and test for now

* always use typeddict from typing_extensions
pin poetry in tests to 1.8.5

.github/workflows/lint.yml

@@ -26,7 +26,7 @@ jobs:
       matrix:
         os:
           - ubuntu-latest
-        python-version: ["3.8.x", "3.9.x", "3.10.x", "3.11.x"]
+        python-version: ["3.9.x", "3.10.x", "3.11.x"]
 
     defaults:
       run:
@@ -49,6 +49,7 @@ jobs:
           virtualenvs-create: true
           virtualenvs-in-project: true
           installer-parallel: true
+          version: 1.8.5
 
       - name: Load cached venv
         id: cached-poetry-dependencies

.github/workflows/test_airflow.yml

@@ -37,6 +37,7 @@ jobs:
           virtualenvs-create: true
           virtualenvs-in-project: true
           installer-parallel: true
+          version: 1.8.5
 
       - name: Load cached venv
         id: cached-poetry-dependencies

.github/workflows/test_build_images.yml

@@ -37,6 +37,7 @@ jobs:
           virtualenvs-create: true
           virtualenvs-in-project: true
           installer-parallel: true
+          version: 1.8.5
 
       - name: Build images
         run: make test-build-images

.github/workflows/test_common.yml

@@ -37,8 +37,6 @@ jobs:
         python-version: ["3.11.x"]
         # Test all python versions on ubuntu only
         include:
-          - python-version: "3.8.x"
-            os: "ubuntu-latest"
           - python-version: "3.9.x"
             os: "ubuntu-latest"
           - python-version: "3.10.x"
@@ -79,6 +77,7 @@ jobs:
           virtualenvs-create: true
           virtualenvs-in-project: true
           installer-parallel: true
+          version: 1.8.5
 
       # NOTE: do not cache. we want to have a clean state each run and we upgrade depdendencies later
       # - name: Load cached venv

.github/workflows/test_dbt_cloud.yml

@@ -52,6 +52,7 @@ jobs:
           virtualenvs-create: true
           virtualenvs-in-project: true
           installer-parallel: true
+          version: 1.8.5
 
       - name: Load cached venv
         id: cached-poetry-dependencies

.github/workflows/test_dbt_runner.yml

@@ -49,6 +49,7 @@ jobs:
           virtualenvs-create: true
           virtualenvs-in-project: true
           installer-parallel: true
+          version: 1.8.5
 
       - name: Load cached venv
         id: cached-poetry-dependencies

.github/workflows/test_destination_athena.yml

@@ -56,6 +56,7 @@ jobs:
           virtualenvs-create: true
           virtualenvs-in-project: true
           installer-parallel: true
+          version: 1.8.5
 
       - name: Load cached venv
         id: cached-poetry-dependencies

.github/workflows/test_destination_athena_iceberg.yml

@@ -56,6 +56,7 @@ jobs:
           virtualenvs-create: true
           virtualenvs-in-project: true
           installer-parallel: true
+          version: 1.8.5
 
       - name: Load cached venv
         id: cached-poetry-dependencies

.github/workflows/test_destination_bigquery.yml

@@ -55,6 +55,7 @@ jobs:
           virtualenvs-create: true
           virtualenvs-in-project: true
           installer-parallel: true
+          version: 1.8.5
 
       - name: Load cached venv
         id: cached-poetry-dependencies

.github/workflows/test_destination_clickhouse.yml

@@ -52,6 +52,7 @@ jobs:
           virtualenvs-create: true
           virtualenvs-in-project: true
           installer-parallel: true
+          version: 1.8.5
 
       - name: Load cached venv
         id: cached-poetry-dependencies

.github/workflows/test_destination_databricks.yml

@@ -55,6 +55,7 @@ jobs:
           virtualenvs-create: true
           virtualenvs-in-project: true
           installer-parallel: true
+          version: 1.8.5
 
       - name: Load cached venv
         id: cached-poetry-dependencies

.github/workflows/test_destination_dremio.yml

@@ -56,6 +56,7 @@ jobs:
           virtualenvs-create: true
           virtualenvs-in-project: true
           installer-parallel: true
+          version: 1.8.5
 
       - name: Load cached venv
         id: cached-poetry-dependencies

.github/workflows/test_destination_lancedb.yml

@@ -53,6 +53,7 @@ jobs:
           virtualenvs-create: true
           virtualenvs-in-project: true
           installer-parallel: true
+          version: 1.8.5
 
       - name: Load cached venv
         id: cached-poetry-dependencies

.github/workflows/test_destination_motherduck.yml

@@ -55,6 +55,7 @@ jobs:
           virtualenvs-create: true
           virtualenvs-in-project: true
           installer-parallel: true
+          version: 1.8.5
 
       - name: Load cached venv
         id: cached-poetry-dependencies

.github/workflows/test_destination_mssql.yml

@@ -60,6 +60,7 @@ jobs:
           virtualenvs-create: true
           virtualenvs-in-project: true
           installer-parallel: true
+          version: 1.8.5
 
       - name: Load cached venv
         id: cached-poetry-dependencies

.github/workflows/test_destination_qdrant.yml

@@ -54,6 +54,7 @@ jobs:
           virtualenvs-create: true
           virtualenvs-in-project: true
           installer-parallel: true
+          version: 1.8.5
 
       - name: Load cached venv
         id: cached-poetry-dependencies

.github/workflows/test_destination_snowflake.yml

@@ -55,6 +55,7 @@ jobs:
           virtualenvs-create: true
           virtualenvs-in-project: true
           installer-parallel: true
+          version: 1.8.5
 
       - name: Load cached venv
         id: cached-poetry-dependencies

.github/workflows/test_destination_synapse.yml

@@ -58,6 +58,7 @@ jobs:
           virtualenvs-create: true
           virtualenvs-in-project: true
           installer-parallel: true
+          version: 1.8.5
 
       - name: Load cached venv
         id: cached-poetry-dependencies

.github/workflows/test_destinations.yml

@@ -67,6 +67,7 @@ jobs:
           virtualenvs-create: true
           virtualenvs-in-project: true
           installer-parallel: true
+          version: 1.8.5
 
       # - name: Load cached venv
       #   id: cached-poetry-dependencies

.github/workflows/test_doc_snippets.yml

@@ -78,6 +78,7 @@ jobs:
           virtualenvs-create: true
           virtualenvs-in-project: true
           installer-parallel: true
+          version: 1.8.5
 
       - name: Load cached venv
         id: cached-poetry-dependencies

.github/workflows/test_local_destinations.yml

@@ -86,6 +86,7 @@ jobs:
           virtualenvs-create: true
           virtualenvs-in-project: true
           installer-parallel: true
+          version: 1.8.5
 
       - name: Load cached venv
         id: cached-poetry-dependencies

.github/workflows/test_local_sources.yml

@@ -73,6 +73,7 @@ jobs:
           virtualenvs-create: true
           virtualenvs-in-project: true
           installer-parallel: true
+          version: 1.8.5
 
       - name: Load cached venv
         id: cached-poetry-dependencies

.github/workflows/test_sqlalchemy_destinations.yml

@@ -77,6 +77,7 @@ jobs:
           virtualenvs-create: true
           virtualenvs-in-project: true
           installer-parallel: true
+          version: 1.8.5
 
       - name: Load cached venv
         id: cached-poetry-dependencies

dlt/cli/pipeline_files.py

@@ -4,11 +4,12 @@ import os
 import yaml
 import posixpath
 from pathlib import Path
-from typing import Dict, NamedTuple, Sequence, Tuple, TypedDict, List, Literal
+from typing import Dict, NamedTuple, Sequence, Tuple, List, Literal
 from dlt.cli.exceptions import VerifiedSourceRepoError
 
 from dlt.common import git
 from dlt.common.storages import FileStorage
+from dlt.common.typing import TypedDict
 
 from dlt.common.reflection.utils import get_module_docstring
 

dlt/common/exceptions.py

@@ -1,4 +1,5 @@
-from typing import Any, AnyStr, Dict, List, Sequence, Optional, Iterable, Type, TypedDict
+from typing import Any, AnyStr, Dict, List, Sequence, Optional, Type
+from dlt.common.typing import TypedDict
 
 
 class ExceptionTrace(TypedDict, total=False):

dlt/common/incremental/typing.py

@@ -1,8 +1,6 @@
-from typing_extensions import TypedDict
-
 from typing import Any, Callable, List, Literal, Optional, Sequence, TypeVar, Union
 
-from dlt.common.typing import TSortOrder, TTableHintTemplate, TColumnNames
+from dlt.common.typing import TSortOrder, TTableHintTemplate, TColumnNames, TypedDict
 
 TCursorValue = TypeVar("TCursorValue", bound=Any)
 LastValueFunc = Callable[[Sequence[TCursorValue]], Any]

dlt/common/libs/pydantic.py

@@ -6,7 +6,6 @@ from typing import (
     Generic,
     Optional,
     Set,
-    TypedDict,
     List,
     Type,
     Union,
@@ -15,6 +14,7 @@ from typing import (
 )
 from typing_extensions import Annotated, get_args, get_origin
 
+from dlt.common.typing import TypedDict
 from dlt.common.data_types import py_type_to_sc_type
 from dlt.common.exceptions import MissingDependencyException
 from dlt.common.schema import DataValidationError

dlt/common/metrics.py

@@ -1,5 +1,6 @@
 import datetime  # noqa: I251
-from typing import Any, Dict, List, NamedTuple, Optional, Tuple, TypedDict  # noqa: 251
+from typing import Any, Dict, List, NamedTuple, Optional, Tuple  # noqa: 251
+from dlt.common.typing import TypedDict
 
 
 class DataWriterMetrics(NamedTuple):

dlt/common/normalizers/json/relational.py

@@ -8,10 +8,10 @@ from typing import (
     Tuple,
     Type,
     cast,
-    TypedDict,
     Any,
 )
 
+from dlt.common.typing import TypedDict
 from dlt.common.normalizers.exceptions import InvalidJsonNormalizer
 from dlt.common.normalizers.typing import TJSONNormalizer
 from dlt.common.normalizers.utils import generate_dlt_id

dlt/common/normalizers/typing.py

@@ -1,7 +1,7 @@
-from typing import List, Optional, Type, TypedDict, Literal, Union
+from typing import List, Optional, Type, Literal, Union
 from types import ModuleType
 
-from dlt.common.typing import StrAny
+from dlt.common.typing import StrAny, TypedDict
 from dlt.common.normalizers.naming import NamingConvention
 
 TNamingConventionReferenceArg = Union[str, Type[NamingConvention], ModuleType]

dlt/common/pipeline.py

@@ -18,12 +18,12 @@ from typing import (
     Sequence,
     Tuple,
     TypeVar,
-    TypedDict,
     Mapping,
     Literal,
 )
 from typing_extensions import NotRequired
 
+from dlt.common.typing import TypedDict
 from dlt.common.configuration import configspec
 from dlt.common.configuration import known_sections
 from dlt.common.configuration.container import Container

dlt/common/runtime/typing.py

@@ -1,9 +1,8 @@
 from typing import (
     List,
     Literal,
-    TypedDict,
 )
-
+from dlt.common.typing import TypedDict
 
 TExecInfoNames = Literal[
     "kubernetes",

dlt/common/schema/typing.py

@@ -10,13 +10,13 @@ from typing import (
     Set,
     Tuple,
     Type,
-    TypedDict,
     NewType,
     Union,
     get_args,
 )
 from typing_extensions import Never
 
+from dlt.common.typing import TypedDict
 from dlt.common.data_types import TDataType
 from dlt.common.normalizers.typing import TNormalizersConfig
 from dlt.common.typing import TSortOrder, TAnyDateTime, TLoaderFileFormat, TColumnNames

dlt/common/storages/fsspec_filesystem.py

@@ -9,7 +9,6 @@ from typing import (
     Literal,
     cast,
     Tuple,
-    TypedDict,
     Optional,
     Union,
     Iterator,
@@ -25,6 +24,7 @@ from fsspec import AbstractFileSystem, register_implementation, get_filesystem_c
 from fsspec.core import url_to_fs
 
 from dlt import version
+from dlt.common.typing import TypedDict
 from dlt.common.pendulum import pendulum
 from dlt.common.configuration.specs import (
     GcpCredentials,

dlt/common/storages/load_package.py

@@ -21,10 +21,10 @@ from typing import (
     cast,
     Any,
     Tuple,
-    TypedDict,
 )
 from typing_extensions import NotRequired
 
+from dlt.common.typing import TypedDict
 from dlt.common.pendulum import pendulum
 from dlt.common.json import json
 from dlt.common.configuration import configspec

dlt/common/typing.py

@@ -45,6 +45,8 @@ from typing_extensions import (
 
 from typing_extensions import is_typeddict as _is_typeddict
 
+from typing_extensions import TypedDict  # noqa: I251
+
 try:
     from types import UnionType  # type: ignore[attr-defined]
 except ImportError:

dlt/common/versioned_state.py

@@ -2,11 +2,12 @@ import base64
 import hashlib
 import binascii
 from copy import copy
-from typing import TypedDict, List, Tuple, Mapping
+from typing import List, Tuple, Mapping
 
 from dlt.common.json import json
 from dlt.common.typing import DictStrAny
 from dlt.common.utils import compressed_b64decode, compressed_b64encode
+from dlt.common.typing import TypedDict
 
 
 class TVersionedState(TypedDict, total=False):

dlt/destinations/sql_client.py

@@ -17,11 +17,10 @@ from typing import (
     AnyStr,
     List,
     Generator,
-    TypedDict,
     cast,
 )
 
-from dlt.common.typing import TFun
+from dlt.common.typing import TFun, TypedDict
 from dlt.common.schema.typing import TTableSchemaColumns
 from dlt.common.destination import DestinationCapabilitiesContext
 from dlt.common.utils import concat_strings_with_limit

dlt/destinations/sql_jobs.py

@@ -1,9 +1,10 @@
-from typing import Any, Dict, List, Sequence, Tuple, cast, TypedDict, Optional, Callable, Union
+from typing import Any, Dict, List, Sequence, Tuple, cast, Optional, Callable, Union
 
 import yaml
 from dlt.common.time import ensure_pendulum_datetime
 from dlt.common.destination.reference import PreparedTableSchema
 from dlt.common.destination.utils import resolve_merge_strategy
+from dlt.common.typing import TypedDict
 
 from dlt.common.schema.typing import (
     TSortOrder,

dlt/extract/hints.py

@@ -1,4 +1,4 @@
-from typing import TypedDict, cast, Any, Optional, Dict, Sequence, Mapping, Union
+from typing import cast, Any, Optional, Dict, Sequence, Mapping, Union
 from typing_extensions import Self
 
 from dlt.common import logger
@@ -19,6 +19,8 @@ from dlt.common.schema.typing import (
     MERGE_STRATEGIES,
     TTableReferenceParam,
 )
+
+from dlt.common.typing import TypedDict
 from dlt.common.schema.utils import (
     DEFAULT_WRITE_DISPOSITION,
     merge_column,

dlt/pipeline/drop.py

@@ -1,4 +1,4 @@
-from typing import Union, Iterable, Optional, List, Dict, Any, Tuple, TypedDict
+from typing import Union, Iterable, Optional, List, Dict, Any, Tuple
 from copy import deepcopy
 from itertools import chain
 from dataclasses import dataclass
@@ -12,6 +12,8 @@ from dlt.common.pipeline import (
     reset_resource_state,
     _delete_source_state_keys,
 )
+from dlt.common.typing import TypedDict
+
 from dlt.common.schema.typing import TSimpleRegex, TTableSchema
 from dlt.common.schema.utils import (
     group_tables_by_resource,

dlt/pipeline/platform.py

@@ -1,5 +1,5 @@
 """Implements SupportsTracking"""
-from typing import Any, cast, TypedDict, List
+from typing import Any, cast, List
 from requests import Session
 
 from dlt.common import logger
@@ -7,6 +7,7 @@ from dlt.common.json import json
 from dlt.common.pipeline import LoadInfo
 from dlt.common.managed_thread_pool import ManagedThreadPool
 from dlt.common.schema.typing import TStoredSchema
+from dlt.common.typing import TypedDict
 
 from dlt.pipeline.trace import PipelineTrace, PipelineStepTrace, TPipelineStep, SupportsPipeline
 

dlt/sources/rest_api/typing.py

@@ -1,5 +1,4 @@
 from dataclasses import dataclass, field
-from typing_extensions import TypedDict
 
 from typing import (
     Any,
@@ -12,6 +11,7 @@ from typing import (
 )
 
 from dlt.common import jsonpath
+from dlt.common.typing import TypedDict
 from dlt.common.schema.typing import (
     TAnySchemaColumns,
 )

dlt/sources/sql_database/schema_types.py

@@ -7,14 +7,13 @@ from typing import (
     List,
     Callable,
     Union,
-    TypedDict,
     Dict,
 )
 from typing_extensions import TypeAlias
 
 from sqlalchemy.exc import NoReferencedTableError
 
-
+from dlt.common.typing import TypedDict
 from dlt.common.libs.sql_alchemy import Table, Column, Row, sqltypes, Select, TypeEngine
 from dlt.common import logger
 from dlt.common.schema.typing import TColumnSchema, TTableSchemaColumns, TTableReference

docs/examples/archive/sources/singer_tap.py

@@ -1,8 +1,9 @@
 import os
 import tempfile
-from typing import Any, Iterator, TypedDict, cast, Union
+from typing import Any, Iterator, cast, Union
 
 import dlt
+from dlt.common.typing import TypedDict
 from dlt.common import json
 from dlt.common.configuration.specs import BaseConfiguration
 from dlt.common.runners.venv import Venv

tests/common/test_typing.py

@@ -16,7 +16,6 @@ from typing import (
     NewType,
     Sequence,
     TypeVar,
-    TypedDict,
     Optional,
     Union,
 )
@@ -48,6 +47,7 @@ from dlt.common.typing import (
     is_callable_type,
     add_value_to_literal,
     get_generic_type_argument_from_instance,
+    TypedDict,
 )
 
 

tests/common/test_validation.py

@@ -10,7 +10,6 @@ from typing import (
     Literal,
     Mapping,
     Sequence,
-    TypedDict,
     TypeVar,
     Optional,
     Union,
@@ -24,7 +23,7 @@ from dlt.common.schema.typing import (
     TWriteDispositionConfig,
 )
 from dlt.common.schema.utils import simple_regex_validator
-from dlt.common.typing import DictStrStr, StrStr, TDataItem, TSortOrder, TColumnNames
+from dlt.common.typing import DictStrStr, StrStr, TDataItem, TSortOrder, TColumnNames, TypedDict
 
 from dlt.common.validation import validate_dict, validate_dict_ignoring_xkeys
 

tests/libs/test_pydantic.py

@@ -24,6 +24,7 @@ from enum import Enum
 from datetime import datetime, date, time  # noqa: I251
 from dlt.common import Decimal
 from dlt.common import json
+from dlt.common.schema.typing import TColumnType
 
 from dlt.common.libs.pydantic import (
     DltConfig,
@@ -724,3 +725,21 @@ def test_skip_json_types_when_skip_nested_types_is_true_and_field_is_not_pydanti
 
     assert "data_dictionary" not in schema
     assert "data_list" not in schema
+
+
+def test_typed_dict_by_python_version():
+    """when using typeddict in pydantic, it should be imported
+    from typing_extensions in python 3.11 and earlier and typing
+    in python 3.12 and later.
+    Here we test that this is properly set up in dlt.
+    """
+
+    class MyModel(BaseModel):
+        # TColumnType inherits from TypedDict
+        column_type: TColumnType
+
+    m = MyModel(column_type={"data_type": "text"})
+    assert m.column_type == {"data_type": "text"}
+
+    with pytest.raises(ValidationError):
+        m = MyModel(column_type={"data_type": "invalid_type"})  # type: ignore[typeddict-item]

tests/load/sources/sql_database/sql_source.py

@@ -1,6 +1,6 @@
 import random
 from copy import deepcopy
-from typing import Dict, List, TypedDict
+from typing import Dict, List
 from uuid import uuid4
 
 import mimesis
@@ -44,6 +44,7 @@ from sqlalchemy.dialects.postgresql import JSONB
 from dlt.common.pendulum import pendulum, timedelta
 from dlt.common.utils import chunks, uniq_id
 from dlt.sources.credentials import ConnectionStringCredentials
+from dlt.common.typing import TypedDict
 
 
 class SQLAlchemySourceDB:

tox.ini

@@ -6,6 +6,8 @@ banned-modules = datetime = use dlt.common.pendulum
                  decimal.Decimal = use dlt.common.Decimal
                  open = use dlt.common.open
                  pendulum = use dlt.common.pendulum
+                 typing.TypedDict = use dlt.common.typing.TypedDict
+                 typing-extensions.TypedDict = use dlt.common.typing.TypedDict
 extend-immutable-calls = dlt.sources.incremental
 per-file-ignores =
     tests/*: T20