mirror of
https://github.com/dlt-hub/dlt.git
synced 2025-12-17 19:31:30 +00:00
re-enable delta tests for read access and other small fixes (#2051)
* re-enable delta tests for read access add rudimentary tests for destination configs fixes small problems in test setup * fix cloudflare compat tests * cleans up env variables set by delta-rs * bumps to version 1.4.0 * fixes delta tests --------- Co-authored-by: Marcin Rudolf <rudolfix@rudolfix.org>
This commit is contained in:
David Scharf
GitHub
1
.github/workflows/test_destinations.yml
vendored
1
.github/workflows/test_destinations.yml
vendored
@@ -22,6 +22,7 @@ env:
|
||||
TESTS__R2_AWS_ACCESS_KEY_ID: a4950a5003b26f5a71ac97ef3848ff4c
|
||||
TESTS__R2_AWS_SECRET_ACCESS_KEY: ${{ secrets.CLOUDFLARE_R2_SECRET_ACCESS_KEY }}
|
||||
TESTS__R2_ENDPOINT_URL: https://9830548e4e4b582989be0811f2a0a97f.r2.cloudflarestorage.com
|
||||
TESTS__R2_REGION_NAME: us-east-1
|
||||
|
||||
# RUNTIME__SENTRY_DSN: https://6f6f7b6f8e0f458a89be4187603b55fe@o1061158.ingest.sentry.io/4504819859914752
|
||||
RUNTIME__LOG_LEVEL: ERROR
|
||||
|
||||
@@ -51,6 +51,8 @@ class AwsCredentialsWithoutDefaults(CredentialsConfiguration):
|
||||
|
||||
def to_object_store_rs_credentials(self) -> Dict[str, str]:
|
||||
# https://docs.rs/object_store/latest/object_store/aws
|
||||
# NOTE: delta rs will set the values below in env variables of the current process
|
||||
# https://github.com/delta-io/delta-rs/blob/bdf1c4e765ca457e49d4fa53335d42736220f57f/rust/src/storage/s3.rs#L257
|
||||
creds = cast(
|
||||
Dict[str, str],
|
||||
without_none(
|
||||
@@ -64,8 +66,8 @@ class AwsCredentialsWithoutDefaults(CredentialsConfiguration):
|
||||
),
|
||||
)
|
||||
|
||||
if "endpoint_url" not in creds: # AWS S3
|
||||
if "region" not in creds:
|
||||
if not self.endpoint_url: # AWS S3
|
||||
if not self.region_name:
|
||||
raise ObjectStoreRsCredentialsException(
|
||||
"`object_store` Rust crate requires AWS region when using AWS S3."
|
||||
)
|
||||
|
||||
118
poetry.lock
generated
118
poetry.lock
generated
@@ -1,4 +1,4 @@
|
||||
# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand.
|
||||
# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand.
|
||||
|
||||
[[package]]
|
||||
name = "about-time"
|
||||
@@ -2489,17 +2489,17 @@ files = [
|
||||
|
||||
[[package]]
|
||||
name = "deltalake"
|
||||
version = "0.19.1"
|
||||
version = "0.21.0"
|
||||
description = "Native Delta Lake Python binding based on delta-rs with Pandas integration"
|
||||
optional = true
|
||||
python-versions = ">=3.8"
|
||||
files = [
|
||||
{file = "deltalake-0.19.1-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:ddaaaa9c85a17791c3997cf320ac11dc1725d16cf4b6f0ff1b130853e7b56cd0"},
|
||||
{file = "deltalake-0.19.1-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:e0184d5a3f0d4f4f1fb992c3bdc8736329b78b6a4faf1a278109ec35d9945c1d"},
|
||||
{file = "deltalake-0.19.1-cp38-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec9d117fcf6c198f3d554be2f3a6291ca3838530650db236741ff48d4d47abb4"},
|
||||
{file = "deltalake-0.19.1-cp38-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:447ef721319ed15f7b5f6da507efd5fed0e6172e5ae55ac044d5b8fc9b812e47"},
|
||||
{file = "deltalake-0.19.1-cp38-abi3-win_amd64.whl", hash = "sha256:b15bc343a9f8f3de80fbedcebd5d9472b539eb0f538a71739c7fcf699089127e"},
|
||||
{file = "deltalake-0.19.1.tar.gz", hash = "sha256:5e09fabb221fb81e989c283c16278eaffb6e85706d98364abcda5c0c6ca73598"},
|
||||
{file = "deltalake-0.21.0-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:4b4a352da534a173d837906357c808cd36dd216974b9df2c3998acb98d04954a"},
|
||||
{file = "deltalake-0.21.0-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:2e7762af03fed0a67ce8f89b6b75356a74e485f52923c73a5c850c6c574d3481"},
|
||||
{file = "deltalake-0.21.0-cp38-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:02870db9dfe4694034fb209cb4822f85ba1a11885e353e11fd0dfb51b0af67d2"},
|
||||
{file = "deltalake-0.21.0-cp38-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:92ccd486243dbe17c45a6ca06b108351dfd07708d34e75405fcb13033c63d176"},
|
||||
{file = "deltalake-0.21.0-cp38-abi3-win_amd64.whl", hash = "sha256:dc84334a0f8df1f4f5c6ca9aaffbeffb896f43de6744d0c77c306ca8ba27041b"},
|
||||
{file = "deltalake-0.21.0.tar.gz", hash = "sha256:88f92cede44b2737431dbf86d43ed3b3c8cb73db56e99138aea1a6d93e9c6821"},
|
||||
]
|
||||
|
||||
[package.dependencies]
|
||||
@@ -3861,106 +3861,6 @@ files = [
|
||||
{file = "google_re2-1.1-4-cp39-cp39-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1f4d4f0823e8b2f6952a145295b1ff25245ce9bb136aff6fe86452e507d4c1dd"},
|
||||
{file = "google_re2-1.1-4-cp39-cp39-win32.whl", hash = "sha256:1afae56b2a07bb48cfcfefaa15ed85bae26a68f5dc7f9e128e6e6ea36914e847"},
|
||||
{file = "google_re2-1.1-4-cp39-cp39-win_amd64.whl", hash = "sha256:aa7d6d05911ab9c8adbf3c225a7a120ab50fd2784ac48f2f0d140c0b7afc2b55"},
|
||||
{file = "google_re2-1.1-5-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:222fc2ee0e40522de0b21ad3bc90ab8983be3bf3cec3d349c80d76c8bb1a4beb"},
|
||||
{file = "google_re2-1.1-5-cp310-cp310-macosx_12_0_x86_64.whl", hash = "sha256:d4763b0b9195b72132a4e7de8e5a9bf1f05542f442a9115aa27cfc2a8004f581"},
|
||||
{file = "google_re2-1.1-5-cp310-cp310-macosx_13_0_arm64.whl", hash = "sha256:209649da10c9d4a93d8a4d100ecbf9cc3b0252169426bec3e8b4ad7e57d600cf"},
|
||||
{file = "google_re2-1.1-5-cp310-cp310-macosx_13_0_x86_64.whl", hash = "sha256:68813aa333c1604a2df4a495b2a6ed065d7c8aebf26cc7e7abb5a6835d08353c"},
|
||||
{file = "google_re2-1.1-5-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:370a23ec775ad14e9d1e71474d56f381224dcf3e72b15d8ca7b4ad7dd9cd5853"},
|
||||
{file = "google_re2-1.1-5-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:14664a66a3ddf6bc9e56f401bf029db2d169982c53eff3f5876399104df0e9a6"},
|
||||
{file = "google_re2-1.1-5-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3ea3722cc4932cbcebd553b69dce1b4a73572823cff4e6a244f1c855da21d511"},
|
||||
{file = "google_re2-1.1-5-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e14bb264c40fd7c627ef5678e295370cd6ba95ca71d835798b6e37502fc4c690"},
|
||||
{file = "google_re2-1.1-5-cp310-cp310-win32.whl", hash = "sha256:39512cd0151ea4b3969c992579c79b423018b464624ae955be685fc07d94556c"},
|
||||
{file = "google_re2-1.1-5-cp310-cp310-win_amd64.whl", hash = "sha256:ac66537aa3bc5504320d922b73156909e3c2b6da19739c866502f7827b3f9fdf"},
|
||||
{file = "google_re2-1.1-5-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:5b5ea68d54890c9edb1b930dcb2658819354e5d3f2201f811798bbc0a142c2b4"},
|
||||
{file = "google_re2-1.1-5-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:33443511b6b83c35242370908efe2e8e1e7cae749c766b2b247bf30e8616066c"},
|
||||
{file = "google_re2-1.1-5-cp311-cp311-macosx_13_0_arm64.whl", hash = "sha256:413d77bdd5ba0bfcada428b4c146e87707452ec50a4091ec8e8ba1413d7e0619"},
|
||||
{file = "google_re2-1.1-5-cp311-cp311-macosx_13_0_x86_64.whl", hash = "sha256:5171686e43304996a34baa2abcee6f28b169806d0e583c16d55e5656b092a414"},
|
||||
{file = "google_re2-1.1-5-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:3b284db130283771558e31a02d8eb8fb756156ab98ce80035ae2e9e3a5f307c4"},
|
||||
{file = "google_re2-1.1-5-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:296e6aed0b169648dc4b870ff47bd34c702a32600adb9926154569ef51033f47"},
|
||||
{file = "google_re2-1.1-5-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:38d50e68ead374160b1e656bbb5d101f0b95fb4cc57f4a5c12100155001480c5"},
|
||||
{file = "google_re2-1.1-5-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2a0416a35921e5041758948bcb882456916f22845f66a93bc25070ef7262b72a"},
|
||||
{file = "google_re2-1.1-5-cp311-cp311-win32.whl", hash = "sha256:a1d59568bbb5de5dd56dd6cdc79907db26cce63eb4429260300c65f43469e3e7"},
|
||||
{file = "google_re2-1.1-5-cp311-cp311-win_amd64.whl", hash = "sha256:72f5a2f179648b8358737b2b493549370debd7d389884a54d331619b285514e3"},
|
||||
{file = "google_re2-1.1-5-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:cbc72c45937b1dc5acac3560eb1720007dccca7c9879138ff874c7f6baf96005"},
|
||||
{file = "google_re2-1.1-5-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:5fadd1417fbef7235fa9453dba4eb102e6e7d94b1e4c99d5fa3dd4e288d0d2ae"},
|
||||
{file = "google_re2-1.1-5-cp312-cp312-macosx_13_0_arm64.whl", hash = "sha256:040f85c63cc02696485b59b187a5ef044abe2f99b92b4fb399de40b7d2904ccc"},
|
||||
{file = "google_re2-1.1-5-cp312-cp312-macosx_13_0_x86_64.whl", hash = "sha256:64e3b975ee6d9bbb2420494e41f929c1a0de4bcc16d86619ab7a87f6ea80d6bd"},
|
||||
{file = "google_re2-1.1-5-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:8ee370413e00f4d828eaed0e83b8af84d7a72e8ee4f4bd5d3078bc741dfc430a"},
|
||||
{file = "google_re2-1.1-5-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:5b89383001079323f693ba592d7aad789d7a02e75adb5d3368d92b300f5963fd"},
|
||||
{file = "google_re2-1.1-5-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:63cb4fdfbbda16ae31b41a6388ea621510db82feb8217a74bf36552ecfcd50ad"},
|
||||
{file = "google_re2-1.1-5-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9ebedd84ae8be10b7a71a16162376fd67a2386fe6361ef88c622dcf7fd679daf"},
|
||||
{file = "google_re2-1.1-5-cp312-cp312-win32.whl", hash = "sha256:c8e22d1692bc2c81173330c721aff53e47ffd3c4403ff0cd9d91adfd255dd150"},
|
||||
{file = "google_re2-1.1-5-cp312-cp312-win_amd64.whl", hash = "sha256:5197a6af438bb8c4abda0bbe9c4fbd6c27c159855b211098b29d51b73e4cbcf6"},
|
||||
{file = "google_re2-1.1-5-cp38-cp38-macosx_12_0_arm64.whl", hash = "sha256:b6727e0b98417e114b92688ad2aa256102ece51f29b743db3d831df53faf1ce3"},
|
||||
{file = "google_re2-1.1-5-cp38-cp38-macosx_12_0_x86_64.whl", hash = "sha256:711e2b6417eb579c61a4951029d844f6b95b9b373b213232efd413659889a363"},
|
||||
{file = "google_re2-1.1-5-cp38-cp38-macosx_13_0_arm64.whl", hash = "sha256:71ae8b3df22c5c154c8af0f0e99d234a450ef1644393bc2d7f53fc8c0a1e111c"},
|
||||
{file = "google_re2-1.1-5-cp38-cp38-macosx_13_0_x86_64.whl", hash = "sha256:94a04e214bc521a3807c217d50cf099bbdd0c0a80d2d996c0741dbb995b5f49f"},
|
||||
{file = "google_re2-1.1-5-cp38-cp38-macosx_14_0_arm64.whl", hash = "sha256:a770f75358508a9110c81a1257721f70c15d9bb592a2fb5c25ecbd13566e52a5"},
|
||||
{file = "google_re2-1.1-5-cp38-cp38-macosx_14_0_x86_64.whl", hash = "sha256:07c9133357f7e0b17c6694d5dcb82e0371f695d7c25faef2ff8117ef375343ff"},
|
||||
{file = "google_re2-1.1-5-cp38-cp38-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:204ca6b1cf2021548f4a9c29ac015e0a4ab0a7b6582bf2183d838132b60c8fda"},
|
||||
{file = "google_re2-1.1-5-cp38-cp38-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f0b95857c2c654f419ca684ec38c9c3325c24e6ba7d11910a5110775a557bb18"},
|
||||
{file = "google_re2-1.1-5-cp38-cp38-win32.whl", hash = "sha256:347ac770e091a0364e822220f8d26ab53e6fdcdeaec635052000845c5a3fb869"},
|
||||
{file = "google_re2-1.1-5-cp38-cp38-win_amd64.whl", hash = "sha256:ec32bb6de7ffb112a07d210cf9f797b7600645c2d5910703fa07f456dd2150e0"},
|
||||
{file = "google_re2-1.1-5-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:eb5adf89060f81c5ff26c28e261e6b4997530a923a6093c9726b8dec02a9a326"},
|
||||
{file = "google_re2-1.1-5-cp39-cp39-macosx_12_0_x86_64.whl", hash = "sha256:a22630c9dd9ceb41ca4316bccba2643a8b1d5c198f21c00ed5b50a94313aaf10"},
|
||||
{file = "google_re2-1.1-5-cp39-cp39-macosx_13_0_arm64.whl", hash = "sha256:544dc17fcc2d43ec05f317366375796351dec44058e1164e03c3f7d050284d58"},
|
||||
{file = "google_re2-1.1-5-cp39-cp39-macosx_13_0_x86_64.whl", hash = "sha256:19710af5ea88751c7768575b23765ce0dfef7324d2539de576f75cdc319d6654"},
|
||||
{file = "google_re2-1.1-5-cp39-cp39-macosx_14_0_arm64.whl", hash = "sha256:f82995a205e08ad896f4bd5ce4847c834fab877e1772a44e5f262a647d8a1dec"},
|
||||
{file = "google_re2-1.1-5-cp39-cp39-macosx_14_0_x86_64.whl", hash = "sha256:63533c4d58da9dc4bc040250f1f52b089911699f0368e0e6e15f996387a984ed"},
|
||||
{file = "google_re2-1.1-5-cp39-cp39-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:79e00fcf0cb04ea35a22b9014712d448725ce4ddc9f08cc818322566176ca4b0"},
|
||||
{file = "google_re2-1.1-5-cp39-cp39-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bc41afcefee2da6c4ed883a93d7f527c4b960cd1d26bbb0020a7b8c2d341a60a"},
|
||||
{file = "google_re2-1.1-5-cp39-cp39-win32.whl", hash = "sha256:486730b5e1f1c31b0abc6d80abe174ce4f1188fe17d1b50698f2bf79dc6e44be"},
|
||||
{file = "google_re2-1.1-5-cp39-cp39-win_amd64.whl", hash = "sha256:4de637ca328f1d23209e80967d1b987d6b352cd01b3a52a84b4d742c69c3da6c"},
|
||||
{file = "google_re2-1.1-6-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:621e9c199d1ff0fdb2a068ad450111a84b3bf14f96dfe5a8a7a0deae5f3f4cce"},
|
||||
{file = "google_re2-1.1-6-cp310-cp310-macosx_12_0_x86_64.whl", hash = "sha256:220acd31e7dde95373f97c3d1f3b3bd2532b38936af28b1917ee265d25bebbf4"},
|
||||
{file = "google_re2-1.1-6-cp310-cp310-macosx_13_0_arm64.whl", hash = "sha256:db34e1098d164f76251a6ece30e8f0ddfd65bb658619f48613ce71acb3f9cbdb"},
|
||||
{file = "google_re2-1.1-6-cp310-cp310-macosx_13_0_x86_64.whl", hash = "sha256:5152bac41d8073977582f06257219541d0fc46ad99b0bbf30e8f60198a43b08c"},
|
||||
{file = "google_re2-1.1-6-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:6191294799e373ee1735af91f55abd23b786bdfd270768a690d9d55af9ea1b0d"},
|
||||
{file = "google_re2-1.1-6-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:070cbafbb4fecbb02e98feb28a1eb292fb880f434d531f38cc33ee314b521f1f"},
|
||||
{file = "google_re2-1.1-6-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8437d078b405a59a576cbed544490fe041140f64411f2d91012e8ec05ab8bf86"},
|
||||
{file = "google_re2-1.1-6-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f00f9a9af8896040e37896d9b9fc409ad4979f1ddd85bb188694a7d95ddd1164"},
|
||||
{file = "google_re2-1.1-6-cp310-cp310-win32.whl", hash = "sha256:df26345f229a898b4fd3cafd5f82259869388cee6268fc35af16a8e2293dd4e5"},
|
||||
{file = "google_re2-1.1-6-cp310-cp310-win_amd64.whl", hash = "sha256:3665d08262c57c9b28a5bdeb88632ad792c4e5f417e5645901695ab2624f5059"},
|
||||
{file = "google_re2-1.1-6-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:b26b869d8aa1d8fe67c42836bf3416bb72f444528ee2431cfb59c0d3e02c6ce3"},
|
||||
{file = "google_re2-1.1-6-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:41fd4486c57dea4f222a6bb7f1ff79accf76676a73bdb8da0fcbd5ba73f8da71"},
|
||||
{file = "google_re2-1.1-6-cp311-cp311-macosx_13_0_arm64.whl", hash = "sha256:0ee378e2e74e25960070c338c28192377c4dd41e7f4608f2688064bd2badc41e"},
|
||||
{file = "google_re2-1.1-6-cp311-cp311-macosx_13_0_x86_64.whl", hash = "sha256:a00cdbf662693367b36d075b29feb649fd7ee1b617cf84f85f2deebeda25fc64"},
|
||||
{file = "google_re2-1.1-6-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:4c09455014217a41499432b8c8f792f25f3df0ea2982203c3a8c8ca0e7895e69"},
|
||||
{file = "google_re2-1.1-6-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:6501717909185327935c7945e23bb5aa8fc7b6f237b45fe3647fa36148662158"},
|
||||
{file = "google_re2-1.1-6-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3510b04790355f199e7861c29234081900e1e1cbf2d1484da48aa0ba6d7356ab"},
|
||||
{file = "google_re2-1.1-6-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8c0e64c187ca406764f9e9ad6e750d62e69ed8f75bf2e865d0bfbc03b642361c"},
|
||||
{file = "google_re2-1.1-6-cp311-cp311-win32.whl", hash = "sha256:2a199132350542b0de0f31acbb3ca87c3a90895d1d6e5235f7792bb0af02e523"},
|
||||
{file = "google_re2-1.1-6-cp311-cp311-win_amd64.whl", hash = "sha256:83bdac8ceaece8a6db082ea3a8ba6a99a2a1ee7e9f01a9d6d50f79c6f251a01d"},
|
||||
{file = "google_re2-1.1-6-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:81985ff894cd45ab5a73025922ac28c0707759db8171dd2f2cc7a0e856b6b5ad"},
|
||||
{file = "google_re2-1.1-6-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:5635af26065e6b45456ccbea08674ae2ab62494008d9202df628df3b267bc095"},
|
||||
{file = "google_re2-1.1-6-cp312-cp312-macosx_13_0_arm64.whl", hash = "sha256:813b6f04de79f4a8fdfe05e2cb33e0ccb40fe75d30ba441d519168f9d958bd54"},
|
||||
{file = "google_re2-1.1-6-cp312-cp312-macosx_13_0_x86_64.whl", hash = "sha256:5ec2f5332ad4fd232c3f2d6748c2c7845ccb66156a87df73abcc07f895d62ead"},
|
||||
{file = "google_re2-1.1-6-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:5a687b3b32a6cbb731647393b7c4e3fde244aa557f647df124ff83fb9b93e170"},
|
||||
{file = "google_re2-1.1-6-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:39a62f9b3db5d3021a09a47f5b91708b64a0580193e5352751eb0c689e4ad3d7"},
|
||||
{file = "google_re2-1.1-6-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ca0f0b45d4a1709cbf5d21f355e5809ac238f1ee594625a1e5ffa9ff7a09eb2b"},
|
||||
{file = "google_re2-1.1-6-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a64b3796a7a616c7861247bd061c9a836b5caf0d5963e5ea8022125601cf7b09"},
|
||||
{file = "google_re2-1.1-6-cp312-cp312-win32.whl", hash = "sha256:32783b9cb88469ba4cd9472d459fe4865280a6b1acdad4480a7b5081144c4eb7"},
|
||||
{file = "google_re2-1.1-6-cp312-cp312-win_amd64.whl", hash = "sha256:259ff3fd2d39035b9cbcbf375995f83fa5d9e6a0c5b94406ff1cc168ed41d6c6"},
|
||||
{file = "google_re2-1.1-6-cp38-cp38-macosx_12_0_arm64.whl", hash = "sha256:e4711bcffe190acd29104d8ecfea0c0e42b754837de3fb8aad96e6cc3c613cdc"},
|
||||
{file = "google_re2-1.1-6-cp38-cp38-macosx_12_0_x86_64.whl", hash = "sha256:4d081cce43f39c2e813fe5990e1e378cbdb579d3f66ded5bade96130269ffd75"},
|
||||
{file = "google_re2-1.1-6-cp38-cp38-macosx_13_0_arm64.whl", hash = "sha256:4f123b54d48450d2d6b14d8fad38e930fb65b5b84f1b022c10f2913bd956f5b5"},
|
||||
{file = "google_re2-1.1-6-cp38-cp38-macosx_13_0_x86_64.whl", hash = "sha256:e1928b304a2b591a28eb3175f9db7f17c40c12cf2d4ec2a85fdf1cc9c073ff91"},
|
||||
{file = "google_re2-1.1-6-cp38-cp38-macosx_14_0_arm64.whl", hash = "sha256:3a69f76146166aec1173003c1f547931bdf288c6b135fda0020468492ac4149f"},
|
||||
{file = "google_re2-1.1-6-cp38-cp38-macosx_14_0_x86_64.whl", hash = "sha256:fc08c388f4ebbbca345e84a0c56362180d33d11cbe9ccfae663e4db88e13751e"},
|
||||
{file = "google_re2-1.1-6-cp38-cp38-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b057adf38ce4e616486922f2f47fc7d19c827ba0a7f69d540a3664eba2269325"},
|
||||
{file = "google_re2-1.1-6-cp38-cp38-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4138c0b933ab099e96f5d8defce4486f7dfd480ecaf7f221f2409f28022ccbc5"},
|
||||
{file = "google_re2-1.1-6-cp38-cp38-win32.whl", hash = "sha256:9693e45b37b504634b1abbf1ee979471ac6a70a0035954592af616306ab05dd6"},
|
||||
{file = "google_re2-1.1-6-cp38-cp38-win_amd64.whl", hash = "sha256:5674d437baba0ea287a5a7f8f81f24265d6ae8f8c09384e2ef7b6f84b40a7826"},
|
||||
{file = "google_re2-1.1-6-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:7783137cb2e04f458a530c6d0ee9ef114815c1d48b9102f023998c371a3b060e"},
|
||||
{file = "google_re2-1.1-6-cp39-cp39-macosx_12_0_x86_64.whl", hash = "sha256:a49b7153935e7a303675f4deb5f5d02ab1305adefc436071348706d147c889e0"},
|
||||
{file = "google_re2-1.1-6-cp39-cp39-macosx_13_0_arm64.whl", hash = "sha256:a96a8bb309182090704593c60bdb369a2756b38fe358bbf0d40ddeb99c71769f"},
|
||||
{file = "google_re2-1.1-6-cp39-cp39-macosx_13_0_x86_64.whl", hash = "sha256:dff3d4be9f27ef8ec3705eed54f19ef4ab096f5876c15fe011628c69ba3b561c"},
|
||||
{file = "google_re2-1.1-6-cp39-cp39-macosx_14_0_arm64.whl", hash = "sha256:40f818b0b39e26811fa677978112a8108269977fdab2ba0453ac4363c35d9e66"},
|
||||
{file = "google_re2-1.1-6-cp39-cp39-macosx_14_0_x86_64.whl", hash = "sha256:8a7e53538cdb40ef4296017acfbb05cab0c19998be7552db1cfb85ba40b171b9"},
|
||||
{file = "google_re2-1.1-6-cp39-cp39-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6ee18e7569fb714e5bb8c42809bf8160738637a5e71ed5a4797757a1fb4dc4de"},
|
||||
{file = "google_re2-1.1-6-cp39-cp39-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1cda4f6d1a7d5b43ea92bc395f23853fba0caf8b1e1efa6e8c48685f912fcb89"},
|
||||
{file = "google_re2-1.1-6-cp39-cp39-win32.whl", hash = "sha256:6a9cdbdc36a2bf24f897be6a6c85125876dc26fea9eb4247234aec0decbdccfd"},
|
||||
{file = "google_re2-1.1-6-cp39-cp39-win_amd64.whl", hash = "sha256:73f646cecfad7cc5b4330b4192c25f2e29730a3b8408e089ffd2078094208196"},
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -10392,4 +10292,4 @@ weaviate = ["weaviate-client"]
|
||||
[metadata]
|
||||
lock-version = "2.0"
|
||||
python-versions = ">=3.8.1,<3.13"
|
||||
content-hash = "aefcc6fc97be767f995183ea8b5bf0c3ee2388a4ca884cfb2192311f696f24f4"
|
||||
content-hash = "f932b4e28b8d08489ca58a67e6844006a21dc73d453c3eae8469cd57760bb891"
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[tool.poetry]
|
||||
name = "dlt"
|
||||
version = "1.3.1a2"
|
||||
version = "1.4.0"
|
||||
description = "dlt is an open-source python-first scalable data loading library that does not require any backend to run."
|
||||
authors = ["dltHub Inc. <services@dlthub.com>"]
|
||||
maintainers = [ "Marcin Rudolf <marcin@dlthub.com>", "Adrian Brudaru <adrian@dlthub.com>", "Anton Burnashev <anton@dlthub.com>", "David Scharf <david@dlthub.com>" ]
|
||||
@@ -83,7 +83,7 @@ clickhouse-driver = { version = ">=0.2.7", optional = true }
|
||||
clickhouse-connect = { version = ">=0.7.7", optional = true }
|
||||
lancedb = { version = ">=0.8.2", optional = true, markers = "python_version >= '3.9'", allow-prereleases = true }
|
||||
tantivy = { version = ">= 0.22.0", optional = true }
|
||||
deltalake = { version = ">=0.19.0", optional = true }
|
||||
deltalake = { version = ">=0.21.0", optional = true }
|
||||
sqlalchemy = { version = ">=1.4", optional = true }
|
||||
alembic = {version = ">1.10.0", optional = true}
|
||||
paramiko = {version = ">=3.3.0", optional = true}
|
||||
|
||||
@@ -43,12 +43,14 @@ if all(driver not in ALL_FILESYSTEM_DRIVERS for driver in ("az", "s3", "gs", "r2
|
||||
)
|
||||
|
||||
|
||||
FS_CREDS: Dict[str, Any] = dlt.secrets.get("destination.filesystem.credentials")
|
||||
if FS_CREDS is None:
|
||||
pytest.skip(
|
||||
msg="`destination.filesystem.credentials` must be configured for these tests.",
|
||||
allow_module_level=True,
|
||||
)
|
||||
@pytest.fixture
|
||||
def fs_creds() -> Dict[str, Any]:
|
||||
creds: Dict[str, Any] = dlt.secrets.get("destination.filesystem.credentials")
|
||||
if creds is None:
|
||||
pytest.skip(
|
||||
msg="`destination.filesystem.credentials` must be configured for these tests.",
|
||||
)
|
||||
return creds
|
||||
|
||||
|
||||
def can_connect(bucket_url: str, object_store_rs_credentials: Dict[str, str]) -> bool:
|
||||
@@ -62,7 +64,7 @@ def can_connect(bucket_url: str, object_store_rs_credentials: Dict[str, str]) ->
|
||||
storage_options=object_store_rs_credentials,
|
||||
)
|
||||
except TableNotFoundError:
|
||||
# this error implies the connection was succesful
|
||||
# this error implies the connection was successful
|
||||
# there is no Delta table at `bucket_url`
|
||||
return True
|
||||
return False
|
||||
@@ -71,7 +73,7 @@ def can_connect(bucket_url: str, object_store_rs_credentials: Dict[str, str]) ->
|
||||
@pytest.mark.parametrize(
|
||||
"driver", [driver for driver in ALL_FILESYSTEM_DRIVERS if driver in ("az")]
|
||||
)
|
||||
def test_azure_object_store_rs_credentials(driver: str) -> None:
|
||||
def test_azure_object_store_rs_credentials(driver: str, fs_creds: Dict[str, Any]) -> None:
|
||||
creds: AnyAzureCredentials
|
||||
|
||||
creds = AzureServicePrincipalCredentialsWithoutDefaults(
|
||||
@@ -81,8 +83,8 @@ def test_azure_object_store_rs_credentials(driver: str) -> None:
|
||||
|
||||
# without SAS token
|
||||
creds = AzureCredentialsWithoutDefaults(
|
||||
azure_storage_account_name=FS_CREDS["azure_storage_account_name"],
|
||||
azure_storage_account_key=FS_CREDS["azure_storage_account_key"],
|
||||
azure_storage_account_name=fs_creds["azure_storage_account_name"],
|
||||
azure_storage_account_key=fs_creds["azure_storage_account_key"],
|
||||
)
|
||||
assert creds.azure_storage_sas_token is None
|
||||
assert can_connect(AZ_BUCKET, creds.to_object_store_rs_credentials())
|
||||
@@ -96,10 +98,9 @@ def test_azure_object_store_rs_credentials(driver: str) -> None:
|
||||
@pytest.mark.parametrize(
|
||||
"driver", [driver for driver in ALL_FILESYSTEM_DRIVERS if driver in ("s3", "r2")]
|
||||
)
|
||||
def test_aws_object_store_rs_credentials(driver: str) -> None:
|
||||
def test_aws_object_store_rs_credentials(driver: str, fs_creds: Dict[str, Any]) -> None:
|
||||
creds: AwsCredentialsWithoutDefaults
|
||||
|
||||
fs_creds = FS_CREDS
|
||||
if driver == "r2":
|
||||
fs_creds = R2_BUCKET_CONFIG["credentials"] # type: ignore[assignment]
|
||||
|
||||
@@ -177,16 +178,16 @@ def test_aws_object_store_rs_credentials(driver: str) -> None:
|
||||
@pytest.mark.parametrize(
|
||||
"driver", [driver for driver in ALL_FILESYSTEM_DRIVERS if driver in ("gs")]
|
||||
)
|
||||
def test_gcp_object_store_rs_credentials(driver) -> None:
|
||||
def test_gcp_object_store_rs_credentials(driver, fs_creds: Dict[str, Any]) -> None:
|
||||
creds: GcpCredentials
|
||||
|
||||
# GcpServiceAccountCredentialsWithoutDefaults
|
||||
creds = GcpServiceAccountCredentialsWithoutDefaults(
|
||||
project_id=FS_CREDS["project_id"],
|
||||
private_key=FS_CREDS["private_key"],
|
||||
project_id=fs_creds["project_id"],
|
||||
private_key=fs_creds["private_key"],
|
||||
# private_key_id must be configured in order for data lake to work
|
||||
private_key_id=FS_CREDS["private_key_id"],
|
||||
client_email=FS_CREDS["client_email"],
|
||||
private_key_id=fs_creds["private_key_id"],
|
||||
client_email=fs_creds["client_email"],
|
||||
)
|
||||
assert can_connect(GCS_BUCKET, creds.to_object_store_rs_credentials())
|
||||
|
||||
|
||||
@@ -120,6 +120,7 @@ R2_BUCKET_CONFIG = dict(
|
||||
aws_access_key_id=dlt.config.get("tests.r2_aws_access_key_id", str),
|
||||
aws_secret_access_key=dlt.config.get("tests.r2_aws_secret_access_key", str),
|
||||
endpoint_url=dlt.config.get("tests.r2_endpoint_url", str),
|
||||
region_name=dlt.config.get("tests.r2_region_name", str),
|
||||
),
|
||||
)
|
||||
|
||||
@@ -374,6 +375,7 @@ def destinations_configs(
|
||||
file_format="parquet",
|
||||
bucket_url=AWS_BUCKET,
|
||||
supports_dbt=False,
|
||||
extra_info="minio",
|
||||
)
|
||||
]
|
||||
destination_configs += [
|
||||
@@ -439,7 +441,7 @@ def destinations_configs(
|
||||
file_format="jsonl",
|
||||
bucket_url=AWS_BUCKET,
|
||||
stage_name="PUBLIC.dlt_s3_stage",
|
||||
extra_info="s3-integration",
|
||||
extra_info="s3-integration-public-stage",
|
||||
),
|
||||
DestinationTestConfiguration(
|
||||
destination_type="snowflake",
|
||||
@@ -515,13 +517,6 @@ def destinations_configs(
|
||||
bucket_url=AWS_BUCKET,
|
||||
extra_info="s3-authorization",
|
||||
),
|
||||
DestinationTestConfiguration(
|
||||
destination_type="dremio",
|
||||
staging=filesystem(destination_name="minio"),
|
||||
file_format="parquet",
|
||||
bucket_url=AWS_BUCKET,
|
||||
supports_dbt=False,
|
||||
),
|
||||
]
|
||||
|
||||
if all_staging_configs:
|
||||
@@ -612,6 +607,7 @@ def destinations_configs(
|
||||
extra_info=bucket + "-delta",
|
||||
table_format="delta",
|
||||
supports_merge=True,
|
||||
file_format="parquet",
|
||||
env_vars=(
|
||||
{
|
||||
"DESTINATION__FILESYSTEM__DELTALAKE_STORAGE_OPTIONS": (
|
||||
|
||||
1
tests/tests/__init__.py
Normal file
1
tests/tests/__init__.py
Normal file
@@ -0,0 +1 @@
|
||||
"""Tests for our test helpers"""
|
||||
0
tests/tests/load/__init__.py
Normal file
0
tests/tests/load/__init__.py
Normal file
49
tests/tests/load/test_utils.py
Normal file
49
tests/tests/load/test_utils.py
Normal file
@@ -0,0 +1,49 @@
|
||||
# NOTE: these should be run with all destinations enabled
|
||||
# NOTE: these are very rudimentary tests, they should be extended
|
||||
|
||||
from typing import Any
|
||||
|
||||
from tests.load.utils import destinations_configs, DEFAULT_BUCKETS, DestinationTestConfiguration
|
||||
|
||||
|
||||
def test_empty_destinations_configs():
|
||||
configs = destinations_configs()
|
||||
assert len(configs) == 0
|
||||
|
||||
|
||||
def _assert_name_uniqueness(configs: Any) -> None:
|
||||
identifiers = [config[0][0].name for config in configs]
|
||||
print(identifiers)
|
||||
assert len(identifiers) == len(set(identifiers)), "Identifier uniqueness violated"
|
||||
|
||||
|
||||
def test_enable_filesystem_configs():
|
||||
# enable local filesystem configs
|
||||
configs = destinations_configs(local_filesystem_configs=True)
|
||||
_assert_name_uniqueness(configs)
|
||||
assert len(configs) == 3
|
||||
|
||||
# enable all buckets configs
|
||||
configs = destinations_configs(all_buckets_filesystem_configs=True)
|
||||
_assert_name_uniqueness(configs)
|
||||
assert len(configs) == len(DEFAULT_BUCKETS) == 7 # hardcoded for now
|
||||
|
||||
# enable with delta tables
|
||||
configs = destinations_configs(
|
||||
all_buckets_filesystem_configs=True, table_format_filesystem_configs=True
|
||||
)
|
||||
_assert_name_uniqueness(configs)
|
||||
assert (
|
||||
len(configs) == len(DEFAULT_BUCKETS) * 2 == 7 * 2
|
||||
) # we have delta now, so double the expected amount
|
||||
|
||||
|
||||
def test_uniqueness_of_names():
|
||||
configs = destinations_configs(
|
||||
default_sql_configs=True,
|
||||
default_vector_configs=True,
|
||||
default_staging_configs=True,
|
||||
all_staging_configs=True,
|
||||
all_buckets_filesystem_configs=True,
|
||||
)
|
||||
_assert_name_uniqueness(configs)
|
||||
@@ -157,9 +157,27 @@ def autouse_test_storage() -> FileStorage:
|
||||
@pytest.fixture(scope="function", autouse=True)
|
||||
def preserve_environ() -> Iterator[None]:
|
||||
saved_environ = environ.copy()
|
||||
yield
|
||||
environ.clear()
|
||||
environ.update(saved_environ)
|
||||
# delta-rs sets those keys without updating environ and there's no
|
||||
# method to refresh environ
|
||||
known_environ = {
|
||||
key_: saved_environ.get(key_)
|
||||
for key_ in [
|
||||
"AWS_ACCESS_KEY_ID",
|
||||
"AWS_SECRET_ACCESS_KEY",
|
||||
"AWS_REGION",
|
||||
"AWS_SESSION_TOKEN",
|
||||
]
|
||||
}
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
environ.clear()
|
||||
environ.update(saved_environ)
|
||||
for key_, value_ in known_environ.items():
|
||||
if value_ is not None or key_ not in environ:
|
||||
environ[key_] = value_ or ""
|
||||
else:
|
||||
del environ[key_]
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
|
||||
Reference in New Issue
Block a user