From b51909db325e364d08651647988abf2ed9e16deb Mon Sep 17 00:00:00 2001 From: Frithjof Winkelmann Date: 2025年7月16日 13:07:46 +0200 Subject: [PATCH] Allow cloud paths --- dataframely/_path.py | 21 +++ dataframely/collection.py | 36 ++-- dataframely/schema.py | 22 ++- pixi.lock | 50 +++++ pixi.toml | 1 + tests/collection/test_read_write_parquet.py | 198 ++++++++++++++++---- tests/schema/test_read_write_parquet.py | 153 +++++++++++---- 7 files changed, 390 insertions(+), 91 deletions(-) create mode 100644 dataframely/_path.py diff --git a/dataframely/_path.py b/dataframely/_path.py new file mode 100644 index 00000000..c73d84ab --- /dev/null +++ b/dataframely/_path.py @@ -0,0 +1,21 @@ +# Copyright (c) QuantCo 2025-2025 +# SPDX-License-Identifier: BSD-3-Clause + +from pathlib import Path +from typing import TypeVar + +from cloudpathlib import CloudPath +from cloudpathlib.local.localpath import LocalPath + +T = TypeVar("T") + + +def handle_cloud_path(path: T | CloudPath) -> T | Path: + if not isinstance(path, (CloudPath)): + return path + + if isinstance(path, LocalPath): + return path.client._cloud_path_to_local(path) + + # TODO: Handle actual cloud paths here. Will need to also return credentials/storage_options dict + raise ValueError("Unsupported path type. Expected LocalPath or CloudPath.") diff --git a/dataframely/collection.py b/dataframely/collection.py index 3580e6fb..8b67ce3c 100644 --- a/dataframely/collection.py +++ b/dataframely/collection.py @@ -12,6 +12,9 @@ import polars as pl import polars.exceptions as plexc +from cloudpathlib import CloudPath + +from dataframely._path import handle_cloud_path from ._base_collection import BaseCollection, CollectionMember from ._filter import Filter @@ -631,7 +634,7 @@ def serialize(cls) -> str: # ---------------------------------- PERSISTENCE --------------------------------- # - def write_parquet(self, directory: str | Path, **kwargs: Any) -> None: + def write_parquet(self, directory: str | Path | CloudPath, **kwargs: Any) -> None: """Write the members of this collection to parquet files in a directory. This method writes one parquet file per member into the provided directory. @@ -654,7 +657,7 @@ def write_parquet(self, directory: str | Path, **kwargs: Any) -> None: """ self._to_parquet(directory, sink=False, **kwargs) - def sink_parquet(self, directory: str | Path, **kwargs: Any) -> None: + def sink_parquet(self, directory: str | Path | CloudPath, **kwargs: Any) -> None: """Stream the members of this collection into parquet files in a directory. This method writes one parquet file per member into the provided directory. @@ -677,10 +680,12 @@ def sink_parquet(self, directory: str | Path, **kwargs: Any) -> None: """ self._to_parquet(directory, sink=True, **kwargs) - def _to_parquet(self, directory: str | Path, *, sink: bool, **kwargs: Any) -> None: + def _to_parquet( + self, directory: str | Path | CloudPath, *, sink: bool, **kwargs: Any + ) -> None: path = Path(directory) if isinstance(directory, str) else directory path.mkdir(parents=True, exist_ok=True) - with open(path / "schema.json", "w") as f: + with (path / "schema.json").open("w") as f: f.write(self.serialize()) member_schemas = self.member_schemas() @@ -704,7 +709,7 @@ def _to_parquet(self, directory: str | Path, *, sink: bool, **kwargs: Any) -> No @classmethod def read_parquet( cls, - directory: str | Path, + directory: str | Path | CloudPath, *, validation: Validation = "warn", **kwargs: Any, @@ -751,9 +756,10 @@ def read_parquet( Be aware that this method suffers from the same limitations as :meth:`serialize`. """ - path = Path(directory) - data = cls._from_parquet(path, scan=True, **kwargs) - if not cls._requires_validation_for_reading_parquets(path, validation): + if not isinstance(directory, Path | CloudPath): + directory = Path(directory) + data = cls._from_parquet(directory, scan=True, **kwargs) + if not cls._requires_validation_for_reading_parquets(directory, validation): cls._validate_input_keys(data) return cls._init(data) return cls.validate(data, cast=True) @@ -761,7 +767,7 @@ def read_parquet( @classmethod def scan_parquet( cls, - directory: str | Path, + directory: str | Path | CloudPath, *, validation: Validation = "warn", **kwargs: Any, @@ -812,17 +818,19 @@ def scan_parquet( Be aware that this method suffers from the same limitations as :meth:`serialize`. """ - path = Path(directory) - data = cls._from_parquet(path, scan=True, **kwargs) - if not cls._requires_validation_for_reading_parquets(path, validation): + if not isinstance(directory, Path | CloudPath): + directory = Path(directory) + data = cls._from_parquet(directory, scan=True, **kwargs) + if not cls._requires_validation_for_reading_parquets(directory, validation): cls._validate_input_keys(data) return cls._init(data) return cls.validate(data, cast=True) @classmethod def _from_parquet( - cls, path: Path, scan: bool, **kwargs: Any + cls, path: Path | CloudPath, scan: bool, **kwargs: Any ) -> dict[str, pl.LazyFrame]: + path = handle_cloud_path(path) data = {} for key in cls.members(): if (source_path := cls._member_source_path(path, key)) is not None: @@ -846,7 +854,7 @@ def _member_source_path(cls, base_path: Path, name: str) -> Path | None: @classmethod def _requires_validation_for_reading_parquets( cls, - directory: Path, + directory: Path | CloudPath, validation: Validation, ) -> bool: if validation == "skip": diff --git a/dataframely/schema.py b/dataframely/schema.py index c6cd78ab..ae6d8700 100644 --- a/dataframely/schema.py +++ b/dataframely/schema.py @@ -14,8 +14,11 @@ import polars as pl import polars.exceptions as plexc import polars.selectors as cs +from cloudpathlib import CloudPath from polars._typing import FileSource, PartitioningScheme +from dataframely._path import handle_cloud_path + from ._base_schema import BaseSchema from ._compat import pa, sa from ._rule import Rule, rule_from_dict, with_evaluation_rules @@ -692,7 +695,11 @@ def _as_dict(cls) -> dict[str, Any]: @classmethod def write_parquet( - cls, df: DataFrame[Self], /, file: str | Path | IO[bytes], **kwargs: Any + cls, + df: DataFrame[Self], + /, + file: str | Path | IO[bytes] | CloudPath, + **kwargs: Any, ) -> None: """Write a typed data frame with this schema to a parquet file. @@ -714,6 +721,7 @@ def write_parquet( :meth:`serialize`. """ metadata = kwargs.pop("metadata", {}) + file = handle_cloud_path(file) df.write_parquet( file, metadata={**metadata, SCHEMA_METADATA_KEY: cls.serialize()}, **kwargs ) @@ -723,7 +731,7 @@ def sink_parquet( cls, lf: LazyFrame[Self], /, - file: str | Path | IO[bytes] | PartitioningScheme, + file: str | Path | IO[bytes] | PartitioningScheme | CloudPath, **kwargs: Any, ) -> None: """Stream a typed lazy frame with this schema to a parquet file. @@ -745,6 +753,7 @@ def sink_parquet( :meth:`serialize`. """ metadata = kwargs.pop("metadata", {}) + file = handle_cloud_path(file) lf.sink_parquet( file, metadata={**metadata, SCHEMA_METADATA_KEY: cls.serialize()}, **kwargs ) @@ -752,7 +761,7 @@ def sink_parquet( @classmethod def read_parquet( cls, - source: FileSource, + source: FileSource | CloudPath, *, validation: Validation = "warn", **kwargs: Any, @@ -796,6 +805,7 @@ def read_parquet( Be aware that this method suffers from the same limitations as :meth:`serialize`. """ + source = handle_cloud_path(source) if not cls._requires_validation_for_reading_parquet(source, validation): return pl.read_parquet(source, **kwargs) # type: ignore return cls.validate(pl.read_parquet(source, **kwargs), cast=True) @@ -803,7 +813,7 @@ def read_parquet( @classmethod def scan_parquet( cls, - source: FileSource, + source: FileSource | CloudPath, *, validation: Validation = "warn", **kwargs: Any, @@ -852,6 +862,7 @@ def scan_parquet( Be aware that this method suffers from the same limitations as :meth:`serialize`. """ + source = handle_cloud_path(source) if not cls._requires_validation_for_reading_parquet(source, validation): return pl.scan_parquet(source, **kwargs) # type: ignore return cls.validate(pl.read_parquet(source, **kwargs), cast=True).lazy() @@ -962,7 +973,7 @@ def _rules_match(lhs: dict[str, Rule], rhs: dict[str, Rule]) -> bool: def read_parquet_metadata_schema( - source: str | Path | IO[bytes] | bytes, + source: str | Path | IO[bytes] | bytes | CloudPath, ) -> type[Schema] | None: """Read a dataframely schema from the metadata of a parquet file. @@ -973,6 +984,7 @@ def read_parquet_metadata_schema( The schema that was serialized to the metadata or ``None`` if no schema metadata is found. """ + source = handle_cloud_path(source) metadata = pl.read_parquet_metadata(source) if (schema_metadata := metadata.get(SCHEMA_METADATA_KEY)) is not None: return deserialize_schema(schema_metadata) diff --git a/pixi.lock b/pixi.lock index 354d6e7f..518ee32a 100644 --- a/pixi.lock +++ b/pixi.lock @@ -17,6 +17,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/certifi-2025年4月26日-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/cffi-1.17.1-py313hfab6e84_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/charset-normalizer-3.4.2-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/cloudpathlib-0.21.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/cmarkgfm-2024年11月20日-py313h536fd9c_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/colorama-0.4.6-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/cpython-3.13.3-py313hd8ed1ab_101.conda @@ -120,6 +121,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/certifi-2025年4月26日-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/cffi-1.17.1-py313h2135053_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/charset-normalizer-3.4.2-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/cloudpathlib-0.21.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/cmarkgfm-2024年11月20日-py313h31d5739_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/colorama-0.4.6-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/cpython-3.13.3-py313hd8ed1ab_101.conda @@ -221,6 +223,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/certifi-2025年4月26日-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/cffi-1.17.1-py313h49682b3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/charset-normalizer-3.4.2-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/cloudpathlib-0.21.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/cmarkgfm-2024年11月20日-py313h63b0ddb_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/colorama-0.4.6-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/cpython-3.13.3-py313hd8ed1ab_101.conda @@ -308,6 +311,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/certifi-2025年4月26日-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/cffi-1.17.1-py313hc845a76_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/charset-normalizer-3.4.2-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/cloudpathlib-0.21.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/cmarkgfm-2024年11月20日-py313h90d716c_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/colorama-0.4.6-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/cpython-3.13.3-py313hd8ed1ab_101.conda @@ -395,6 +399,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/certifi-2025年4月26日-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/win-64/cffi-1.17.1-py313ha7868ed_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/charset-normalizer-3.4.2-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/cloudpathlib-0.21.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/win-64/cmarkgfm-2024年11月20日-py313ha7868ed_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/colorama-0.4.6-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/cpython-3.13.3-py313hd8ed1ab_101.conda @@ -522,6 +527,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-64/cffi-1.17.1-py312h06ac9bb_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/cfgv-3.3.1-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/charset-normalizer-3.4.2-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/cloudpathlib-0.21.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/colorama-0.4.6-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/comm-0.2.2-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/coverage-7.8.2-py312h178313f_0.conda @@ -800,6 +806,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/cffi-1.17.1-py313h2135053_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/cfgv-3.3.1-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/charset-normalizer-3.4.2-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/cloudpathlib-0.21.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/colorama-0.4.6-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/comm-0.2.2-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/coverage-7.8.2-py313h7815b11_0.conda @@ -1075,6 +1082,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-64/cffi-1.17.1-py312hf857d28_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/cfgv-3.3.1-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/charset-normalizer-3.4.2-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/cloudpathlib-0.21.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/colorama-0.4.6-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/comm-0.2.2-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/coverage-7.8.2-py312h3520af0_0.conda @@ -1340,6 +1348,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-arm64/cffi-1.17.1-py313hc845a76_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/cfgv-3.3.1-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/charset-normalizer-3.4.2-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/cloudpathlib-0.21.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/colorama-0.4.6-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/comm-0.2.2-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/coverage-7.8.2-py313ha9b7d5b_0.conda @@ -1599,6 +1608,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/win-64/cffi-1.17.1-py313ha7868ed_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/cfgv-3.3.1-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/charset-normalizer-3.4.2-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/cloudpathlib-0.21.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/colorama-0.4.6-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/comm-0.2.2-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/win-64/coverage-7.8.2-py313hb4c8b1a_0.conda @@ -1833,6 +1843,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/certifi-2025年4月26日-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/cffi-1.17.1-py312h06ac9bb_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/charset-normalizer-3.4.2-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/cloudpathlib-0.21.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/colorama-0.4.6-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/cpython-3.12.10-py312hd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/decorator-5.2.1-pyhd8ed1ab_0.conda @@ -1991,6 +2002,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/certifi-2025年4月26日-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/cffi-1.17.1-py313h2135053_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/charset-normalizer-3.4.2-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/cloudpathlib-0.21.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/colorama-0.4.6-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/cpython-3.13.3-py313hd8ed1ab_101.conda - conda: https://conda.anaconda.org/conda-forge/noarch/decorator-5.2.1-pyhd8ed1ab_0.conda @@ -2145,6 +2157,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/certifi-2025年4月26日-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/cffi-1.17.1-py312hf857d28_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/charset-normalizer-3.4.2-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/cloudpathlib-0.21.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/colorama-0.4.6-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/cpython-3.12.10-py312hd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/decorator-5.2.1-pyhd8ed1ab_0.conda @@ -2288,6 +2301,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/certifi-2025年4月26日-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/cffi-1.17.1-py313hc845a76_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/charset-normalizer-3.4.2-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/cloudpathlib-0.21.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/colorama-0.4.6-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/cpython-3.13.3-py313hd8ed1ab_101.conda - conda: https://conda.anaconda.org/conda-forge/noarch/decorator-5.2.1-pyhd8ed1ab_0.conda @@ -2432,6 +2446,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/certifi-2025年4月26日-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/win-64/cffi-1.17.1-py313ha7868ed_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/charset-normalizer-3.4.2-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/cloudpathlib-0.21.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/colorama-0.4.6-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/cpython-3.13.3-py313hd8ed1ab_101.conda - conda: https://conda.anaconda.org/conda-forge/noarch/decorator-5.2.1-pyhd8ed1ab_0.conda @@ -2892,6 +2907,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-64/bzip2-1.0.8-h4bc722e_7.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/c-ares-1.34.5-hb9d3cd8_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2025年6月15日-hbd8a1cb_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/cloudpathlib-0.21.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/colorama-0.4.6-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/coverage-7.9.2-py313h8060acc_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/cpython-3.13.5-py313hd8ed1ab_102.conda @@ -3037,6 +3053,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/bzip2-1.0.8-h68df207_7.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/c-ares-1.34.5-h86ecc28_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2025年6月15日-hbd8a1cb_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/cloudpathlib-0.21.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/colorama-0.4.6-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/coverage-7.9.2-py313h7815b11_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/cpython-3.13.5-py313hd8ed1ab_102.conda @@ -3179,6 +3196,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-64/bzip2-1.0.8-hfdf4475_7.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/c-ares-1.34.5-hf13058a_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2025年6月15日-hbd8a1cb_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/cloudpathlib-0.21.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/colorama-0.4.6-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/coverage-7.9.2-py312h3520af0_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/cpython-3.12.11-py312hd8ed1ab_0.conda @@ -3310,6 +3328,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-arm64/bzip2-1.0.8-h99b78c6_7.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/c-ares-1.34.5-h5505292_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2025年6月15日-hbd8a1cb_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/cloudpathlib-0.21.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/colorama-0.4.6-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/coverage-7.9.2-py313ha9b7d5b_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/cpython-3.13.5-py313hd8ed1ab_102.conda @@ -3435,6 +3454,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/win-64/bzip2-1.0.8-h2466b09_7.conda - conda: https://conda.anaconda.org/conda-forge/win-64/c-ares-1.34.5-h2466b09_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2025年6月15日-h4c7d964_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/cloudpathlib-0.21.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/colorama-0.4.6-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/win-64/coverage-7.9.2-py313hd650c13_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/cpython-3.13.5-py313hd8ed1ab_102.conda @@ -3563,6 +3583,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-64/bzip2-1.0.8-h4bc722e_7.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/c-ares-1.34.5-hb9d3cd8_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2025年7月9日-hbd8a1cb_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/cloudpathlib-0.21.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/colorama-0.4.6-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/coverage-7.9.2-py310h89163eb_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/cpython-3.10.18-py310hd8ed1ab_0.conda @@ -3710,6 +3731,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/bzip2-1.0.8-h68df207_7.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/c-ares-1.34.5-h86ecc28_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2025年7月9日-hbd8a1cb_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/cloudpathlib-0.21.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/colorama-0.4.6-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/coverage-7.9.2-py310h66848f9_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/cpython-3.10.18-py310hd8ed1ab_0.conda @@ -3855,6 +3877,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-64/bzip2-1.0.8-hfdf4475_7.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/c-ares-1.34.5-hf13058a_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2025年7月9日-hbd8a1cb_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/cloudpathlib-0.21.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/colorama-0.4.6-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/coverage-7.9.2-py310h8e2f543_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/cpython-3.10.18-py310hd8ed1ab_0.conda @@ -3986,6 +4009,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-arm64/bzip2-1.0.8-h99b78c6_7.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/c-ares-1.34.5-h5505292_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2025年7月9日-hbd8a1cb_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/cloudpathlib-0.21.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/colorama-0.4.6-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/coverage-7.9.2-py310hc74094e_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/cpython-3.10.18-py310hd8ed1ab_0.conda @@ -4111,6 +4135,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/win-64/bzip2-1.0.8-h2466b09_7.conda - conda: https://conda.anaconda.org/conda-forge/win-64/c-ares-1.34.5-h2466b09_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2025年7月9日-h4c7d964_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/cloudpathlib-0.21.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/colorama-0.4.6-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/win-64/coverage-7.9.2-py310hdb0e946_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/cpython-3.10.18-py310hd8ed1ab_0.conda @@ -4239,6 +4264,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-64/bzip2-1.0.8-h4bc722e_7.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/c-ares-1.34.5-hb9d3cd8_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2025年4月26日-hbd8a1cb_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/cloudpathlib-0.21.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/colorama-0.4.6-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/coverage-7.8.2-py311h2dc5d0c_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/cpython-3.11.12-py311hd8ed1ab_0.conda @@ -4385,6 +4411,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/bzip2-1.0.8-h68df207_7.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/c-ares-1.34.5-h86ecc28_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2025年4月26日-hbd8a1cb_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/cloudpathlib-0.21.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/colorama-0.4.6-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/coverage-7.8.2-py311ha09ea12_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/cpython-3.11.12-py311hd8ed1ab_0.conda @@ -4528,6 +4555,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-64/bzip2-1.0.8-hfdf4475_7.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/c-ares-1.34.5-hf13058a_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2025年4月26日-hbd8a1cb_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/cloudpathlib-0.21.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/colorama-0.4.6-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/coverage-7.8.2-py311ha3cf9ac_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/cpython-3.11.12-py311hd8ed1ab_0.conda @@ -4658,6 +4686,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-arm64/bzip2-1.0.8-h99b78c6_7.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/c-ares-1.34.5-h5505292_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2025年4月26日-hbd8a1cb_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/cloudpathlib-0.21.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/colorama-0.4.6-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/coverage-7.8.2-py311h4921393_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/cpython-3.11.12-py311hd8ed1ab_0.conda @@ -4782,6 +4811,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/win-64/bzip2-1.0.8-h2466b09_7.conda - conda: https://conda.anaconda.org/conda-forge/win-64/c-ares-1.34.5-h2466b09_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2025年4月26日-h4c7d964_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/cloudpathlib-0.21.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/colorama-0.4.6-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/win-64/coverage-7.8.2-py311h5082efb_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/cpython-3.11.12-py311hd8ed1ab_0.conda @@ -4909,6 +4939,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-64/bzip2-1.0.8-h4bc722e_7.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/c-ares-1.34.5-hb9d3cd8_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2025年4月26日-hbd8a1cb_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/cloudpathlib-0.21.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/colorama-0.4.6-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/coverage-7.8.2-py312h178313f_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/cpython-3.12.10-py312hd8ed1ab_0.conda @@ -5055,6 +5086,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/bzip2-1.0.8-h68df207_7.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/c-ares-1.34.5-h86ecc28_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2025年4月26日-hbd8a1cb_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/cloudpathlib-0.21.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/colorama-0.4.6-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/coverage-7.8.2-py312h74ce7d3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/cpython-3.12.10-py312hd8ed1ab_0.conda @@ -5198,6 +5230,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-64/bzip2-1.0.8-hfdf4475_7.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/c-ares-1.34.5-hf13058a_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2025年4月26日-hbd8a1cb_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/cloudpathlib-0.21.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/colorama-0.4.6-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/coverage-7.8.2-py312h3520af0_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/cpython-3.12.10-py312hd8ed1ab_0.conda @@ -5328,6 +5361,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-arm64/bzip2-1.0.8-h99b78c6_7.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/c-ares-1.34.5-h5505292_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2025年4月26日-hbd8a1cb_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/cloudpathlib-0.21.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/colorama-0.4.6-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/coverage-7.8.2-py312h998013c_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/cpython-3.12.10-py312hd8ed1ab_0.conda @@ -5452,6 +5486,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/win-64/bzip2-1.0.8-h2466b09_7.conda - conda: https://conda.anaconda.org/conda-forge/win-64/c-ares-1.34.5-h2466b09_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2025年4月26日-h4c7d964_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/cloudpathlib-0.21.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/colorama-0.4.6-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/win-64/coverage-7.8.2-py312h31fea79_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/cpython-3.12.10-py312hd8ed1ab_0.conda @@ -5579,6 +5614,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-64/bzip2-1.0.8-h4bc722e_7.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/c-ares-1.34.5-hb9d3cd8_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2025年4月26日-hbd8a1cb_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/cloudpathlib-0.21.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/colorama-0.4.6-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/coverage-7.8.2-py313h8060acc_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/cpython-3.13.3-py313hd8ed1ab_101.conda @@ -5723,6 +5759,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/bzip2-1.0.8-h68df207_7.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/c-ares-1.34.5-h86ecc28_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2025年4月26日-hbd8a1cb_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/cloudpathlib-0.21.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/colorama-0.4.6-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/coverage-7.8.2-py313h7815b11_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/cpython-3.13.3-py313hd8ed1ab_101.conda @@ -5864,6 +5901,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-64/bzip2-1.0.8-hfdf4475_7.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/c-ares-1.34.5-hf13058a_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2025年4月26日-hbd8a1cb_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/cloudpathlib-0.21.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/colorama-0.4.6-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/coverage-7.8.2-py313h717bdf5_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/cpython-3.13.3-py313hd8ed1ab_101.conda @@ -5994,6 +6032,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-arm64/bzip2-1.0.8-h99b78c6_7.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/c-ares-1.34.5-h5505292_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2025年4月26日-hbd8a1cb_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/cloudpathlib-0.21.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/colorama-0.4.6-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/coverage-7.8.2-py313ha9b7d5b_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/cpython-3.13.3-py313hd8ed1ab_101.conda @@ -6118,6 +6157,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/win-64/bzip2-1.0.8-h2466b09_7.conda - conda: https://conda.anaconda.org/conda-forge/win-64/c-ares-1.34.5-h2466b09_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2025年4月26日-h4c7d964_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/cloudpathlib-0.21.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/colorama-0.4.6-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/win-64/coverage-7.8.2-py313hb4c8b1a_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/cpython-3.13.3-py313hd8ed1ab_101.conda @@ -8984,6 +9024,16 @@ packages: license_family: MIT size: 50481 timestamp: 1746214981991 +- conda: https://conda.anaconda.org/conda-forge/noarch/cloudpathlib-0.21.1-pyhd8ed1ab_0.conda + sha256: 74da6f68f637627a5b53b1bf962fad6e34d2d529f0f32e4ec56bbf27dcc63624 + md5: d401b7d72e2cf55444b12110ed953c9d + depends: + - python>=3.9 + - typing_extensions + license: MIT + license_family: MIT + size: 44459 + timestamp: 1747332027476 - conda: https://conda.anaconda.org/conda-forge/linux-64/cmarkgfm-2024年11月20日-py313h536fd9c_0.conda sha256: 7e5225d77174501196f3b97e1418f1759f063b227e2e7e82e6db86c9592273b9 md5: 0fc2d9182e2d2fd2d8c94f424b4adec5 diff --git a/pixi.toml b/pixi.toml index ce55b502..bf0976b6 100644 --- a/pixi.toml +++ b/pixi.toml @@ -14,6 +14,7 @@ rust = "=1.85" numpy = "*" polars = ">=1.30" pytest-mock = ">=3.14.1,<4" +cloudpathlib = ">=0.21.1,<0.22" [host-dependencies] maturin = ">=1.7,<2" diff --git a/tests/collection/test_read_write_parquet.py b/tests/collection/test_read_write_parquet.py index a2e507bb..cfce3835 100644 --- a/tests/collection/test_read_write_parquet.py +++ b/tests/collection/test_read_write_parquet.py @@ -1,6 +1,8 @@ # Copyright (c) QuantCo 2025-2025 # SPDX-License-Identifier: BSD-3-Clause +import random +import string from collections.abc import Callable from pathlib import Path from typing import Any, TypeVar @@ -8,6 +10,8 @@ import polars as pl import pytest import pytest_mock +from cloudpathlib import CloudPath +from cloudpathlib.local import LocalS3Path from polars.testing import assert_frame_equal import dataframely as dy @@ -17,14 +21,18 @@ C = TypeVar("C", bound=dy.Collection) -def _write_parquet_typed(collection: dy.Collection, path: Path, lazy: bool) -> None: +def _write_parquet_typed( + collection: dy.Collection, path: Path | CloudPath, lazy: bool +) -> None: if lazy: collection.sink_parquet(path) else: collection.write_parquet(path) -def _write_parquet(collection: dy.Collection, path: Path, lazy: bool) -> None: +def _write_parquet( + collection: dy.Collection, path: Path | CloudPath, lazy: bool +) -> None: if lazy: collection.sink_parquet(path) else: @@ -32,24 +40,28 @@ def _write_parquet(collection: dy.Collection, path: Path, lazy: bool) -> None: (path / "schema.json").unlink() -def _read_parquet(collection: type[C], path: Path, lazy: bool, **kwargs: Any) -> C: +def _read_parquet( + collection: type[C], path: Path | CloudPath, lazy: bool, **kwargs: Any +) -> C: if lazy: return collection.scan_parquet(path, **kwargs) else: return collection.read_parquet(path, **kwargs) -def _write_collection_with_no_schema(tmp_path: Path, lazy: bool) -> type[dy.Collection]: +def _write_collection_with_no_schema( + path: Path | CloudPath, lazy: bool +) -> type[dy.Collection]: collection_type = create_collection( "test", {"a": create_schema("test", {"a": dy.Int64(), "b": dy.String()})} ) collection = collection_type.create_empty() - _write_parquet(collection, tmp_path, lazy) + _write_parquet(collection, path, lazy) return collection_type def _write_collection_with_incorrect_schema( - tmp_path: Path, lazy: bool + path: Path | CloudPath, lazy: bool ) -> type[dy.Collection]: collection_type = create_collection( "test", {"a": create_schema("test", {"a": dy.Int64(), "b": dy.String()})} @@ -63,10 +75,22 @@ def _write_collection_with_incorrect_schema( }, ) collection = other_collection_type.create_empty() - _write_parquet_typed(collection, tmp_path, lazy) + _write_parquet_typed(collection, path, lazy) return collection_type +@pytest.fixture +def cloud_path() -> CloudPath: + """Fixture to provide a cloud path.""" + bucket_name = "".join( + random.choice(string.ascii_uppercase + string.digits) for _ in range(10) + ) + + path = LocalS3Path(f"s3://{bucket_name}/") + path.client._cloud_path_to_local(path).mkdir(exist_ok=True, parents=True) + return path + + # ------------------------------------------------------------------------------------ # @@ -88,18 +112,27 @@ class MyCollection(dy.Collection): "read_fn", [MyCollection.scan_parquet, MyCollection.read_parquet] ) @pytest.mark.parametrize("kwargs", [{}, {"partition_by": "a"}]) +@pytest.mark.parametrize("path_fixture", ["tmp_path", "cloud_path"]) def test_read_write_parquet( - tmp_path: Path, read_fn: Callable[[Path], MyCollection], kwargs: dict[str, Any] + path_fixture: str, + read_fn: Callable[[Path | CloudPath], MyCollection], + kwargs: dict[str, Any], + request: pytest.FixtureRequest, ) -> None: + path = request.getfixturevalue(path_fixture) + assert isinstance(path, Path | LocalS3Path), ( + "Path fixture must be a Path or LocalS3Path" + ) + collection = MyCollection.cast( { "first": pl.LazyFrame({"a": [1, 2, 3]}), "second": pl.LazyFrame({"a": [1, 2], "b": [10, 15]}), } ) - collection.write_parquet(tmp_path, **kwargs) + collection.write_parquet(path, **kwargs) - read = read_fn(tmp_path) + read = read_fn(path) assert_frame_equal(collection.first, read.first) assert collection.second is not None assert read.second is not None @@ -110,13 +143,22 @@ def test_read_write_parquet( "read_fn", [MyCollection.scan_parquet, MyCollection.read_parquet] ) @pytest.mark.parametrize("kwargs", [{}, {"partition_by": "a"}]) +@pytest.mark.parametrize("path_fixture", ["tmp_path", "cloud_path"]) def test_read_write_parquet_optional( - tmp_path: Path, read_fn: Callable[[Path], MyCollection], kwargs: dict[str, Any] + path_fixture: str, + read_fn: Callable[[Path | CloudPath], MyCollection], + kwargs: dict[str, Any], + request: pytest.FixtureRequest, ) -> None: + path = request.getfixturevalue(path_fixture) + assert isinstance(path, (Path | CloudPath)), ( + "Path fixture must be a Path or CloudPath" + ) + collection = MyCollection.cast({"first": pl.LazyFrame({"a": [1, 2, 3]})}) - collection.write_parquet(tmp_path, **kwargs) + collection.write_parquet(path, **kwargs) - read = read_fn(tmp_path) + read = read_fn(path) assert_frame_equal(collection.first, read.first) assert collection.second is None assert read.second is None @@ -127,19 +169,29 @@ def test_read_write_parquet_optional( @pytest.mark.parametrize("validation", ["warn", "allow", "forbid", "skip"]) @pytest.mark.parametrize("lazy", [True, False]) +@pytest.mark.parametrize("path_fixture", ["tmp_path", "cloud_path"]) def test_read_write_parquet_if_schema_matches( - tmp_path: Path, mocker: pytest_mock.MockerFixture, validation: Any, lazy: bool + path_fixture: str, + mocker: pytest_mock.MockerFixture, + validation: Any, + lazy: bool, + request: pytest.FixtureRequest, ) -> None: + path = request.getfixturevalue(path_fixture) + assert isinstance(path, (Path | LocalS3Path)), ( + "Path fixture must be a Path or LocalS3Path" + ) + # Arrange collection_type = create_collection( "test", {"a": create_schema("test", {"a": dy.Int64(), "b": dy.String()})} ) collection = collection_type.create_empty() - _write_parquet_typed(collection, tmp_path, lazy) + _write_parquet_typed(collection, path, lazy) # Act spy = mocker.spy(collection_type, "validate") - _read_parquet(collection_type, tmp_path, lazy, validation=validation) + _read_parquet(collection_type, path, lazy, validation=validation) # Assert spy.assert_not_called() @@ -149,11 +201,20 @@ def test_read_write_parquet_if_schema_matches( @pytest.mark.parametrize("lazy", [True, False]) +@pytest.mark.parametrize("path_fixture", ["tmp_path", "cloud_path"]) def test_read_write_parquet_validation_warn_no_schema( - tmp_path: Path, mocker: pytest_mock.MockerFixture, lazy: bool + path_fixture: str, + mocker: pytest_mock.MockerFixture, + lazy: bool, + request: pytest.FixtureRequest, ) -> None: # Arrange - collection = _write_collection_with_no_schema(tmp_path, lazy) + path = request.getfixturevalue(path_fixture) + assert isinstance(path, (Path | LocalS3Path)), ( + "Path fixture must be a Path or LocalS3Path" + ) + + collection = _write_collection_with_no_schema(path, lazy) # Act spy = mocker.spy(collection, "validate") @@ -161,18 +222,27 @@ def test_read_write_parquet_validation_warn_no_schema( UserWarning, match=r"requires validation: no collection schema to check validity", ): - _read_parquet(collection, tmp_path, lazy) + _read_parquet(collection, path, lazy) # Assert spy.assert_called_once() @pytest.mark.parametrize("lazy", [True, False]) +@pytest.mark.parametrize("path_fixture", ["tmp_path", "cloud_path"]) def test_read_write_parquet_validation_warn_invalid_schema( - tmp_path: Path, mocker: pytest_mock.MockerFixture, lazy: bool + path_fixture: str, + mocker: pytest_mock.MockerFixture, + lazy: bool, + request: pytest.FixtureRequest, ) -> None: # Arrange - collection = _write_collection_with_incorrect_schema(tmp_path, lazy) + path = request.getfixturevalue(path_fixture) + assert isinstance(path, (Path | LocalS3Path)), ( + "Path fixture must be a Path or LocalS3Path" + ) + + collection = _write_collection_with_incorrect_schema(path, lazy) # Act spy = mocker.spy(collection, "validate") @@ -180,7 +250,7 @@ def test_read_write_parquet_validation_warn_invalid_schema( UserWarning, match=r"requires validation: current collection schema does not match", ): - _read_parquet(collection, tmp_path, lazy) + _read_parquet(collection, path, lazy) # Assert spy.assert_called_once() @@ -190,30 +260,48 @@ def test_read_write_parquet_validation_warn_invalid_schema( @pytest.mark.parametrize("lazy", [True, False]) +@pytest.mark.parametrize("path_fixture", ["tmp_path", "cloud_path"]) def test_read_write_parquet_validation_allow_no_schema( - tmp_path: Path, mocker: pytest_mock.MockerFixture, lazy: bool + path_fixture: str, + mocker: pytest_mock.MockerFixture, + lazy: bool, + request: pytest.FixtureRequest, ) -> None: # Arrange - collection = _write_collection_with_no_schema(tmp_path, lazy) + path = request.getfixturevalue(path_fixture) + assert isinstance(path, (Path | LocalS3Path)), ( + "Path fixture must be a Path or LocalS3Path" + ) + + collection = _write_collection_with_no_schema(path, lazy) # Act spy = mocker.spy(collection, "validate") - _read_parquet(collection, tmp_path, lazy, validation="allow") + _read_parquet(collection, path, lazy, validation="allow") # Assert spy.assert_called_once() @pytest.mark.parametrize("lazy", [True, False]) +@pytest.mark.parametrize("path_fixture", ["tmp_path", "cloud_path"]) def test_read_write_parquet_validation_allow_invalid_schema( - tmp_path: Path, mocker: pytest_mock.MockerFixture, lazy: bool + path_fixture: str, + mocker: pytest_mock.MockerFixture, + lazy: bool, + request: pytest.FixtureRequest, ) -> None: # Arrange - collection = _write_collection_with_incorrect_schema(tmp_path, lazy) + path = request.getfixturevalue(path_fixture) + assert isinstance(path, (Path | LocalS3Path)), ( + "Path fixture must be a Path or LocalS3Path" + ) + + collection = _write_collection_with_incorrect_schema(path, lazy) # Act spy = mocker.spy(collection, "validate") - _read_parquet(collection, tmp_path, lazy, validation="allow") + _read_parquet(collection, path, lazy, validation="allow") # Assert spy.assert_called_once() @@ -223,63 +311,93 @@ def test_read_write_parquet_validation_allow_invalid_schema( @pytest.mark.parametrize("lazy", [True, False]) +@pytest.mark.parametrize("path_fixture", ["tmp_path", "cloud_path"]) def test_read_write_parquet_validation_forbid_no_schema( - tmp_path: Path, lazy: bool + path_fixture: str, lazy: bool, request: pytest.FixtureRequest ) -> None: # Arrange - collection = _write_collection_with_no_schema(tmp_path, lazy) + path = request.getfixturevalue(path_fixture) + assert isinstance(path, (Path | LocalS3Path)), ( + "Path fixture must be a Path or LocalS3Path" + ) + + collection = _write_collection_with_no_schema(path, lazy) # Act with pytest.raises( ValidationRequiredError, match=r"without validation: no collection schema to check validity", ): - _read_parquet(collection, tmp_path, lazy, validation="forbid") + _read_parquet(collection, path, lazy, validation="forbid") @pytest.mark.parametrize("lazy", [True, False]) +@pytest.mark.parametrize("path_fixture", ["tmp_path", "cloud_path"]) def test_read_write_parquet_validation_forbid_invalid_schema( - tmp_path: Path, lazy: bool + path_fixture: str, lazy: bool, request: pytest.FixtureRequest ) -> None: # Arrange - collection = _write_collection_with_incorrect_schema(tmp_path, lazy) + path = request.getfixturevalue(path_fixture) + assert isinstance(path, (Path | LocalS3Path)), ( + "Path fixture must be a Path or LocalS3Path" + ) + + collection = _write_collection_with_incorrect_schema(path, lazy) # Act with pytest.raises( ValidationRequiredError, match=r"without validation: current collection schema does not match", ): - _read_parquet(collection, tmp_path, lazy, validation="forbid") + _read_parquet(collection, path, lazy, validation="forbid") # --------------------------------- VALIDATION "SKIP" -------------------------------- # @pytest.mark.parametrize("lazy", [True, False]) +@pytest.mark.parametrize("path_fixture", ["tmp_path", "cloud_path"]) def test_read_write_parquet_validation_skip_no_schema( - tmp_path: Path, mocker: pytest_mock.MockerFixture, lazy: bool + path_fixture: str, + mocker: pytest_mock.MockerFixture, + lazy: bool, + request: pytest.FixtureRequest, ) -> None: # Arrange - collection = _write_collection_with_no_schema(tmp_path, lazy) + path = request.getfixturevalue(path_fixture) + assert isinstance(path, Path | LocalS3Path), ( + "Path fixture must be a Path or LocalS3Path" + ) + + collection = _write_collection_with_no_schema(path, lazy) # Act spy = mocker.spy(collection, "validate") - _read_parquet(collection, tmp_path, lazy, validation="skip") + _read_parquet(collection, path, lazy, validation="skip") # Assert spy.assert_not_called() @pytest.mark.parametrize("lazy", [True, False]) +@pytest.mark.parametrize("path_fixture", ["tmp_path", "cloud_path"]) def test_read_write_parquet_validation_skip_invalid_schema( - tmp_path: Path, mocker: pytest_mock.MockerFixture, lazy: bool + path_fixture: str, + mocker: pytest_mock.MockerFixture, + lazy: bool, + request: pytest.FixtureRequest, ) -> None: # Arrange - collection = _write_collection_with_incorrect_schema(tmp_path, lazy) + path = request.getfixturevalue(path_fixture) + assert isinstance(path, Path | LocalS3Path), ( + "Path fixture must be a Path or LocalS3Path" + ) + + collection = _write_collection_with_incorrect_schema(path, lazy) # Act spy = mocker.spy(collection, "validate") - _read_parquet(collection, tmp_path, lazy, validation="skip") + _read_parquet(collection, path, lazy, validation="skip") # Assert spy.assert_not_called() diff --git a/tests/schema/test_read_write_parquet.py b/tests/schema/test_read_write_parquet.py index 9135de1a..9de50552 100644 --- a/tests/schema/test_read_write_parquet.py +++ b/tests/schema/test_read_write_parquet.py @@ -7,9 +7,12 @@ import polars as pl import pytest import pytest_mock +from cloudpathlib import CloudPath +from cloudpathlib.local import LocalS3Path from polars.testing import assert_frame_equal import dataframely as dy +from dataframely._path import handle_cloud_path from dataframely.exc import ValidationRequiredError from dataframely.testing import create_schema @@ -17,7 +20,7 @@ def _write_parquet_typed( - schema: type[S], df: dy.DataFrame[S], path: Path, lazy: bool + schema: type[S], df: dy.DataFrame[S], path: Path | CloudPath, lazy: bool ) -> None: if lazy: schema.sink_parquet(df.lazy(), path) @@ -25,7 +28,8 @@ def _write_parquet_typed( schema.write_parquet(df, path) -def _write_parquet(df: pl.DataFrame, path: Path, lazy: bool) -> None: +def _write_parquet(df: pl.DataFrame, path: Path | CloudPath, lazy: bool) -> None: + path = handle_cloud_path(path) if lazy: df.lazy().sink_parquet(path) else: @@ -33,7 +37,7 @@ def _write_parquet(df: pl.DataFrame, path: Path, lazy: bool) -> None: def _read_parquet( - schema: type[S], path: Path, lazy: bool, **kwargs: Any + schema: type[S], path: Path | CloudPath, lazy: bool, **kwargs: Any ) -> dy.DataFrame[S]: if lazy: return schema.scan_parquet(path, **kwargs).collect() @@ -41,14 +45,18 @@ def _read_parquet( return schema.read_parquet(path, **kwargs) -def _write_parquet_with_no_schema(tmp_path: Path, lazy: bool) -> type[dy.Schema]: +def _write_parquet_with_no_schema( + tmp_path: Path | CloudPath, lazy: bool +) -> type[dy.Schema]: schema = create_schema("test", {"a": dy.Int64(), "b": dy.String()}) df = schema.create_empty() _write_parquet(df, tmp_path / "test.parquet", lazy) return schema -def _write_parquet_with_incorrect_schema(tmp_path: Path, lazy: bool) -> type[dy.Schema]: +def _write_parquet_with_incorrect_schema( + tmp_path: Path | CloudPath, lazy: bool +) -> type[dy.Schema]: schema = create_schema("test", {"a": dy.Int64(), "b": dy.String()}) other_schema = create_schema( "test", {"a": dy.Int64(primary_key=True), "b": dy.String()} @@ -58,22 +66,39 @@ def _write_parquet_with_incorrect_schema(tmp_path: Path, lazy: bool) -> type[dy. return schema +@pytest.fixture +def cloud_path() -> CloudPath: + """Fixture to provide a cloud path.""" + path = LocalS3Path("s3://test-bucket/") + path.client._cloud_path_to_local(path).mkdir(exist_ok=True, parents=True) + return path + + # ------------------------------------------------------------------------------------ # @pytest.mark.parametrize("validation", ["warn", "allow", "forbid", "skip"]) @pytest.mark.parametrize("lazy", [True, False]) +@pytest.mark.parametrize("path_fixture", ["tmp_path", "cloud_path"]) def test_read_write_parquet_if_schema_matches( - tmp_path: Path, mocker: pytest_mock.MockerFixture, validation: Any, lazy: bool + path_fixture: str, + mocker: pytest_mock.MockerFixture, + validation: Any, + lazy: bool, + request: pytest.FixtureRequest, ) -> None: + path = request.getfixturevalue(path_fixture) + assert isinstance(path, Path | LocalS3Path), ( + "Path fixture must be a Path or LocalS3Path" + ) # Arrange schema = create_schema("test", {"a": dy.Int64(), "b": dy.String()}) df = schema.create_empty() - _write_parquet_typed(schema, df, tmp_path / "test.parquet", lazy) + _write_parquet_typed(schema, df, path / "test.parquet", lazy) # Act spy = mocker.spy(schema, "validate") - out = _read_parquet(schema, tmp_path / "test.parquet", lazy, validation=validation) + out = _read_parquet(schema, path / "test.parquet", lazy, validation=validation) # Assert spy.assert_not_called() @@ -84,36 +109,54 @@ def test_read_write_parquet_if_schema_matches( @pytest.mark.parametrize("lazy", [True, False]) +@pytest.mark.parametrize("path_fixture", ["tmp_path", "cloud_path"]) def test_read_write_parquet_validation_warn_no_schema( - tmp_path: Path, mocker: pytest_mock.MockerFixture, lazy: bool + path_fixture: str, + mocker: pytest_mock.MockerFixture, + lazy: bool, + request: pytest.FixtureRequest, ) -> None: # Arrange - schema = _write_parquet_with_no_schema(tmp_path, lazy) + path = request.getfixturevalue(path_fixture) + assert isinstance(path, Path | LocalS3Path), ( + "Path fixture must be a Path or LocalS3Path" + ) + + schema = _write_parquet_with_no_schema(path, lazy) # Act spy = mocker.spy(schema, "validate") with pytest.warns( UserWarning, match=r"requires validation: no schema to check validity" ): - _read_parquet(schema, tmp_path / "test.parquet", lazy) + _read_parquet(schema, path / "test.parquet", lazy) # Assert spy.assert_called_once() @pytest.mark.parametrize("lazy", [True, False]) +@pytest.mark.parametrize("path_fixture", ["tmp_path", "cloud_path"]) def test_read_write_parquet_validation_warn_invalid_schema( - tmp_path: Path, mocker: pytest_mock.MockerFixture, lazy: bool + path_fixture: str, + mocker: pytest_mock.MockerFixture, + lazy: bool, + request: pytest.FixtureRequest, ) -> None: # Arrange - schema = _write_parquet_with_incorrect_schema(tmp_path, lazy) + path = request.getfixturevalue(path_fixture) + assert isinstance(path, Path | LocalS3Path), ( + "Path fixture must be a Path or LocalS3Path" + ) + + schema = _write_parquet_with_incorrect_schema(path, lazy) # Act spy = mocker.spy(schema, "validate") with pytest.warns( UserWarning, match=r"requires validation: current schema does not match" ): - _read_parquet(schema, tmp_path / "test.parquet", lazy) + _read_parquet(schema, path / "test.parquet", lazy) # Assert spy.assert_called_once() @@ -123,30 +166,47 @@ def test_read_write_parquet_validation_warn_invalid_schema( @pytest.mark.parametrize("lazy", [True, False]) +@pytest.mark.parametrize("path_fixture", ["tmp_path", "cloud_path"]) def test_read_write_parquet_validation_allow_no_schema( - tmp_path: Path, mocker: pytest_mock.MockerFixture, lazy: bool + path_fixture: str, + mocker: pytest_mock.MockerFixture, + lazy: bool, + request: pytest.FixtureRequest, ) -> None: # Arrange - schema = _write_parquet_with_no_schema(tmp_path, lazy) + path = request.getfixturevalue(path_fixture) + assert isinstance(path, Path | LocalS3Path), ( + "Path fixture must be a Path or LocalS3Path" + ) + schema = _write_parquet_with_no_schema(path, lazy) # Act spy = mocker.spy(schema, "validate") - _read_parquet(schema, tmp_path / "test.parquet", lazy, validation="allow") + _read_parquet(schema, path / "test.parquet", lazy, validation="allow") # Assert spy.assert_called_once() @pytest.mark.parametrize("lazy", [True, False]) +@pytest.mark.parametrize("path_fixture", ["tmp_path", "cloud_path"]) def test_read_write_parquet_validation_allow_invalid_schema( - tmp_path: Path, mocker: pytest_mock.MockerFixture, lazy: bool + path_fixture: str, + mocker: pytest_mock.MockerFixture, + lazy: bool, + request: pytest.FixtureRequest, ) -> None: # Arrange - schema = _write_parquet_with_incorrect_schema(tmp_path, lazy) + path = request.getfixturevalue(path_fixture) + assert isinstance(path, Path | LocalS3Path), ( + "Path fixture must be a Path or LocalS3Path" + ) + + schema = _write_parquet_with_incorrect_schema(path, lazy) # Act spy = mocker.spy(schema, "validate") - _read_parquet(schema, tmp_path / "test.parquet", lazy, validation="allow") + _read_parquet(schema, path / "test.parquet", lazy, validation="allow") # Assert spy.assert_called_once() @@ -156,63 +216,92 @@ def test_read_write_parquet_validation_allow_invalid_schema( @pytest.mark.parametrize("lazy", [True, False]) +@pytest.mark.parametrize("path_fixture", ["tmp_path", "cloud_path"]) def test_read_write_parquet_validation_forbid_no_schema( - tmp_path: Path, lazy: bool + path_fixture: str, lazy: bool, request: pytest.FixtureRequest ) -> None: # Arrange - schema = _write_parquet_with_no_schema(tmp_path, lazy) + path = request.getfixturevalue(path_fixture) + assert isinstance(path, Path | LocalS3Path), ( + "Path fixture must be a Path or LocalS3Path" + ) + + schema = _write_parquet_with_no_schema(path, lazy) # Act with pytest.raises( ValidationRequiredError, match=r"without validation: no schema to check validity", ): - _read_parquet(schema, tmp_path / "test.parquet", lazy, validation="forbid") + _read_parquet(schema, path / "test.parquet", lazy, validation="forbid") @pytest.mark.parametrize("lazy", [True, False]) +@pytest.mark.parametrize("path_fixture", ["tmp_path", "cloud_path"]) def test_read_write_parquet_validation_forbid_invalid_schema( - tmp_path: Path, lazy: bool + path_fixture: str, lazy: bool, request: pytest.FixtureRequest ) -> None: # Arrange - schema = _write_parquet_with_incorrect_schema(tmp_path, lazy) + path = request.getfixturevalue(path_fixture) + assert isinstance(path, Path | LocalS3Path), ( + "Path fixture must be a Path or LocalS3Path" + ) + + schema = _write_parquet_with_incorrect_schema(path, lazy) # Act with pytest.raises( ValidationRequiredError, match=r"without validation: current schema does not match", ): - _read_parquet(schema, tmp_path / "test.parquet", lazy, validation="forbid") + _read_parquet(schema, path / "test.parquet", lazy, validation="forbid") # --------------------------------- VALIDATION "SKIP" -------------------------------- # @pytest.mark.parametrize("lazy", [True, False]) +@pytest.mark.parametrize("path_fixture", ["tmp_path", "cloud_path"]) def test_read_write_parquet_validation_skip_no_schema( - tmp_path: Path, mocker: pytest_mock.MockerFixture, lazy: bool + path_fixture: str, + mocker: pytest_mock.MockerFixture, + lazy: bool, + request: pytest.FixtureRequest, ) -> None: # Arrange - schema = _write_parquet_with_no_schema(tmp_path, lazy) + path = request.getfixturevalue(path_fixture) + assert isinstance(path, Path | LocalS3Path), ( + "Path fixture must be a Path or LocalS3Path" + ) + schema = _write_parquet_with_no_schema(path, lazy) # Act spy = mocker.spy(schema, "validate") - _read_parquet(schema, tmp_path / "test.parquet", lazy, validation="skip") + _read_parquet(schema, path / "test.parquet", lazy, validation="skip") # Assert spy.assert_not_called() @pytest.mark.parametrize("lazy", [True, False]) +@pytest.mark.parametrize("path_fixture", ["tmp_path", "cloud_path"]) def test_read_write_parquet_validation_skip_invalid_schema( - tmp_path: Path, mocker: pytest_mock.MockerFixture, lazy: bool + mocker: pytest_mock.MockerFixture, + lazy: bool, + path_fixture: str, + request: pytest.FixtureRequest, ) -> None: + path = request.getfixturevalue(path_fixture) + assert isinstance(path, Path | LocalS3Path), ( + "Path fixture must be a Path or LocalS3Path" + ) + # Arrange - schema = _write_parquet_with_incorrect_schema(tmp_path, lazy) + schema = _write_parquet_with_incorrect_schema(path, lazy) # Act spy = mocker.spy(schema, "validate") - _read_parquet(schema, tmp_path / "test.parquet", lazy, validation="skip") + _read_parquet(schema, path / "test.parquet", lazy, validation="skip") # Assert spy.assert_not_called()

AltStyle によって変換されたページ (->オリジナル) /