Commit f307a0a

authored

ENH (string dtype): convert string_view columns to future string dtype instead of object dtype in Parquet/Feather IO (#60235)

* ENH (string dtype): convert string_view columns to future string dtype instead of object dtype in Parquet IO * move test to feather * fixup

1 parent cccf1e6 commit f307a0aCopy full SHA for f307a0a

File tree

+27

-2

lines changed

+27

-2

lines changed

Lines changed: 7 additions & 2 deletions

Original file line number	Diff line number	Diff line change
`@@ -4,6 +4,7 @@`
`4`	`4`
`5`	`5`	`import numpy as np`
`6`	`6`
	`7`	`+from pandas.compat import pa_version_under18p0`
`7`	`8`	`from pandas.compat._optional import import_optional_dependency`
`8`	`9`
`9`	`10`	`import pandas as pd`
`@@ -35,7 +36,11 @@ def _arrow_dtype_mapping() -> dict:`
`35`	`36`	`def arrow_string_types_mapper() -> Callable:`
`36`	`37`	`pa = import_optional_dependency("pyarrow")`
`37`	`38`
`38`		`- return {`
	`39`	`+ mapping= {`
`39`	`40`	`pa.string(): pd.StringDtype(na_value=np.nan),`
`40`	`41`	`pa.large_string(): pd.StringDtype(na_value=np.nan),`
`41`		`- }.get`
	`42`	`+ }`
	`43`	`+ if not pa_version_under18p0:`
	`44`	`+ mapping[pa.string_view()] = pd.StringDtype(na_value=np.nan)`
	`45`	`+`
	`46`	`+ return mapping.get`

Lines changed: 20 additions & 0 deletions

Original file line number	Diff line number	Diff line change
`@@ -6,6 +6,8 @@`
`6`	`6`	`import numpy as np`
`7`	`7`	`import pytest`
`8`	`8`
	`9`	`+from pandas.compat.pyarrow import pa_version_under18p0`
	`10`	`+`
`9`	`11`	`import pandas as pd`
`10`	`12`	`import pandas._testing as tm`
`11`	`13`
`@@ -249,6 +251,24 @@ def test_string_inference(self, tmp_path):`
`249`	`251`	`)`
`250`	`252`	`tm.assert_frame_equal(result, expected)`
`251`	`253`
	`254`	`+ @pytest.mark.skipif(pa_version_under18p0, reason="not supported before 18.0")`
	`255`	`+ def test_string_inference_string_view_type(self, tmp_path):`
	`256`	`+ # GH#54798`
	`257`	`+ import pyarrow as pa`
	`258`	`+ from pyarrow import feather`
	`259`	`+`
	`260`	`+ path = tmp_path / "string_view.parquet"`
	`261`	`+ table = pa.table({"a": pa.array([None, "b", "c"], pa.string_view())})`
	`262`	`+ feather.write_feather(table, path)`
	`263`	`+`
	`264`	`+ with pd.option_context("future.infer_string", True):`
	`265`	`+ result = read_feather(path)`
	`266`	`+`
	`267`	`+ expected = pd.DataFrame(`
	`268`	`+ data={"a": [None, "b", "c"]}, dtype=pd.StringDtype(na_value=np.nan)`
	`269`	`+ )`
	`270`	`+ tm.assert_frame_equal(result, expected)`
	`271`	`+`
`252`	`272`	`def test_out_of_bounds_datetime_to_feather(self):`
`253`	`273`	`# GH#47832`
`254`	`274`	`df = pd.DataFrame(`

Comments

(0)