diff --git a/CHANGELOG.md b/CHANGELOG.md index 074b229c..f07c08ae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ and this project adheres to [Semantic Versioning][]. - {func}`~ehrdata.io.omop.setup_variables` takes `value_range` to leave out implausible data points before the aggregation, as a mapping of variables to bounds such as `value_range={"heart_rate": (0, 300)}`. The data points outside the range neither count nor tell a unit. ([#347](https://github.com/theislab/ehrdata/pull/347)) @eroell - {func}`~ehrdata.io.omop.setup_variables` takes `unit_conversions`, a table of factors and offsets from the units a concept was recorded in to one unit, to convert the values before the aggregation and the value range. Converted data points carry the target unit in `unit_concept_id` and `unit_source_value`, and a data point in a unit the table does not list for its concept raises. ([#348](https://github.com/theislab/ehrdata/pull/348)) @eroell - {func}`~ehrdata.io.omop.setup_variables` and {func}`~ehrdata.io.omop.setup_interval_variables` take `num_intervals_before` to add intervals before the start of each observation, such as a look-back before a visit. ([#350](https://github.com/theislab/ehrdata/pull/350)) @eroell +- {func}`~ehrdata.io.write_h5ed` takes `convert_strings_to_categoricals`, as {func}`~ehrdata.io.write_zarr` does, and both convert the same columns: the string columns of `.obs`, `.var` and `.tem` in which a value repeats or is missing, including the datetimes and timedeltas written as ISO 8601 strings. ([#358](https://github.com/theislab/ehrdata/issues/358)) @eroell ### Changed @@ -27,6 +28,7 @@ and this project adheres to [Semantic Versioning][]. - {func}`~ehrdata.io.omop.setup_interval_variables` aggregates only the intervals that hold data points, as {func}`~ehrdata.io.omop.setup_variables` does, so memory and time follow the number of data points. ([#349](https://github.com/theislab/ehrdata/pull/349)) @eroell - {func}`~ehrdata.io.omop.setup_variables` and {func}`~ehrdata.io.omop.setup_interval_variables` with `aggregation_strategy="mode"` keep the smallest of several equally frequent values, instead of an arbitrary one. ([#341](https://github.com/theislab/ehrdata/pull/341)) @eroell - {func}`~ehrdata.io.omop.setup_variables` and {func}`~ehrdata.io.omop.setup_interval_variables` with `observation_table="person_visit_occurrence"` attribute a data point whose `visit_occurrence_id` names a visit of its person to that visit only. Before, a data point belonged to every visit of its person whose window held it, so overlapping visits of a person shared their data points, and with `clip_to_end_date=False` a visit's intervals after its end took the data points of the person's next visit. Data points without a `visit_occurrence_id`, or with one that names no visit of their person, still belong to every visit whose window holds them. ([#344](https://github.com/theislab/ehrdata/pull/344)) @eroell +- {func}`~ehrdata.io.write_h5ed` and {func}`~ehrdata.io.write_zarr` keep string columns as strings by default (`convert_strings_to_categoricals=False`), so an {class}`~ehrdata.EHRData` reads back with the dtypes it was written with, from `.h5ed` and `.zarr` alike. Before, {func}`~ehrdata.io.write_h5ed` converted the string columns of `.obs` and `.var` in which a value repeats or is missing to categoricals, and {func}`~ehrdata.io.write_zarr` also those of `.tem`. Files written before keep their categorical columns; pass `convert_strings_to_categoricals=True` for the previous conversion. ([#358](https://github.com/theislab/ehrdata/issues/358)) @eroell ### Fixed diff --git a/src/ehrdata/io/_ondisk.py b/src/ehrdata/io/_ondisk.py index c0f80787..1ca01c96 100644 --- a/src/ehrdata/io/_ondisk.py +++ b/src/ehrdata/io/_ondisk.py @@ -27,6 +27,8 @@ if TYPE_CHECKING: from collections.abc import Callable + import pandas as pd + from ehrdata import EHRData from ehrdata._types import GroupStorageType @@ -127,6 +129,17 @@ def encode_for_disk(edata: EHRData) -> tuple[ad.AnnData, dict[str, sparse.COO]]: return adata, sparse_3d_data +def strings_to_categoricals(adata: ad.AnnData, tem: pd.DataFrame) -> None: + """Convert the string columns of ``.obs``, ``.var`` and ``.tem`` in which a value repeats or is missing to categoricals. + + Converts in place, so pass the copies built for writing (see :func:`encode_for_disk`), never the caller's data. + The writers of every format call this, so that the same columns come back as categoricals from every format. + """ + adata.strings_to_categoricals(adata.obs) + adata.strings_to_categoricals(adata.var) + adata.strings_to_categoricals(tem) + + def decode_init_dict(init: dict[str, Any]) -> dict[str, Any]: """Restore relocated 3D arrays from ``.obsm`` back into ``X``/``layers`` in a dict of init kwargs.""" obsm = dict(init.get("obsm") or {}) diff --git a/src/ehrdata/io/h5ed.py b/src/ehrdata/io/h5ed.py index 0a046a12..73097bc4 100644 --- a/src/ehrdata/io/h5ed.py +++ b/src/ehrdata/io/h5ed.py @@ -28,6 +28,7 @@ decode_init_dict, encode_for_disk, read_mapping_with_coo, + strings_to_categoricals, ) if TYPE_CHECKING: @@ -127,6 +128,7 @@ def write_h5ed( *, compression: Literal["gzip", "lzf"] | None = None, compression_opts: int | None = None, + convert_strings_to_categoricals: bool = False, ) -> None: """Write :class:`~ehrdata.EHRData` objects to an ehrdata hdf5 file (`.h5ed`). @@ -134,7 +136,7 @@ def write_h5ed( To write the file, `X` and `layers` cannot be written as `object` dtype. If any of these fields is of `object` dtype, this function will attempt to cast it to a numeric dtype; if this fails, the field will be casted to a string dtype. The same holds for the `object` columns of `.obs`, `.var` and `.tem`; integers and booleans among them stay exact, and missing values stay missing. - Datetime and timedelta columns are written as ISO 8601 strings, read back as categorical in `.obs` and `.var` if a value repeats or is missing. + Datetime and timedelta columns are written as ISO 8601 strings, read back as categorical with `convert_strings_to_categoricals=True` if a value repeats or is missing. Args: @@ -143,6 +145,9 @@ def write_h5ed( compression: Optional file compression. Setting compression to 'gzip' can save disk space but will slow down writing and subsequent reading. compression_opts: For [`lzf`, `gzip`], see the h5py :ref:`dataset_compression`. + convert_strings_to_categoricals: Convert the string columns of `.obs`, `.var` and `.tem` in which a value repeats or is missing to `categorical` dtype. + By default, strings are kept. + The :class:`~ehrdata.EHRData` passed is left unchanged. Examples: >>> import ehrdata as ed @@ -152,10 +157,17 @@ def write_h5ed( filename = Path(filename) adata, sparse_3d_data = encode_for_disk(edata) + tem = _cast_dataframe_columns_to_writable_dtype(edata.tem, "tem") + + if convert_strings_to_categoricals: + strings_to_categoricals(adata, tem) + adata.write_h5ad( filename, compression=compression, compression_opts=compression_opts, + # converted above, with .tem; AnnData.write_h5ad would otherwise convert .obs and .var by default + convert_strings_to_categoricals=False, ) with h5py.File(filename, "a") as f: if sparse_3d_data: @@ -164,7 +176,7 @@ def write_h5ed( write_coo_h5( obsm_group.create_group(key), coo, compression=compression, compression_opts=compression_opts ) - ad.io.write_elem(f, "tem", _cast_dataframe_columns_to_writable_dtype(edata.tem, "tem")) + ad.io.write_elem(f, "tem", tem) # Identify the file as ehrdata, namespaced to not clash with anndata's own encoding attrs. f.attrs[EHRDATA_ENCODING_TYPE_KEY] = EHRDATA_ENCODING_TYPE f.attrs[EHRDATA_ONDISK_VERSION_KEY] = str(EHRDATA_ONDISK_VERSION) diff --git a/src/ehrdata/io/zarr.py b/src/ehrdata/io/zarr.py index 34587622..8cc7c0d1 100644 --- a/src/ehrdata/io/zarr.py +++ b/src/ehrdata/io/zarr.py @@ -25,6 +25,7 @@ decode_init_dict, encode_for_disk, read_mapping_with_coo, + strings_to_categoricals, ) if TYPE_CHECKING: @@ -54,7 +55,7 @@ def read_zarr( Examples: >>> import ehrdata as ed >>> edata = ed.dt.mimic_2() - >>> ed.io.write_zarr("mimic_2.zarr", edata) + >>> ed.io.write_zarr(edata, "mimic_2.zarr") >>> edata_from_zarr = ed.io.read_zarr("mimic_2.zarr") """ from ehrdata import EHRData @@ -117,7 +118,7 @@ def write_zarr( filename: str | Path, *, chunks: Literal["auto" | "ehrdata_auto"] = "auto", - convert_strings_to_categoricals: bool = True, + convert_strings_to_categoricals: bool = False, ) -> None: """Write :class:`~ehrdata.EHRData` objects to disk. @@ -131,7 +132,9 @@ def write_zarr( edata: Central data object. filename: Name of the output file, can also be prefixed with relative or absolute path to save the file to. chunks: Specify strategy of how data should be chunked. For simplicity, currently only 2 options are available: `"auto"` will write the data with :func:`~anndata.io.write_elem`'s default settings. `"ehrdata_auto"` will write the data chunked (and sharded) based on a heuristic that loosely speaking writes slightly smaller chunks. - convert_strings_to_categoricals: Convert columns of `str` dtype in `.obs` and `.var` and `.tem` to `categorical` dtype. + convert_strings_to_categoricals: Convert the string columns of `.obs`, `.var` and `.tem` in which a value repeats or is missing to `categorical` dtype. + By default, strings are kept. + The :class:`~ehrdata.EHRData` passed is left unchanged. Examples: >>> import ehrdata as ed @@ -146,9 +149,7 @@ def write_zarr( tem = _cast_dataframe_columns_to_writable_dtype(edata.tem, "tem") if convert_strings_to_categoricals: - adata.strings_to_categoricals(adata.obs) - adata.strings_to_categoricals(adata.var) - adata.strings_to_categoricals(tem) + strings_to_categoricals(adata, tem) # write_sharded this is a slightly modified version from https://anndata.readthedocs.io/en/stable/tutorials/zarr-v3.html # write_sharded is intended as a future blueprint of implementing better chunking defaults for ehrdata based based on real usecases diff --git a/tests/conftest.py b/tests/conftest.py index 323ace48..9259296c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -344,6 +344,35 @@ def edata_330(X_numpy_33, obs_31, var_31): return EHRData(X=X_numpy_33, obs=obs_31, var=var_31) +@pytest.fixture +def edata_string_columns(): + """EHRData whose `.obs`, `.var` and `.tem` hold string columns with and without repeated or missing values.""" + # 2D, as backed reading does not support 3D arrays + return EHRData( + X=np.zeros((3, 2)), + obs=pd.DataFrame( + { + "sex": ["F", "M", "F"], + "person_source_value": ["p1", "p2", "p3"], + "site": pd.array(["a", "a", pd.NA], dtype="string"), + "cohort": pd.Categorical(["x", "y", "z"]), + }, + index=["0", "1", "2"], + ), + var=pd.DataFrame({"unit": ["bpm", "bpm"], "name": ["heart rate", "blood pressure"]}, index=["hr", "bp"]), + tem=pd.DataFrame( + { + "kind": ["visit", "visit", "follow-up"], + "label": ["first", "second", "third"], + # written as ISO 8601 strings + "date": pd.to_datetime(["2000-01-01", "2000-01-01", "2000-01-02"]), + "duration": pd.to_timedelta(["1h", "1h", "2h"]), + }, + index=["t0", "t1", "t2"], + ), + ) + + @pytest.fixture def adata_33(X_numpy_33, obs_31, var_31): return ad.AnnData(X=X_numpy_33, obs=obs_31, var=var_31) diff --git a/tests/io/test_h5ed.py b/tests/io/test_h5ed.py index a1552711..d7256028 100644 --- a/tests/io/test_h5ed.py +++ b/tests/io/test_h5ed.py @@ -416,6 +416,85 @@ def test_write_read_h5ed_casts_object_layers(X_any_array_type_322, with_X, tmp_p _assert_object_layers_read_back(edata, read_h5ed(path)) +@pytest.mark.parametrize("convert_strings_to_categoricals", [True, False]) +@pytest.mark.parametrize( + ("write", "read", "suffix"), [(write_h5ed, read_h5ed, "h5ed"), (write_zarr, read_zarr, "zarr")] +) +def test_write_read_cast_columns_convert_strings_to_categoricals( + write, read, suffix, convert_strings_to_categoricals, tmp_path +): + # https://github.com/theislab/ehrdata/issues/358 + edata = _edata_with_columns_anndata_cannot_write(np.zeros((3, 2))) + path = tmp_path / f"string_columns.{suffix}" + + write(edata, path, convert_strings_to_categoricals=convert_strings_to_categoricals) + edata_read = read(path, harmonize_missing_values=False) + + # strings, datetimes and mixed values with repeated or missing values + for column in ["birth_datetime", "gender_source_value", "race_source_value", "death_date", "concept_code"]: + assert isinstance(edata_read.obs[column].dtype, pd.CategoricalDtype) is convert_strings_to_categoricals + assert isinstance(edata_read.tem["phase"].dtype, pd.CategoricalDtype) is convert_strings_to_categoricals + # no repeated or missing values + assert not isinstance(edata_read.obs["person_source_value"].dtype, pd.CategoricalDtype) + assert not isinstance(edata_read.var["valid_start_date"].dtype, pd.CategoricalDtype) + assert not isinstance(edata_read.tem["timestamp"].dtype, pd.CategoricalDtype) + + _assert_columns_anndata_cannot_write_read_back(edata, edata_read) + + +@pytest.mark.parametrize( + ("kwargs", "converted"), + [ + ({"convert_strings_to_categoricals": True}, True), + ({"convert_strings_to_categoricals": False}, False), + ({}, False), + ], + ids=["convert", "keep", "default"], +) +def test_write_read_string_columns_same_from_h5ed_and_zarr(edata_string_columns, kwargs, converted, tmp_path): + # https://github.com/theislab/ehrdata/issues/358 + edata = edata_string_columns + obs, var, tem = edata.obs.copy(), edata.var.copy(), edata.tem.copy() + + write_h5ed(edata, tmp_path / "edata.h5ed", **kwargs) + write_zarr(edata, tmp_path / "edata.zarr", **kwargs) + edata_read = read_zarr(tmp_path / "edata.zarr") + + # repeated or missing values + for frame, column in [ + ("obs", "sex"), + ("obs", "site"), + ("var", "unit"), + ("tem", "kind"), + ("tem", "date"), + ("tem", "duration"), + ]: + assert (getattr(edata_read, frame)[column].dtype == "category") is converted, column + # unique values + for frame, column in [("obs", "person_source_value"), ("var", "name"), ("tem", "label")]: + assert getattr(edata_read, frame)[column].dtype != "category", column + assert edata_read.obs["cohort"].dtype == "category" + assert edata_read.obs["sex"].astype(str).tolist() == ["F", "M", "F"] + assert edata_read.tem["date"].astype(str).tolist() == [ + "2000-01-01T00:00:00", + "2000-01-01T00:00:00", + "2000-01-02T00:00:00", + ] + + # .h5ed reads back the same, also backed + for edata_h5ed in [ + read_h5ed(tmp_path / "edata.h5ed"), + read_h5ed(tmp_path / "edata.h5ed", backed="r", harmonize_missing_values=False, cast_variables_to_float=False), + ]: + for frame in ("obs", "var", "tem"): + pd.testing.assert_frame_equal(getattr(edata_h5ed, frame), getattr(edata_read, frame)) + + # the input is left unchanged + pd.testing.assert_frame_equal(edata.obs, obs) + pd.testing.assert_frame_equal(edata.var, var) + pd.testing.assert_frame_equal(edata.tem, tem) + + @pytest.mark.parametrize( ("write", "read", "suffix"), [(write_h5ed, read_h5ed, "h5ed"), (write_zarr, read_zarr, "zarr")] ) diff --git a/tests/io/test_zarr.py b/tests/io/test_zarr.py index a1fec6ee..5cd1b65f 100644 --- a/tests/io/test_zarr.py +++ b/tests/io/test_zarr.py @@ -136,20 +136,17 @@ def test_write_read_zarr_basic(edata_name, chunks, request, tmp_path): _assert_dtype_object_array_with_missing_values_equal(edata.layers[key], edata_read.layers[key]) _check_aligned_anndata_parts_equal(edata, edata_read) - pd.testing.assert_frame_equal(edata.tem.iloc[:, :1], edata_read.tem.iloc[:, :1]) + pd.testing.assert_frame_equal(edata.tem, edata_read.tem) # check the test file is an ehrdata zarr store store = zarr.open(store_path) assert store.attrs["encoding-type"] == "ehrdata" assert store.attrs["ehrdata-encoding-version"] == EHRDATA_ONDISK_VERSION - # check success of convert_strings_to_categoricals - if "obs_col_2" in edata_read.obs.columns: - assert edata_read.obs["obs_col_2"].dtype == "category" - if "var_col_2" in edata_read.var.columns: - assert edata_read.var["var_col_2"].dtype == "category" - if "tem_col_2" in edata_read.tem.columns: - assert edata_read.tem["tem_col_2"].dtype == "category" + # strings are kept by default + for frame, column in [("obs", "obs_col_2"), ("var", "var_col_2")]: + if column in getattr(edata, frame).columns: + assert getattr(edata_read, frame)[column].dtype == getattr(edata, frame)[column].dtype def test_write_zarr_chunks_error(edata_333, tmp_path): @@ -341,16 +338,3 @@ def test_write_read_zarr_casts_object_layers(X_any_array_type_322, with_X, tmp_p write_zarr(edata, path) _assert_object_layers_read_back(edata, read_zarr(path)) - - -def test_write_zarr_keeps_string_columns_without_convert_strings_to_categoricals(X_any_array_type_322, tmp_path): - edata = _edata_with_columns_anndata_cannot_write(X_any_array_type_322) - path = tmp_path / "string_columns.zarr" - - write_zarr(edata, path, convert_strings_to_categoricals=False) - edata_read = read_zarr(path, harmonize_missing_values=False) - - # each would become categorical if converted - for column in ["birth_datetime", "gender_source_value", "race_source_value", "death_date", "concept_code"]: - assert not isinstance(edata_read.obs[column].dtype, pd.CategoricalDtype) - assert not isinstance(edata_read.tem["phase"].dtype, pd.CategoricalDtype)