Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ and this project adheres to [Semantic Versioning][].
- {func}`~ehrdata.io.omop.setup_variables` takes `value_range` to leave out implausible data points before the aggregation, as a mapping of variables to bounds such as `value_range={"heart_rate": (0, 300)}`. The data points outside the range neither count nor tell a unit. ([#347](https://github.com/theislab/ehrdata/pull/347)) @eroell
- {func}`~ehrdata.io.omop.setup_variables` takes `unit_conversions`, a table of factors and offsets from the units a concept was recorded in to one unit, to convert the values before the aggregation and the value range. Converted data points carry the target unit in `unit_concept_id` and `unit_source_value`, and a data point in a unit the table does not list for its concept raises. ([#348](https://github.com/theislab/ehrdata/pull/348)) @eroell
- {func}`~ehrdata.io.omop.setup_variables` and {func}`~ehrdata.io.omop.setup_interval_variables` take `num_intervals_before` to add intervals before the start of each observation, such as a look-back before a visit. ([#350](https://github.com/theislab/ehrdata/pull/350)) @eroell
- {func}`~ehrdata.io.write_h5ed` takes `convert_strings_to_categoricals`, as {func}`~ehrdata.io.write_zarr` does, and both convert the same columns: the string columns of `.obs`, `.var` and `.tem` in which a value repeats or is missing, including the datetimes and timedeltas written as ISO 8601 strings. ([#358](https://github.com/theislab/ehrdata/issues/358)) @eroell

### Changed

Expand All @@ -27,6 +28,7 @@ and this project adheres to [Semantic Versioning][].
- {func}`~ehrdata.io.omop.setup_interval_variables` aggregates only the intervals that hold data points, as {func}`~ehrdata.io.omop.setup_variables` does, so memory and time follow the number of data points. ([#349](https://github.com/theislab/ehrdata/pull/349)) @eroell
- {func}`~ehrdata.io.omop.setup_variables` and {func}`~ehrdata.io.omop.setup_interval_variables` with `aggregation_strategy="mode"` keep the smallest of several equally frequent values, instead of an arbitrary one. ([#341](https://github.com/theislab/ehrdata/pull/341)) @eroell
- {func}`~ehrdata.io.omop.setup_variables` and {func}`~ehrdata.io.omop.setup_interval_variables` with `observation_table="person_visit_occurrence"` attribute a data point whose `visit_occurrence_id` names a visit of its person to that visit only. Before, a data point belonged to every visit of its person whose window held it, so overlapping visits of a person shared their data points, and with `clip_to_end_date=False` a visit's intervals after its end took the data points of the person's next visit. Data points without a `visit_occurrence_id`, or with one that names no visit of their person, still belong to every visit whose window holds them. ([#344](https://github.com/theislab/ehrdata/pull/344)) @eroell
- {func}`~ehrdata.io.write_h5ed` and {func}`~ehrdata.io.write_zarr` keep string columns as strings by default (`convert_strings_to_categoricals=False`), so an {class}`~ehrdata.EHRData` reads back with the dtypes it was written with, from `.h5ed` and `.zarr` alike. Before, {func}`~ehrdata.io.write_h5ed` converted the string columns of `.obs` and `.var` in which a value repeats or is missing to categoricals, and {func}`~ehrdata.io.write_zarr` also those of `.tem`. Files written before keep their categorical columns; pass `convert_strings_to_categoricals=True` for the previous conversion. ([#358](https://github.com/theislab/ehrdata/issues/358)) @eroell

### Fixed

Expand Down
13 changes: 13 additions & 0 deletions src/ehrdata/io/_ondisk.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,8 @@
if TYPE_CHECKING:
from collections.abc import Callable

import pandas as pd

from ehrdata import EHRData
from ehrdata._types import GroupStorageType

Expand Down Expand Up @@ -127,6 +129,17 @@ def encode_for_disk(edata: EHRData) -> tuple[ad.AnnData, dict[str, sparse.COO]]:
return adata, sparse_3d_data


def strings_to_categoricals(adata: ad.AnnData, tem: pd.DataFrame) -> None:
"""Convert the string columns of ``.obs``, ``.var`` and ``.tem`` in which a value repeats or is missing to categoricals.

Converts in place, so pass the copies built for writing (see :func:`encode_for_disk`), never the caller's data.
The writers of every format call this, so that the same columns come back as categoricals from every format.
"""
adata.strings_to_categoricals(adata.obs)
adata.strings_to_categoricals(adata.var)
adata.strings_to_categoricals(tem)


def decode_init_dict(init: dict[str, Any]) -> dict[str, Any]:
"""Restore relocated 3D arrays from ``.obsm`` back into ``X``/``layers`` in a dict of init kwargs."""
obsm = dict(init.get("obsm") or {})
Expand Down
16 changes: 14 additions & 2 deletions src/ehrdata/io/h5ed.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
decode_init_dict,
encode_for_disk,
read_mapping_with_coo,
strings_to_categoricals,
)

if TYPE_CHECKING:
Expand Down Expand Up @@ -127,14 +128,15 @@ def write_h5ed(
*,
compression: Literal["gzip", "lzf"] | None = None,
compression_opts: int | None = None,
convert_strings_to_categoricals: bool = False,
) -> None:
"""Write :class:`~ehrdata.EHRData` objects to an ehrdata hdf5 file (`.h5ed`).

`.h5ed` is the ehrdata on-disk format.
To write the file, `X` and `layers` cannot be written as `object` dtype.
If any of these fields is of `object` dtype, this function will attempt to cast it to a numeric dtype; if this fails, the field will be casted to a string dtype.
The same holds for the `object` columns of `.obs`, `.var` and `.tem`; integers and booleans among them stay exact, and missing values stay missing.
Datetime and timedelta columns are written as ISO 8601 strings, read back as categorical in `.obs` and `.var` if a value repeats or is missing.
Datetime and timedelta columns are written as ISO 8601 strings, read back as categorical with `convert_strings_to_categoricals=True` if a value repeats or is missing.


Args:
Expand All @@ -143,6 +145,9 @@ def write_h5ed(
compression: Optional file compression.
Setting compression to 'gzip' can save disk space but will slow down writing and subsequent reading.
compression_opts: For [`lzf`, `gzip`], see the h5py :ref:`dataset_compression`.
convert_strings_to_categoricals: Convert the string columns of `.obs`, `.var` and `.tem` in which a value repeats or is missing to `categorical` dtype.
By default, strings are kept.
The :class:`~ehrdata.EHRData` passed is left unchanged.

Examples:
>>> import ehrdata as ed
Expand All @@ -152,10 +157,17 @@ def write_h5ed(
filename = Path(filename)

adata, sparse_3d_data = encode_for_disk(edata)
tem = _cast_dataframe_columns_to_writable_dtype(edata.tem, "tem")

if convert_strings_to_categoricals:
strings_to_categoricals(adata, tem)

adata.write_h5ad(
filename,
compression=compression,
compression_opts=compression_opts,
# converted above, with .tem; AnnData.write_h5ad would otherwise convert .obs and .var by default
convert_strings_to_categoricals=False,
)
with h5py.File(filename, "a") as f:
if sparse_3d_data:
Expand All @@ -164,7 +176,7 @@ def write_h5ed(
write_coo_h5(
obsm_group.create_group(key), coo, compression=compression, compression_opts=compression_opts
)
ad.io.write_elem(f, "tem", _cast_dataframe_columns_to_writable_dtype(edata.tem, "tem"))
ad.io.write_elem(f, "tem", tem)
# Identify the file as ehrdata, namespaced to not clash with anndata's own encoding attrs.
f.attrs[EHRDATA_ENCODING_TYPE_KEY] = EHRDATA_ENCODING_TYPE
f.attrs[EHRDATA_ONDISK_VERSION_KEY] = str(EHRDATA_ONDISK_VERSION)
Expand Down
13 changes: 7 additions & 6 deletions src/ehrdata/io/zarr.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
decode_init_dict,
encode_for_disk,
read_mapping_with_coo,
strings_to_categoricals,
)

if TYPE_CHECKING:
Expand Down Expand Up @@ -54,7 +55,7 @@ def read_zarr(
Examples:
>>> import ehrdata as ed
>>> edata = ed.dt.mimic_2()
>>> ed.io.write_zarr("mimic_2.zarr", edata)
>>> ed.io.write_zarr(edata, "mimic_2.zarr")
>>> edata_from_zarr = ed.io.read_zarr("mimic_2.zarr")
"""
from ehrdata import EHRData
Expand Down Expand Up @@ -117,7 +118,7 @@ def write_zarr(
filename: str | Path,
*,
chunks: Literal["auto" | "ehrdata_auto"] = "auto",
convert_strings_to_categoricals: bool = True,
convert_strings_to_categoricals: bool = False,
) -> None:
"""Write :class:`~ehrdata.EHRData` objects to disk.

Expand All @@ -131,7 +132,9 @@ def write_zarr(
edata: Central data object.
filename: Name of the output file, can also be prefixed with relative or absolute path to save the file to.
chunks: Specify strategy of how data should be chunked. For simplicity, currently only 2 options are available: `"auto"` will write the data with :func:`~anndata.io.write_elem`'s default settings. `"ehrdata_auto"` will write the data chunked (and sharded) based on a heuristic that loosely speaking writes slightly smaller chunks.
convert_strings_to_categoricals: Convert columns of `str` dtype in `.obs` and `.var` and `.tem` to `categorical` dtype.
convert_strings_to_categoricals: Convert the string columns of `.obs`, `.var` and `.tem` in which a value repeats or is missing to `categorical` dtype.
By default, strings are kept.
The :class:`~ehrdata.EHRData` passed is left unchanged.

Examples:
>>> import ehrdata as ed
Expand All @@ -146,9 +149,7 @@ def write_zarr(
tem = _cast_dataframe_columns_to_writable_dtype(edata.tem, "tem")

if convert_strings_to_categoricals:
adata.strings_to_categoricals(adata.obs)
adata.strings_to_categoricals(adata.var)
adata.strings_to_categoricals(tem)
strings_to_categoricals(adata, tem)

# write_sharded this is a slightly modified version from https://anndata.readthedocs.io/en/stable/tutorials/zarr-v3.html
# write_sharded is intended as a future blueprint of implementing better chunking defaults for ehrdata based based on real usecases
Expand Down
29 changes: 29 additions & 0 deletions tests/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -344,6 +344,35 @@ def edata_330(X_numpy_33, obs_31, var_31):
return EHRData(X=X_numpy_33, obs=obs_31, var=var_31)


@pytest.fixture
def edata_string_columns():
"""EHRData whose `.obs`, `.var` and `.tem` hold string columns with and without repeated or missing values."""
# 2D, as backed reading does not support 3D arrays
return EHRData(
X=np.zeros((3, 2)),
obs=pd.DataFrame(
{
"sex": ["F", "M", "F"],
"person_source_value": ["p1", "p2", "p3"],
"site": pd.array(["a", "a", pd.NA], dtype="string"),
"cohort": pd.Categorical(["x", "y", "z"]),
},
index=["0", "1", "2"],
),
var=pd.DataFrame({"unit": ["bpm", "bpm"], "name": ["heart rate", "blood pressure"]}, index=["hr", "bp"]),
tem=pd.DataFrame(
{
"kind": ["visit", "visit", "follow-up"],
"label": ["first", "second", "third"],
# written as ISO 8601 strings
"date": pd.to_datetime(["2000-01-01", "2000-01-01", "2000-01-02"]),
"duration": pd.to_timedelta(["1h", "1h", "2h"]),
},
index=["t0", "t1", "t2"],
),
)


@pytest.fixture
def adata_33(X_numpy_33, obs_31, var_31):
return ad.AnnData(X=X_numpy_33, obs=obs_31, var=var_31)
Expand Down
79 changes: 79 additions & 0 deletions tests/io/test_h5ed.py
Original file line number Diff line number Diff line change
Expand Up @@ -416,6 +416,85 @@ def test_write_read_h5ed_casts_object_layers(X_any_array_type_322, with_X, tmp_p
_assert_object_layers_read_back(edata, read_h5ed(path))


@pytest.mark.parametrize("convert_strings_to_categoricals", [True, False])
@pytest.mark.parametrize(
("write", "read", "suffix"), [(write_h5ed, read_h5ed, "h5ed"), (write_zarr, read_zarr, "zarr")]
)
def test_write_read_cast_columns_convert_strings_to_categoricals(
write, read, suffix, convert_strings_to_categoricals, tmp_path
):
# https://github.com/theislab/ehrdata/issues/358
edata = _edata_with_columns_anndata_cannot_write(np.zeros((3, 2)))
path = tmp_path / f"string_columns.{suffix}"

write(edata, path, convert_strings_to_categoricals=convert_strings_to_categoricals)
edata_read = read(path, harmonize_missing_values=False)

# strings, datetimes and mixed values with repeated or missing values
for column in ["birth_datetime", "gender_source_value", "race_source_value", "death_date", "concept_code"]:
assert isinstance(edata_read.obs[column].dtype, pd.CategoricalDtype) is convert_strings_to_categoricals
assert isinstance(edata_read.tem["phase"].dtype, pd.CategoricalDtype) is convert_strings_to_categoricals
# no repeated or missing values
assert not isinstance(edata_read.obs["person_source_value"].dtype, pd.CategoricalDtype)
assert not isinstance(edata_read.var["valid_start_date"].dtype, pd.CategoricalDtype)
assert not isinstance(edata_read.tem["timestamp"].dtype, pd.CategoricalDtype)

_assert_columns_anndata_cannot_write_read_back(edata, edata_read)


@pytest.mark.parametrize(
("kwargs", "converted"),
[
({"convert_strings_to_categoricals": True}, True),
({"convert_strings_to_categoricals": False}, False),
({}, False),
],
ids=["convert", "keep", "default"],
)
def test_write_read_string_columns_same_from_h5ed_and_zarr(edata_string_columns, kwargs, converted, tmp_path):
# https://github.com/theislab/ehrdata/issues/358
edata = edata_string_columns
obs, var, tem = edata.obs.copy(), edata.var.copy(), edata.tem.copy()

write_h5ed(edata, tmp_path / "edata.h5ed", **kwargs)
write_zarr(edata, tmp_path / "edata.zarr", **kwargs)
edata_read = read_zarr(tmp_path / "edata.zarr")

# repeated or missing values
for frame, column in [
("obs", "sex"),
("obs", "site"),
("var", "unit"),
("tem", "kind"),
("tem", "date"),
("tem", "duration"),
]:
assert (getattr(edata_read, frame)[column].dtype == "category") is converted, column
# unique values
for frame, column in [("obs", "person_source_value"), ("var", "name"), ("tem", "label")]:
assert getattr(edata_read, frame)[column].dtype != "category", column
assert edata_read.obs["cohort"].dtype == "category"
assert edata_read.obs["sex"].astype(str).tolist() == ["F", "M", "F"]
assert edata_read.tem["date"].astype(str).tolist() == [
"2000-01-01T00:00:00",
"2000-01-01T00:00:00",
"2000-01-02T00:00:00",
]

# .h5ed reads back the same, also backed
for edata_h5ed in [
read_h5ed(tmp_path / "edata.h5ed"),
read_h5ed(tmp_path / "edata.h5ed", backed="r", harmonize_missing_values=False, cast_variables_to_float=False),
]:
for frame in ("obs", "var", "tem"):
pd.testing.assert_frame_equal(getattr(edata_h5ed, frame), getattr(edata_read, frame))

# the input is left unchanged
pd.testing.assert_frame_equal(edata.obs, obs)
pd.testing.assert_frame_equal(edata.var, var)
pd.testing.assert_frame_equal(edata.tem, tem)


@pytest.mark.parametrize(
("write", "read", "suffix"), [(write_h5ed, read_h5ed, "h5ed"), (write_zarr, read_zarr, "zarr")]
)
Expand Down
26 changes: 5 additions & 21 deletions tests/io/test_zarr.py
Original file line number Diff line number Diff line change
Expand Up @@ -136,20 +136,17 @@ def test_write_read_zarr_basic(edata_name, chunks, request, tmp_path):
_assert_dtype_object_array_with_missing_values_equal(edata.layers[key], edata_read.layers[key])

_check_aligned_anndata_parts_equal(edata, edata_read)
pd.testing.assert_frame_equal(edata.tem.iloc[:, :1], edata_read.tem.iloc[:, :1])
pd.testing.assert_frame_equal(edata.tem, edata_read.tem)

# check the test file is an ehrdata zarr store
store = zarr.open(store_path)
assert store.attrs["encoding-type"] == "ehrdata"
assert store.attrs["ehrdata-encoding-version"] == EHRDATA_ONDISK_VERSION

# check success of convert_strings_to_categoricals
if "obs_col_2" in edata_read.obs.columns:
assert edata_read.obs["obs_col_2"].dtype == "category"
if "var_col_2" in edata_read.var.columns:
assert edata_read.var["var_col_2"].dtype == "category"
if "tem_col_2" in edata_read.tem.columns:
assert edata_read.tem["tem_col_2"].dtype == "category"
# strings are kept by default
for frame, column in [("obs", "obs_col_2"), ("var", "var_col_2")]:
if column in getattr(edata, frame).columns:
assert getattr(edata_read, frame)[column].dtype == getattr(edata, frame)[column].dtype


def test_write_zarr_chunks_error(edata_333, tmp_path):
Expand Down Expand Up @@ -341,16 +338,3 @@ def test_write_read_zarr_casts_object_layers(X_any_array_type_322, with_X, tmp_p
write_zarr(edata, path)

_assert_object_layers_read_back(edata, read_zarr(path))


def test_write_zarr_keeps_string_columns_without_convert_strings_to_categoricals(X_any_array_type_322, tmp_path):
edata = _edata_with_columns_anndata_cannot_write(X_any_array_type_322)
path = tmp_path / "string_columns.zarr"

write_zarr(edata, path, convert_strings_to_categoricals=False)
edata_read = read_zarr(path, harmonize_missing_values=False)

# each would become categorical if converted
for column in ["birth_datetime", "gender_source_value", "race_source_value", "death_date", "concept_code"]:
assert not isinstance(edata_read.obs[column].dtype, pd.CategoricalDtype)
assert not isinstance(edata_read.tem["phase"].dtype, pd.CategoricalDtype)
Loading