Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ and this project adheres to [Semantic Versioning][].
- {func}`~ehrdata.io.omop.setup_variables` takes `unit_conversions`, a table of factors and offsets from the units a concept was recorded in to one unit, to convert the values before the aggregation and the value range. Converted data points carry the target unit in `unit_concept_id` and `unit_source_value`, and a data point in a unit the table does not list for its concept raises. ([#348](https://github.com/theislab/ehrdata/pull/348)) @eroell
- {func}`~ehrdata.io.omop.setup_variables` and {func}`~ehrdata.io.omop.setup_interval_variables` take `num_intervals_before` to add intervals before the start of each observation, such as a look-back before a visit. ([#350](https://github.com/theislab/ehrdata/pull/350)) @eroell
- {func}`~ehrdata.io.write_h5ed` takes `convert_strings_to_categoricals`, as {func}`~ehrdata.io.write_zarr` does, and both convert the same columns: the string columns of `.obs`, `.var` and `.tem` in which a value repeats or is missing, including the datetimes and timedeltas written as ISO 8601 strings. ([#358](https://github.com/theislab/ehrdata/issues/358)) @eroell
- {func}`~ehrdata.io.write_elem` and {func}`~ehrdata.io.read_elem` write and read an {class}`~ehrdata.EHRData` into and from a group of an HDF5 file or a Zarr store, such as a file that holds other data next to it. The group holds the same layout as a file written with {func}`~ehrdata.io.write_h5ed` or a store written with {func}`~ehrdata.io.write_zarr`, with which they share the implementation; the on-disk format is unchanged. ([#360](https://github.com/theislab/ehrdata/pull/360)) @eroell

### Changed

Expand Down
2 changes: 2 additions & 0 deletions docs/api/io_index.md
Original file line number Diff line number Diff line change
Expand Up @@ -13,10 +13,12 @@
:nosignatures:

io.read_csv
io.read_elem
io.read_fhir
io.read_h5ed
io.read_meds
io.read_zarr
io.write_elem
io.write_h5ed
io.write_meds
io.write_zarr
Expand Down
1 change: 1 addition & 0 deletions src/ehrdata/io/__init__.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
from . import omop
from .csv import read_csv
from .elem import read_elem, write_elem
from .events import from_events
from .fhir import read_fhir
from .h5ed import read_h5ad, read_h5ed, write_h5ad, write_h5ed
Expand Down
32 changes: 31 additions & 1 deletion src/ehrdata/io/_ondisk.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
EHRDATA_ONDISK_VERSION_KEY,
)
from ehrdata.io._array_casting import _cast_dataframe_columns_to_writable_dtype, _cast_object_array_to_float_or_str
from ehrdata.io._coo_codec import is_coo_group, read_coo
from ehrdata.io._coo_codec import _binsparse_dtype_str, is_coo_group, read_coo

if TYPE_CHECKING:
from collections.abc import Callable
Expand Down Expand Up @@ -140,6 +140,23 @@ def strings_to_categoricals(adata: ad.AnnData, tem: pd.DataFrame) -> None:
adata.strings_to_categoricals(tem)


def encode_for_writing(
edata: EHRData, *, convert_strings_to_categoricals: bool
) -> tuple[ad.AnnData, dict[str, sparse.COO], pd.DataFrame]:
"""Encode and check an :class:`~ehrdata.EHRData` for every writer, before anything is written.

Returns the :class:`~anndata.AnnData` and the sparse 3D arrays of :func:`encode_for_disk`, and the cast `.tem`.
Raises for data that cannot be written, so that a writer does not replace existing data with a failed write.
"""
adata, sparse_3d_data = encode_for_disk(edata)
for coo in sparse_3d_data.values():
_binsparse_dtype_str(coo.dtype)
tem = _cast_dataframe_columns_to_writable_dtype(edata.tem, "tem")
if convert_strings_to_categoricals:
strings_to_categoricals(adata, tem)
return adata, sparse_3d_data, tem


def decode_init_dict(init: dict[str, Any]) -> dict[str, Any]:
"""Restore relocated 3D arrays from ``.obsm`` back into ``X``/``layers`` in a dict of init kwargs."""
obsm = dict(init.get("obsm") or {})
Expand Down Expand Up @@ -174,6 +191,19 @@ def read_mapping_with_coo(group: GroupStorageType) -> dict[str, Any]:
return out


# other children are ignored, as by anndata
_INIT_KEYS = ("X", "obs", "var", "obsm", "varm", "obsp", "varp", "layers", "uns", "raw", "tem")


def read_init_dict(group: GroupStorageType) -> dict[str, Any]:
"""Read the AnnData slots and `.tem` of a group as init kwargs, ignoring other children."""
return {
k: read_mapping_with_coo(group[k]) if k == "obsm" else ad.io.read_elem(group[k])
for k in _INIT_KEYS
if k in group
}


def _check_020_ehrdata_on_disk_format(f: Any) -> bool:
"""Return True if the file/store carries the ehrdata 0.2.0 on-disk stamp.

Expand Down
136 changes: 136 additions & 0 deletions src/ehrdata/io/elem.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,136 @@
from __future__ import annotations

from typing import TYPE_CHECKING, Literal

import h5py
import zarr

from ehrdata.io.h5ed import _read_h5ed_elem, _write_h5ed_elem
from ehrdata.io.zarr import _read_zarr_elem, _write_zarr_elem

if TYPE_CHECKING:
from ehrdata import EHRData


def write_elem(
parent: h5py.Group | zarr.Group,
key: str,
edata: EHRData,
*,
convert_strings_to_categoricals: bool = False,
compression: Literal["gzip", "lzf"] | None = None,
compression_opts: int | None = None,
chunks: Literal["auto", "ehrdata_auto"] = "auto",
) -> None:
"""Write :class:`~ehrdata.EHRData` objects into a group of an HDF5 file or a Zarr store.

Use this to write an :class:`~ehrdata.EHRData` as one part of a larger file or store, next to other data.
`parent[key]` holds the same layout as a file written with :func:`~ehrdata.io.write_h5ed` for an :class:`h5py.Group`,
and as a store written with :func:`~ehrdata.io.write_zarr` for a :class:`zarr.Group`, and
the data is cast for writing as these functions do.
An existing `parent[key]` is replaced, as with :func:`anndata.io.write_elem`.
Zarr stores with consolidated metadata cannot be written to.
Read it back with :func:`~ehrdata.io.read_elem`.

Args:
parent: The group to write into, such as an open :class:`h5py.File`, a :class:`zarr.Group`, or a subgroup of these.
Zarr groups of Zarr format 2 and 3 are supported.
key: The name of the new group in `parent`, without `/`.
To write into a nested group, pass that group as `parent`.
To write a whole file or store, use :func:`~ehrdata.io.write_h5ed` or :func:`~ehrdata.io.write_zarr`.
edata: Central data object.
convert_strings_to_categoricals: Convert the string columns of `.obs`, `.var` and `.tem` in which a value repeats or is missing to `categorical` dtype.
By default, strings are kept.
The :class:`~ehrdata.EHRData` passed is left unchanged.
compression: Optional compression for an :class:`h5py.Group`, see :func:`~ehrdata.io.write_h5ed`.
Raises for a :class:`zarr.Group`.
compression_opts: For [`lzf`, `gzip`], see the h5py :ref:`dataset_compression`.
Raises for a :class:`zarr.Group`.
chunks: Chunking strategy for a :class:`zarr.Group`, see :func:`~ehrdata.io.write_zarr`.
`"ehrdata_auto"` needs a group of Zarr format 3, and raises for an :class:`h5py.Group`.

Examples:
>>> import h5py
>>> import ehrdata as ed
>>> edata = ed.dt.ehrdata_blobs(n_variables=3, n_observations=5)
>>> with h5py.File("bundle.h5", "w") as f:
... ed.io.write_elem(f, "edata", edata)
... f["other"] = [1, 2, 3]
"""
if not isinstance(key, str):
msg = f"key must be a str, not {type(key).__name__}."
raise TypeError(msg)
if key == "/":
msg = "key='/' would write the whole file or store; use write_h5ed or write_zarr instead."
raise ValueError(msg)
if not key or "/" in key:
msg = f"key {key!r} must be a non-empty name without '/'; pass a nested group as `parent` instead."
raise ValueError(msg)

if isinstance(parent, h5py.Group):
if chunks != "auto":
msg = "`chunks` only applies to a zarr.Group; use `compression` for an h5py.Group."
raise ValueError(msg)
_write_h5ed_elem(
parent,
key,
edata,
compression=compression,
compression_opts=compression_opts,
convert_strings_to_categoricals=convert_strings_to_categoricals,
)
elif isinstance(parent, zarr.Group):
if compression is not None or compression_opts is not None:
msg = "`compression` and `compression_opts` only apply to an h5py.Group; use `chunks` for a zarr.Group."
raise ValueError(msg)
_write_zarr_elem(
parent, key, edata, chunks=chunks, convert_strings_to_categoricals=convert_strings_to_categoricals
)
else:
msg = f"parent must be an h5py.Group or a zarr.Group, not {type(parent).__name__}."
raise TypeError(msg)


def read_elem(
group: h5py.Group | zarr.Group,
*,
harmonize_missing_values: bool = True,
cast_variables_to_float: bool = True,
) -> EHRData:
"""Read an :class:`~ehrdata.EHRData` from a group of an HDF5 file or a Zarr store.

Reads a group written with :func:`~ehrdata.io.write_elem`, :func:`~ehrdata.io.write_h5ed` or :func:`~ehrdata.io.write_zarr`,
fully into memory.
Like :func:`~ehrdata.io.read_h5ed` and :func:`~ehrdata.io.read_zarr`, it also reads :class:`~anndata.AnnData` groups.

Args:
group: The group to read, such as `f["edata"]` of an open :class:`h5py.File` or a :class:`zarr.Group`.
harmonize_missing_values: Whether to call `ehrdata.harmonize_missing_values` on all detected layers.
cast_variables_to_float: For non-numeric arrays, try to cast the values for each variable to dtype `np.float64`.
If the cast fails for the values of one variable, then the values of these variable remain unaltered.
This can be helpful to recover arrays that were of dtype `object` when they were written to disk.

Examples:
>>> import h5py
>>> import ehrdata as ed
>>> edata = ed.dt.ehrdata_blobs(n_variables=3, n_observations=5)
>>> with h5py.File("bundle.h5", "w") as f:
... ed.io.write_elem(f, "edata", edata)
>>> with h5py.File("bundle.h5", "r") as f:
... edata_2 = ed.io.read_elem(f["edata"])
"""
if isinstance(group, h5py.Group):
if group.attrs.get("encoding-type") != "anndata":
msg = (
f"The group {group.name!r} holds no EHRData or AnnData: its 'encoding-type' attribute is not 'anndata'."
)
raise ValueError(msg)
return _read_h5ed_elem(
group, harmonize_missing_values=harmonize_missing_values, cast_variables_to_float=cast_variables_to_float
)
if isinstance(group, zarr.Group):
return _read_zarr_elem(
group, harmonize_missing_values=harmonize_missing_values, cast_variables_to_float=cast_variables_to_float
)
msg = f"group must be an h5py.Group or a zarr.Group, not {type(group).__name__}."
raise TypeError(msg)
103 changes: 60 additions & 43 deletions src/ehrdata/io/h5ed.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
from __future__ import annotations

import warnings
from pathlib import Path
from typing import TYPE_CHECKING, Literal

import anndata as ad
Expand All @@ -17,21 +16,19 @@
EHRDATA_ONDISK_VERSION_KEY,
)
from ehrdata.core.ehrdata import _silence_anndata_nd_warning
from ehrdata.io._array_casting import (
_cast_dataframe_columns_to_writable_dtype,
_cast_variables_to_float,
)
from ehrdata.io._array_casting import _cast_variables_to_float
from ehrdata.io._coo_codec import write_coo_h5
from ehrdata.io._ondisk import (
_allow_write_nullable_strings,
_check_020_ehrdata_on_disk_format,
decode_init_dict,
encode_for_disk,
read_mapping_with_coo,
strings_to_categoricals,
encode_for_writing,
read_init_dict,
)

if TYPE_CHECKING:
from pathlib import Path

from ehrdata import EHRData


Expand Down Expand Up @@ -78,19 +75,10 @@ def read_h5ed(
raise ValueError(msg)

if backed in {None, False}:
dictionary_for_init = {}

with h5py.File(filename, "r") as f:
for k in dict(f):
if k.startswith("raw."):
continue
# obsm may hold ehrdata COO groups anndata can't read, so read it child-by-child.
dictionary_for_init[k] = read_mapping_with_coo(f["obsm"]) if k == "obsm" else ad.io.read_elem(f[k])
is_ehrdata_020 = _check_020_ehrdata_on_disk_format(f)

if is_ehrdata_020:
dictionary_for_init = decode_init_dict(dictionary_for_init)
edata = EHRData(**dictionary_for_init)
return _read_h5ed_elem(
f, harmonize_missing_values=harmonize_missing_values, cast_variables_to_float=cast_variables_to_float
)

else:
mode = backed
Expand All @@ -112,6 +100,19 @@ def read_h5ed(
raise NotImplementedError(msg_0)
edata = EHRData.from_adata(adata, tem=tem)

return edata


def _read_h5ed_elem(group: h5py.Group, *, harmonize_missing_values: bool, cast_variables_to_float: bool) -> EHRData:
"""Read an :class:`~ehrdata.EHRData` from an HDF5 group in the `.h5ed` layout, fully into memory."""
from ehrdata import EHRData

dictionary_for_init = read_init_dict(group)

if _check_020_ehrdata_on_disk_format(group):
dictionary_for_init = decode_init_dict(dictionary_for_init)
edata = EHRData(**dictionary_for_init)

if harmonize_missing_values:
_harmonize_on_read(edata)

Expand All @@ -121,7 +122,6 @@ def read_h5ed(
return edata


@_allow_write_nullable_strings
def write_h5ed(
edata: EHRData,
filename: str | Path,
Expand Down Expand Up @@ -154,32 +154,49 @@ def write_h5ed(
>>> edata = ed.dt.mimic_2()
>>> ed.io.write_h5ed(edata, "mimic_2.h5ed")
"""
filename = Path(filename)
with h5py.File(filename, "w") as f:
_write_h5ed_elem(
f,
"/",
edata,
compression=compression,
compression_opts=compression_opts,
convert_strings_to_categoricals=convert_strings_to_categoricals,
)

adata, sparse_3d_data = encode_for_disk(edata)
tem = _cast_dataframe_columns_to_writable_dtype(edata.tem, "tem")

if convert_strings_to_categoricals:
strings_to_categoricals(adata, tem)
@_allow_write_nullable_strings
def _write_h5ed_elem(
parent: h5py.Group,
key: str,
edata: EHRData,
*,
compression: Literal["gzip", "lzf"] | None,
compression_opts: int | None,
convert_strings_to_categoricals: bool,
) -> None:
"""Write an :class:`~ehrdata.EHRData` to `parent[key]` in the `.h5ed` layout; `key="/"` writes into the file root."""
adata, sparse_3d_data, tem = encode_for_writing(
edata, convert_strings_to_categoricals=convert_strings_to_categoricals
)

adata.write_h5ad(
filename,
compression=compression,
compression_opts=compression_opts,
# converted above, with .tem; AnnData.write_h5ad would otherwise convert .obs and .var by default
convert_strings_to_categoricals=False,
ad.io.write_elem(
parent, key, adata, dataset_kwargs={"compression": compression, "compression_opts": compression_opts}
)
with h5py.File(filename, "a") as f:
if sparse_3d_data:
obsm_group = f.require_group("obsm")
for key, coo in sparse_3d_data.items():
write_coo_h5(
obsm_group.create_group(key), coo, compression=compression, compression_opts=compression_opts
)
ad.io.write_elem(f, "tem", tem)
# Identify the file as ehrdata, namespaced to not clash with anndata's own encoding attrs.
f.attrs[EHRDATA_ENCODING_TYPE_KEY] = EHRDATA_ENCODING_TYPE
f.attrs[EHRDATA_ONDISK_VERSION_KEY] = str(EHRDATA_ONDISK_VERSION)
group = parent[key]
# anndata writes a null `raw`; .h5ed files have none
if "raw" in group:
del group["raw"]
if sparse_3d_data:
obsm_group = group.require_group("obsm")
for obsm_key, coo in sparse_3d_data.items():
write_coo_h5(
obsm_group.create_group(obsm_key), coo, compression=compression, compression_opts=compression_opts
)
ad.io.write_elem(group, "tem", tem)
# Identify the group as ehrdata, namespaced to not clash with anndata's own encoding attrs.
group.attrs[EHRDATA_ENCODING_TYPE_KEY] = EHRDATA_ENCODING_TYPE
group.attrs[EHRDATA_ONDISK_VERSION_KEY] = str(EHRDATA_ONDISK_VERSION)


def read_h5ad(
Expand Down
Loading
Loading