Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion docs/COMMANDS.md
Original file line number Diff line number Diff line change
Expand Up @@ -287,6 +287,8 @@ adata concat a.h5ad b.h5ad -o m.h5ad --keys a,b --index-unique - --uns-merge sam
| `--index-unique` | Delimiter for suffixing obs names with their key |
| `--merge` / `--uns-merge` | `drop` (default), `same`, `unique`, `first`, `only` |
| `--fill-value` | Value for dense cells introduced by an outer join |
| `--chunk`, `-C` | Row chunk size while streaming |
| `--zarr-format` | Zarr version to write; defaults to the inputs' when they agree, otherwise v3 |

obs columns keep their dtypes: categoricals union their category sets, nullable
columns keep their masks, and a column missing from one input is padded rather
Expand Down Expand Up @@ -399,7 +401,8 @@ adata subset data.h5ad -o cortex.h5ad --obs barcodes.txt
New Zarr stores follow the source store's version, so a v2 input is not
silently upgraded. `--zarr-format 2|3` on `create`, `subset`, `split`,
`concat`, `convert` and `recast` overrides that. Writing a Zarr store from
an `.h5ad` source defaults to v3.
an `.h5ad` source defaults to v3. `concat` follows its Zarr inputs when they
share one version (h5ad inputs do not count) and writes v3 when they differ.

```bash
adata create out.zarr --n-obs 100 --n-var 50 --zarr-format 2
Expand Down
4 changes: 3 additions & 1 deletion src/adata/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -433,7 +433,9 @@ def concat(
1024, "--chunk", "-C", help="Row chunk size while streaming"
),
zarr_format: Optional[int] = typer.Option(
None, "--zarr-format", help="Zarr spec version to write (2 or 3)"
None,
"--zarr-format",
help="Zarr spec version to write (defaults to the inputs' when they agree)",
),
) -> None:
"""
Expand Down
15 changes: 15 additions & 0 deletions src/adata/core/concat.py
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@
copy_tree,
create_dataset,
is_dataset,
Store,
is_group,
open_store,
)
Expand Down Expand Up @@ -828,6 +829,17 @@ def _hashable(value: Any) -> Any:
return value


def _inputs_zarr_format(stores: Sequence[Store]) -> Optional[int]:
"""The Zarr version the inputs agree on, or None to take the default.

A Zarr output follows its inputs, as every other command's follows its
source, so v2 inputs are not silently upgraded. h5ad inputs have no
version and do not count; Zarr inputs of both versions get the default.
"""
versions = {s.zarr_format for s in stores} - {None}
return versions.pop() if len(versions) == 1 else None


def concat_on_disk(
files: Sequence[Path],
output: Path,
Expand Down Expand Up @@ -907,6 +919,9 @@ def concat_on_disk(
# half-written store behind.
check_matrix_encodings(roots, console)

if zarr_format is None:
zarr_format = _inputs_zarr_format(stores)

# Written beside the output and renamed into place once complete, so
# a failure part way leaves nothing that reads as a finished store.
with staged_outputs([output]) as (temp,), open_store(
Expand Down
52 changes: 52 additions & 0 deletions tests/test_commands_phase2.py
Original file line number Diff line number Diff line change
Expand Up @@ -413,6 +413,58 @@ def test_concat_drops_uns_by_default(tmp_path):
assert dict(ad.read_h5ad(out).uns) == {}


def _as_zarr(h5ad: Path, version: int) -> Path:
"""The same store as Zarr of the given version."""
path = h5ad.with_suffix(".zarr")
previous = ad.settings.zarr_write_format
ad.settings.zarr_write_format = version
try:
ad.read_h5ad(h5ad).write_zarr(path)
finally:
ad.settings.zarr_write_format = previous
return path


def _zarr_version(path: Path) -> int:
import zarr

return zarr.open_group(str(path), mode="r").metadata.zarr_format


@pytest.mark.parametrize(
"inputs,expected",
[
((2, 2), 2), # issue #36: v2 inputs came out v3
((3, 3), 3),
((2, 3), 3), # no agreement: the default
((None, 2), 2), # an h5ad input has no version and does not count
],
)
def test_concat_follows_the_inputs_zarr_version(tmp_path, inputs, expected):
paths = []
for i, version in enumerate(inputs):
h5ad = _make(tmp_path / f"in{i}.h5ad", [f"c{i}"], ["g1"], batch=str(i))
paths.append(h5ad if version is None else _as_zarr(h5ad, version))
out = tmp_path / "m.zarr"

result = runner.invoke(app, ["concat", *map(str, paths), "-o", str(out)])
assert result.exit_code == 0, _out(result)
assert _zarr_version(out) == expected
assert ad.read_zarr(out).shape == (2, 1)


def test_concat_zarr_format_overrides_the_inputs(tmp_path):
a = _as_zarr(_make(tmp_path / "a.h5ad", ["c1"], ["g1"], batch="A"), 2)
b = _as_zarr(_make(tmp_path / "b.h5ad", ["c2"], ["g1"], batch="B"), 2)
out = tmp_path / "m.zarr"

result = runner.invoke(
app, ["concat", str(a), str(b), "-o", str(out), "--zarr-format", "3"]
)
assert result.exit_code == 0, _out(result)
assert _zarr_version(out) == 3


def test_concat_warns_about_duplicate_obs_names(tmp_path):
a = _make(tmp_path / "a.h5ad", ["c1"], ["g1"], batch="A")
b = _make(tmp_path / "b.h5ad", ["c1"], ["g1"], batch="B")
Expand Down
Loading