diff --git a/src/main/java/dev/zarr/zarrjava/v2/Array.java b/src/main/java/dev/zarr/zarrjava/v2/Array.java index 5bbfbdb2..04fd6873 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/Array.java +++ b/src/main/java/dev/zarr/zarrjava/v2/Array.java @@ -13,6 +13,7 @@ import dev.zarr.zarrjava.utils.Utils; import dev.zarr.zarrjava.v2.codec.Codec; import dev.zarr.zarrjava.v2.codec.core.BytesCodec; +import dev.zarr.zarrjava.v2.codec.core.FortranOrderCodec; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -36,6 +37,12 @@ protected Array(StoreHandle storeHandle, ArrayMetadata arrayMetadata) throws IOE this.metadata = arrayMetadata; this.codecPipeline = new CodecPipeline(Utils.concatArrays( new Codec[]{}, + // "order": "F" means the chunk is serialized column-major. Reversing all axes before + // the BytesCodec turns its row-major serialization into a column-major one. For rank + // 0 and 1 both orders are identical, so no codec is needed there. + metadata.order == Order.F && metadata.ndim() > 1 + ? new Codec[]{new FortranOrderCodec()} + : new Codec[]{}, metadata.filters == null ? new Codec[]{} : metadata.filters, new Codec[]{new BytesCodec(arrayMetadata.endianness.toEndian())}, metadata.compressor == null ? new Codec[]{} : new Codec[]{metadata.compressor} diff --git a/src/main/java/dev/zarr/zarrjava/v2/codec/core/FortranOrderCodec.java b/src/main/java/dev/zarr/zarrjava/v2/codec/core/FortranOrderCodec.java new file mode 100644 index 00000000..a0074c93 --- /dev/null +++ b/src/main/java/dev/zarr/zarrjava/v2/codec/core/FortranOrderCodec.java @@ -0,0 +1,102 @@ +package dev.zarr.zarrjava.v2.codec.core; + +import dev.zarr.zarrjava.ZarrException; +import dev.zarr.zarrjava.core.ArrayMetadata; +import dev.zarr.zarrjava.core.codec.ArrayArrayCodec; +import dev.zarr.zarrjava.v2.codec.Codec; +import ucar.ma2.Array; + +/** + * Realizes the Fortran (column-major) chunk layout that Zarr v2 selects with {@code "order": "F"}. + *

+ * Zarr v2 stores the elements of a chunk either in row-major ({@code "C"}) or column-major + * ({@code "F"}) order. Zarr v3 dropped the {@code order} key and expresses the same thing with the + * {@code transpose} codec, so the v2 {@code order} key is modelled here the same way: as an + * array → array codec that reverses all axes. The row-major serialization of an array with + * reversed axes is exactly the column-major serialization of the original array. + *

+ * This codec is never part of the {@code .zarray} document. It is inserted into the codec pipeline + * by {@link dev.zarr.zarrjava.v2.Array} whenever the metadata says {@code "order": "F"}, and the + * {@code order} key alone remains the on-disk representation. + */ +public class FortranOrderCodec extends ArrayArrayCodec implements Codec { + + /** + * Reversing all axes is its own inverse, so encoding and decoding differ only in what they do + * with the resulting view. + */ + private Array reverseAxes(Array chunkArray) throws ZarrException { + int ndim = resolveArrayMetadata().ndim(); + if (chunkArray.getRank() != ndim) { + throw new ZarrException( + "Chunk array has rank " + chunkArray.getRank() + ", but the array has " + ndim + + " dimensions."); + } + int[] reversedAxes = new int[ndim]; + for (int i = 0; i < ndim; i++) { + reversedAxes[i] = ndim - 1 - i; + } + return chunkArray.permute(reversedAxes); + } + + /** + * Turns the logical chunk into the axis-reversed view that the following + * {@link dev.zarr.zarrjava.core.codec.core.BytesCodec} then serializes row-major, which yields + * the column-major serialization of the logical chunk. The view is handed on as-is: the + * {@code BytesCodec} flattens in logical index order, so no copy is needed here. + */ + @Override + public Array encode(Array chunkArray) throws ZarrException { + return reverseAxes(chunkArray); + } + + /** + * Turns the axis-reversed array that came out of the + * {@link dev.zarr.zarrjava.core.codec.core.BytesCodec} back into the logical chunk. + *

+ * The result is materialized with {@link Array#copy()} rather than returned as a permuted view. + * A view's logical order is correct for iterators and index-based access, but its backing store + * is still in column-major order, so linear accessors such as {@code getInt(int)} would return + * elements in the wrong order. Chunks decoded here can be handed straight to callers by the + * single-full-chunk fast path in {@link dev.zarr.zarrjava.core.Array#read(long[], long[])}, so + * they have to behave like any other decoded chunk. + */ + @Override + public Array decode(Array chunkArray) throws ZarrException { + return reverseAxes(chunkArray).copy(); + } + + /** + * Reports the axis-reversed shapes to the following codec, so that the + * {@link dev.zarr.zarrjava.core.codec.core.BytesCodec} reads and writes the chunk bytes with the + * reversed chunk shape. + */ + @Override + public ArrayMetadata.CoreArrayMetadata resolveArrayMetadata() throws ZarrException { + super.resolveArrayMetadata(); + int ndim = arrayMetadata.ndim(); + long[] reversedShape = new long[ndim]; + int[] reversedChunkShape = new int[ndim]; + for (int i = 0; i < ndim; i++) { + reversedShape[i] = arrayMetadata.shape[ndim - 1 - i]; + reversedChunkShape[i] = arrayMetadata.chunkShape[ndim - 1 - i]; + } + return new ArrayMetadata.CoreArrayMetadata( + reversedShape, + reversedChunkShape, + arrayMetadata.dataType, + arrayMetadata.parsedFillValue + ); + } + + @Override + public long computeEncodedSize(long inputByteLength, + ArrayMetadata.CoreArrayMetadata arrayMetadata) { + return inputByteLength; + } + + @Override + public Codec evolveFromCoreArrayMetadata(ArrayMetadata.CoreArrayMetadata arrayMetadata) { + return this; + } +} diff --git a/src/test/java/dev/zarr/zarrjava/ZarrPythonTests.java b/src/test/java/dev/zarr/zarrjava/ZarrPythonTests.java index 5c22b5fc..655e1636 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrPythonTests.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrPythonTests.java @@ -355,4 +355,82 @@ public void testGroupReadWriteV3() throws Exception { Assertions.assertArrayEquals(new int[]{16, 16, 16}, result.getShape()); assertIsTestdata(result, dataType); } + + /** + * The Zarr v2 {@code order} cases, as (shape, chunks, order). + */ + static Stream orderProviderV2() { + return Stream.of( + new Object[]{"3,4", "3,4", "C"}, + new Object[]{"3,4", "3,4", "F"}, + new Object[]{"2,3,4", "2,3,4", "F"}, + // Chunks divide neither dimension evenly, so the trailing chunks are partial. + new Object[]{"5,7", "2,3", "F"}, + new Object[]{"5,7,3", "2,3,2", "F"}, + // For rank 1 both orders describe the same layout. + new Object[]{"10", "4", "F"} + ); + } + + /** + * Live counterpart to {@link ZarrV2OrderTest}, which covers the same ground offline against + * committed fixtures. This runs against the installed zarr-python so that a change in the + * reference implementation is noticed rather than silently diverging from the fixtures. + */ + @ParameterizedTest + @MethodSource("orderProviderV2") + public void testReadOrderV2(String shape, String chunks, String order) throws Exception { + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT) + .resolve("testReadOrderV2", shape + "_" + chunks + "_" + order); + run_python_script("zarr_python_order_v2.py", "write", storeHandle.toPath().toString(), + shape, chunks, order); + + dev.zarr.zarrjava.v2.Array array = dev.zarr.zarrjava.v2.Array.open(storeHandle); + ucar.ma2.Array result = array.read(); + + Assertions.assertEquals(dev.zarr.zarrjava.v2.Order.valueOf(order), array.metadata().order); + for (int i = 0; i < result.getSize(); i++) { + Assertions.assertEquals(i, result.getInt(i), + "element " + i + " of a " + order + "-order array"); + } + } + + /** + * The write direction, verified by zarr-python. zarr-java used to write {@code "order": "F"} + * into {@code .zarray} while laying the chunk out row-major, which its own reader then read back + * correctly — so only an external reader can catch it. + */ + @ParameterizedTest + @MethodSource("orderProviderV2") + public void testWriteOrderV2(String shape, String chunks, String order) throws Exception { + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT) + .resolve("testWriteOrderV2", shape + "_" + chunks + "_" + order); + + long[] shapeArray = Arrays.stream(shape.split(",")).mapToLong(Long::parseLong).toArray(); + int[] chunksArray = Arrays.stream(chunks.split(",")).mapToInt(Integer::parseInt).toArray(); + + dev.zarr.zarrjava.v2.Array array = dev.zarr.zarrjava.v2.Array.create( + storeHandle, + dev.zarr.zarrjava.v2.Array.metadataBuilder() + .withShape(shapeArray) + .withChunks(chunksArray) + .withDataType(dev.zarr.zarrjava.v2.DataType.INT32) + .withOrder(dev.zarr.zarrjava.v2.Order.valueOf(order)) + .withFillValue(0) + .build() + ); + + int[] intShape = new int[shapeArray.length]; + for (int i = 0; i < shapeArray.length; i++) { + intShape[i] = (int) shapeArray[i]; + } + ucar.ma2.Array data = ucar.ma2.Array.factory(ucar.ma2.DataType.INT, intShape); + for (int i = 0; i < data.getSize(); i++) { + data.setInt(i, i); + } + array.write(new long[shapeArray.length], data); + + run_python_script("zarr_python_order_v2.py", "read", storeHandle.toPath().toString(), + shape, chunks, order); + } } diff --git a/src/test/java/dev/zarr/zarrjava/ZarrV2OrderTest.java b/src/test/java/dev/zarr/zarrjava/ZarrV2OrderTest.java new file mode 100644 index 00000000..d2a5d1e6 --- /dev/null +++ b/src/test/java/dev/zarr/zarrjava/ZarrV2OrderTest.java @@ -0,0 +1,352 @@ +package dev.zarr.zarrjava; + +import dev.zarr.zarrjava.store.FilesystemStore; +import dev.zarr.zarrjava.v2.Array; +import dev.zarr.zarrjava.v2.ArrayMetadata; +import dev.zarr.zarrjava.v2.DataType; +import dev.zarr.zarrjava.v2.Order; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.EnumSource; +import org.junit.jupiter.params.provider.MethodSource; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** + * Tests for the Zarr v2 {@code order} key, which selects whether a chunk is serialized row-major + * ({@code "C"}) or column-major ({@code "F"}). + *

+ * These tests are deliberately built around the committed golden fixtures in + * {@code testdata/golden/}, which were produced by zarr-python. A Java-only round-trip cannot detect + * a wrong {@code order} implementation: the writer and the reader shared the same bug, so they + * agreed with each other while disagreeing with every other Zarr implementation. Comparing against + * bytes written by the reference implementation is what makes these tests meaningful, and doing it + * against committed fixtures keeps them fast and Python-free. + * + * @see dev.zarr.zarrjava.v2.codec.core.FortranOrderCodec + */ +public class ZarrV2OrderTest extends ZarrTest { + + static final Path GOLDEN = TESTDATA.resolve("golden"); + + /** + * The golden fixtures, as (fixture name, shape, chunks, order). Every fixture holds + * {@code arange(prod(shape))} as {@code int32} with no compressor, so its chunk files are the raw + * element bytes. + */ + static Stream goldenFixtures() { + return Stream.of( + Arguments.of("v2_order_c", new long[]{3, 4}, new int[]{3, 4}, Order.C), + Arguments.of("v2_order_f", new long[]{3, 4}, new int[]{3, 4}, Order.F), + Arguments.of("v2_order_f_3d", new long[]{2, 3, 4}, new int[]{2, 3, 4}, Order.F), + Arguments.of("v2_order_f_multichunk", new long[]{5, 7}, new int[]{2, 3}, Order.F) + ); + } + + /** + * Every golden fixture must read back as {@code arange}, whatever its {@code order}. Before the + * {@code order} key was honoured, the F-order 3x4 fixture read as + * {@code [0, 4, 8, 1, 5, 9, 2, 6, 10, 3, 7, 11]} instead of {@code [0 .. 11]}. + */ + @ParameterizedTest + @MethodSource("goldenFixtures") + public void testReadGoldenFixture(String name, long[] shape, int[] chunks, Order order) + throws IOException, ZarrException { + Array array = Array.open(GOLDEN.resolve(name)); + + ArrayMetadata metadata = array.metadata(); + Assertions.assertEquals(order, metadata.order, "order was not parsed from .zarray"); + Assertions.assertArrayEquals(shape, metadata.shape); + Assertions.assertArrayEquals(chunks, metadata.chunks); + + Assertions.assertArrayEquals(expectedArange(shape), readAsLongs(array.read())); + } + + /** + * The write direction, checked offline: writing {@code arange} through zarr-java must reproduce + * the reference implementation's chunk bytes exactly. This is the direction a Java-only + * round-trip test cannot see. Before the fix, zarr-java wrote {@code "order": "F"} into + * {@code .zarray} while laying the elements out row-major, so every other implementation + * misread the result. + */ + @ParameterizedTest + @MethodSource("goldenFixtures") + public void testWriteGoldenFixtureBytes(String name, long[] shape, int[] chunks, Order order) + throws IOException, ZarrException { + Path expectedPath = GOLDEN.resolve(name); + Path actualPath = TESTOUTPUT.resolve("v2_order_write").resolve(name); + + Array array = Array.create( + new FilesystemStore(actualPath).resolve(), + Array.metadataBuilder() + .withShape(shape) + .withChunks(chunks) + .withDataType(DataType.INT32) + .withOrder(order) + .withFillValue(0) + .build(), + true + ); + array.write(new long[shape.length], arange(shape)); + + List expectedChunks = chunkFileNames(expectedPath); + List actualChunks = chunkFileNames(actualPath); + Assertions.assertEquals(expectedChunks, actualChunks, + "zarr-java did not write the same set of chunk files as zarr-python"); + + for (String chunk : expectedChunks) { + Assertions.assertArrayEquals( + Files.readAllBytes(expectedPath.resolve(chunk)), + Files.readAllBytes(actualPath.resolve(chunk)), + "chunk '" + chunk + "' of " + name + " does not match the bytes written by zarr-python" + ); + } + } + + /** + * A read that happens to cover exactly one full chunk takes the fast path in + * {@link dev.zarr.zarrjava.core.Array#read(long[], long[])}, which returns the decoded chunk + * straight to the caller instead of copying it into a fresh output array. The decoded chunk + * therefore has to be physically contiguous in row-major order: a permuted view would answer + * iterators and {@code get(int[])} correctly while linear accessors such as {@code getInt(int)} + * silently walked the column-major backing store. + */ + @Test + public void testSingleFullChunkReadIsPhysicallyContiguous() throws IOException, ZarrException { + // The 3x4 fixture has a single 3x4 chunk, so read() returns it through the fast path. + ucar.ma2.Array result = Array.open(GOLDEN.resolve("v2_order_f")).read(); + + for (int i = 0; i < result.getSize(); i++) { + Assertions.assertEquals(i, result.getInt(i), + "linear access at " + i + " walked the column-major backing store"); + } + Assertions.assertEquals(3, result.getInt(result.getIndex().set(0, 3)), + "index-based access disagrees with linear access"); + } + + /** + * C and F order must hold the same logical values but must not produce the same bytes on disk. + * The second half of that is what actually pins the behaviour down: if {@code order} were + * ignored again, both stores would come out byte-identical and this would fail. + */ + @Test + public void testCAndFOrderAgreeLogicallyButDifferOnDisk() throws IOException, ZarrException { + long[] shape = {3, 4}; + Path cPath = TESTOUTPUT.resolve("v2_order_compare_c"); + Path fPath = TESTOUTPUT.resolve("v2_order_compare_f"); + + Array cArray = writeArange(cPath, shape, new int[]{3, 4}, Order.C); + Array fArray = writeArange(fPath, shape, new int[]{3, 4}, Order.F); + + Assertions.assertArrayEquals(readAsLongs(cArray.read()), readAsLongs(fArray.read()), + "C and F order must expose the same logical values"); + + byte[] cBytes = Files.readAllBytes(cPath.resolve("0.0")); + byte[] fBytes = Files.readAllBytes(fPath.resolve("0.0")); + Assertions.assertFalse(java.util.Arrays.equals(cBytes, fBytes), + "C and F order produced identical chunk bytes, so 'order' was ignored"); + } + + /** + * For rank 0 and rank 1 the two orders describe the same layout, so they must produce identical + * bytes. This guards the {@code ndim() > 1} short-circuit in the v2 codec pipeline. + */ + @Test + public void testOrderIsIrrelevantForOneDimension() throws IOException, ZarrException { + long[] shape = {10}; + Path cPath = TESTOUTPUT.resolve("v2_order_1d_c"); + Path fPath = TESTOUTPUT.resolve("v2_order_1d_f"); + + Array cArray = writeArange(cPath, shape, new int[]{4}, Order.C); + Array fArray = writeArange(fPath, shape, new int[]{4}, Order.F); + + Assertions.assertArrayEquals(expectedArange(shape), readAsLongs(cArray.read())); + Assertions.assertArrayEquals(expectedArange(shape), readAsLongs(fArray.read())); + + for (String chunk : chunkFileNames(cPath)) { + Assertions.assertArrayEquals( + Files.readAllBytes(cPath.resolve(chunk)), + Files.readAllBytes(fPath.resolve(chunk)), + "1D arrays must serialize identically in C and F order" + ); + } + } + + /** + * F order has to keep working once a compressor sits behind it in the pipeline, and for a chunk + * grid that does not divide the shape evenly. The round-trip assertion alone cannot detect a + * broken {@code order}; it is the pipeline-integration half of this test, checking that the + * axis reversal is ordered correctly relative to the compressor and survives partial-chunk + * copying. {@link #testCompressedCAndFOrderDifferOnDisk()} supplies the on-disk check. + */ + @ParameterizedTest + @EnumSource(Order.class) + public void testOrderRoundTripsWithCompressorAndPartialChunks(Order order) + throws IOException, ZarrException { + long[] shape = {5, 7, 3}; + Path path = TESTOUTPUT.resolve("v2_order_zlib_" + order); + + Array array = writeCompressedArange(path, shape, order); + + Assertions.assertArrayEquals(expectedArange(shape), readAsLongs(array.read())); + + // A cutout that starts and ends inside a chunk, to check the axis reversal survives + // partial-chunk copying rather than only whole-chunk reads. + ucar.ma2.Array cutout = array.read(new long[]{1, 2, 1}, new long[]{3, 4, 2}); + long[] expected = new long[3 * 4 * 2]; + int i = 0; + for (long z = 1; z < 4; z++) { + for (long y = 2; y < 6; y++) { + for (long x = 1; x < 3; x++) { + expected[i++] = z * 7 * 3 + y * 3 + x; + } + } + } + Assertions.assertArrayEquals(expected, readAsLongs(cutout)); + } + + /** + * The on-disk half of {@link #testOrderRoundTripsWithCompressorAndPartialChunks(Order)}: with a + * compressor in the pipeline the two orders must still produce different chunk bytes. + */ + @Test + public void testCompressedCAndFOrderDifferOnDisk() throws IOException, ZarrException { + long[] shape = {5, 7, 3}; + Path cPath = TESTOUTPUT.resolve("v2_order_zlib_compare_c"); + Path fPath = TESTOUTPUT.resolve("v2_order_zlib_compare_f"); + + writeCompressedArange(cPath, shape, Order.C); + writeCompressedArange(fPath, shape, Order.F); + + Assertions.assertFalse( + java.util.Arrays.equals( + Files.readAllBytes(cPath.resolve("0.0.0")), + Files.readAllBytes(fPath.resolve("0.0.0"))), + "C and F order produced identical compressed chunk bytes, so 'order' was ignored" + ); + } + + /** + * {@code order} must survive the metadata rewrites done by {@code resize} and attribute updates, + * otherwise an F-order array silently turns into a C-order one whose chunks are now misread. + */ + @Test + public void testOrderSurvivesMetadataRewrites() throws IOException, ZarrException { + Path path = TESTOUTPUT.resolve("v2_order_rewrite"); + Array array = writeArange(path, new long[]{3, 4}, new int[]{3, 4}, Order.F); + + Array resized = array.resize(new long[]{6, 4}); + Assertions.assertEquals(Order.F, resized.metadata().order, "resize dropped the order"); + + Array withAttributes = resized.updateAttributes(attributes -> { + attributes.put("key", "value"); + return attributes; + }); + Assertions.assertEquals(Order.F, withAttributes.metadata().order, + "updateAttributes dropped the order"); + + // The original data must still be readable through the rewritten metadata. + ucar.ma2.Array original = withAttributes.read(new long[]{0, 0}, new long[]{3, 4}); + Assertions.assertArrayEquals(expectedArange(new long[]{3, 4}), readAsLongs(original)); + } + + // --- helpers ------------------------------------------------------------------------------- + + private static Array writeArange(Path path, long[] shape, int[] chunks, Order order) + throws IOException, ZarrException { + Array array = Array.create( + new FilesystemStore(path).resolve(), + Array.metadataBuilder() + .withShape(shape) + .withChunks(chunks) + .withDataType(DataType.INT32) + .withOrder(order) + .withFillValue(0) + .build(), + true + ); + array.write(new long[shape.length], arange(shape)); + return array; + } + + private static Array writeCompressedArange(Path path, long[] shape, Order order) + throws IOException, ZarrException { + Array array = Array.create( + new FilesystemStore(path).resolve(), + Array.metadataBuilder() + .withShape(shape) + .withChunks(2, 3, 2) + .withDataType(DataType.FLOAT64) + .withOrder(order) + .withFillValue(0) + .withZlibCompressor(5) + .build(), + true + ); + array.write(new long[shape.length], arange(shape, DataType.FLOAT64)); + return array; + } + + private static ucar.ma2.Array arange(long[] shape) { + return arange(shape, DataType.INT32); + } + + private static ucar.ma2.Array arange(long[] shape, DataType dataType) { + int[] intShape = new int[shape.length]; + for (int i = 0; i < shape.length; i++) { + intShape[i] = (int) shape[i]; + } + ucar.ma2.Array array = ucar.ma2.Array.factory(dataType.getMA2DataType(), intShape); + for (int i = 0; i < array.getSize(); i++) { + array.setLong(i, i); + } + return array; + } + + private static long[] expectedArange(long[] shape) { + int size = 1; + for (long dim : shape) { + size *= (int) dim; + } + long[] expected = new long[size]; + for (int i = 0; i < size; i++) { + expected[i] = i; + } + return expected; + } + + /** + * Flattens in row-major index order, independent of the element type, so that a single + * assertion works for every dtype under test. + */ + private static long[] readAsLongs(ucar.ma2.Array array) { + List values = new ArrayList<>(); + ucar.ma2.IndexIterator iterator = array.getIndexIterator(); + while (iterator.hasNext()) { + values.add(iterator.getLongNext()); + } + long[] result = new long[values.size()]; + for (int i = 0; i < result.length; i++) { + result[i] = values.get(i); + } + return result; + } + + private static List chunkFileNames(Path storePath) throws IOException { + try (Stream files = Files.list(storePath)) { + return files.map(path -> path.getFileName().toString()) + .filter(name -> !name.startsWith(".")) + .sorted(Comparator.naturalOrder()) + .collect(Collectors.toList()); + } + } +} diff --git a/src/test/python-scripts/generate_v2_order_golden.py b/src/test/python-scripts/generate_v2_order_golden.py new file mode 100644 index 00000000..360b3cf7 --- /dev/null +++ b/src/test/python-scripts/generate_v2_order_golden.py @@ -0,0 +1,66 @@ +"""Regenerates the committed Zarr v2 ``order`` golden fixtures in testdata/golden/. + +zarr-java's writer and reader shared the same ``order: "F"`` bug, so a Java-only +round-trip test agreed with itself and could never detect it. These fixtures are +written by zarr-python, the reference implementation, and are committed so that +both the read direction (open a known-correct store) and the write direction +(compare the bytes zarr-java produces against known-correct bytes) are covered +offline, with no Python needed at test time. + +Run from the repository root after a zarr-python upgrade: + + uv run src/test/python-scripts/generate_v2_order_golden.py + +The fixtures are deliberately tiny (a few hundred bytes in total) and use no +compressor, so the chunk files are the raw element bytes. +""" + +import pathlib +import shutil + +import numpy as np +import zarr + +GOLDEN = pathlib.Path("testdata/golden") + +# name -> (shape, chunks, order) +# int32 throughout: 4 bytes per element keeps the chunk files readable in a hex dump. +CASES = { + # Minimal C/F pair. Same logical values, different chunk bytes. + "v2_order_c": ((3, 4), (3, 4), "C"), + "v2_order_f": ((3, 4), (3, 4), "F"), + # Rank > 2, so that reversing all axes is exercised rather than a plain 2D swap. + "v2_order_f_3d": ((2, 3, 4), (2, 3, 4), "F"), + # Chunk shape divides neither dimension evenly, so the trailing chunks are + # partially outside the array and get fill values. + "v2_order_f_multichunk": ((5, 7), (2, 3), "F"), +} + + +def main() -> None: + for name, (shape, chunks, order) in CASES.items(): + path = GOLDEN / name + shutil.rmtree(path, ignore_errors=True) + values = np.arange(int(np.prod(shape)), dtype="int32").reshape(shape) + array = zarr.create_array( + store=str(path), + shape=shape, + chunks=chunks, + dtype="int32", + zarr_format=2, + order=order, + compressors=None, + fill_value=0, + ) + array[...] = values + assert np.array_equal(array[...], values), name + # zarr-python always writes .zattrs; an empty one carries no information. + zattrs = path / ".zattrs" + if zattrs.is_file() and zattrs.read_text().strip() in ("{}", ""): + zattrs.unlink() + chunk_files = sorted(p.name for p in path.iterdir() if p.name != ".zarray") + print(f"{name}: shape={shape} chunks={chunks} order={order} -> {chunk_files}") + + +if __name__ == "__main__": + main() diff --git a/src/test/python-scripts/zarr_python_order_v2.py b/src/test/python-scripts/zarr_python_order_v2.py new file mode 100644 index 00000000..00543d3d --- /dev/null +++ b/src/test/python-scripts/zarr_python_order_v2.py @@ -0,0 +1,64 @@ +"""Live zarr-python oracle for the Zarr v2 ``order`` key. + +Usage: + zarr_python_order_v2.py write + zarr_python_order_v2.py read + +``shape`` and ``chunks`` are comma-separated, e.g. ``5,7,3``. ``order`` is ``C`` or ``F``. + +``write`` creates a store holding ``arange(prod(shape))`` for zarr-java to read. +``read`` opens a store written by zarr-java and asserts it holds exactly that, so a +wrong layout on the Java side fails here with a non-zero exit code. + +Both directions are needed: zarr-java's reader and writer shared the same ``order`` +bug, so they agreed with each other and a Java-only round-trip stayed green. +""" + +import sys + +import numpy as np +import zarr + + +def expected(shape): + return np.arange(int(np.prod(shape)), dtype="int32").reshape(shape) + + +def main(): + mode, path, shape_arg, chunks_arg, order = sys.argv[1:6] + shape = tuple(int(s) for s in shape_arg.split(",")) + chunks = tuple(int(c) for c in chunks_arg.split(",")) + values = expected(shape) + + if mode == "write": + array = zarr.create_array( + store=path, + shape=shape, + chunks=chunks, + dtype="int32", + zarr_format=2, + order=order, + compressors=None, + fill_value=0, + ) + array[...] = values + print(f"wrote {path} shape={shape} chunks={chunks} order={order}") + elif mode == "read": + array = zarr.open_array(path, mode="r", zarr_format=2) + assert array.metadata.order == order, ( + f"expected order {order!r} in .zarray, found {array.metadata.order!r}" + ) + assert array.shape == shape, f"expected shape {shape}, found {array.shape}" + actual = array[...] + if not np.array_equal(actual, values): + raise AssertionError( + f"zarr-python read {path} (order={order}) incorrectly.\n" + f"expected:\n{values}\nactual:\n{actual}" + ) + print(f"verified {path} shape={shape} chunks={chunks} order={order}") + else: + raise SystemExit(f"unknown mode {mode!r}") + + +if __name__ == "__main__": + main() diff --git a/testdata/golden/v2_order_c/.zarray b/testdata/golden/v2_order_c/.zarray new file mode 100644 index 00000000..10493342 --- /dev/null +++ b/testdata/golden/v2_order_c/.zarray @@ -0,0 +1,17 @@ +{ + "shape": [ + 3, + 4 + ], + "chunks": [ + 3, + 4 + ], + "dtype": "