From 801372bc7fe2db030a425a9bf1577e6ae20308b2 Mon Sep 17 00:00:00 2001 From: Konstantin Date: Thu, 27 Aug 2026 15:17:24 +0200 Subject: [PATCH] Add float16 data type (v3 and v2) float16 is part of the Zarr v3 core spec, not an optional extension -- the zarr-extensions entry for it is only a pointer back to the core spec's data types section. Before this change, opening any float16 array failed at metadata parse with "Cannot deserialize value of type DataType from String float16". No new codec is needed: the existing bytes codec handles any fixed-itemsize data type. The work is the in-memory representation, because neither Java nor ucar.ma2 has a 16-bit float. float16 is therefore held as ucar.ma2.FLOAT and converted at the codec boundary, which makes it the first data type whose encoded width (2 bytes) differs from its in-memory width (4 bytes). - utils/Float16: IEEE 754 binary16 <-> float conversions. Float.float16ToFloat is Java 20+ and this project targets Java 8, so the bit manipulation is written out, covering subnormals, signed zeros, infinities, NaN payloads and round-half-to-even ties. - core/DataType.isHalfPrecisionFloat(): default false, so the bytes codec can detect the width mismatch without importing the v2/v3 enums. - core/codec/core/BytesCodec: widen on decode, narrow on encode. The float16 branch precedes the ArrayFloat branch, since float16 is held in an ArrayFloat and must not be written as float32. - core/ArrayMetadata.parseFillValue: accepts numbers, NaN, +/-Infinity, and the hex/binary forms. The 2 bytes of a hex fill value are a binary16 bit pattern, not a truncated float32, so they are widened rather than read as a float. - v3/DataType.FLOAT16 ("float16"), v2/DataType.FLOAT16 ("f2"). v2 needed no byte-count change: getByteCount() parses the dtype string, giving 2 for f2. Blosc typesize follows getByteCount() and so becomes 2, matching the encoded width. Narrowing is lossy and documented as such: values at or above 65520.0 become infinity, values below 2^-25 become signed zero, and everything else is rounded to 11 significand bits. ZarrTest.testdata/assertIsTestdata now pre-quantize float test values, because 1024 of the 4096 values they generate are not exactly representable in binary16 (2049 stores as 2048, 4095 as 4096); the quantized values match np.arange(4096, dtype='float16') element for element. Verified against numpy 2.4.6 and zarr-python 3.1.6: - 39 conversion cases asserted against numpy bit patterns, plus an exhaustive sweep proving all 65536 binary16 patterns round-trip (NaN stays NaN). - Round-trips in both endiannesses, through blosc/zstd/gzip/sharding/crc32c, and for v2 and v3. - Cross-implementation in both directions: zarr-python writes / zarr-java reads and zarr-java writes / zarr-python reads, for v2 and v3 and both byte orders. The chunk bytes are byte-identical to zarr-python's in all four configurations, including the partial trailing chunk. - Confirmed the interop test has teeth: injecting a symmetric byte-swap into both encode and decode still passes the zarr-java round-trip, while failing all 8 interop cases. Follow-ups, deliberately not attempted here: - complex64/complex128: ucar.ma2 has no complex type, so the in-memory representation (trailing dimension of 2 vs. a custom ArrayObject) is a design decision for the maintainers rather than something to settle in this PR. - r* (r8, r16, ...) and fixed_length_utf32: parameterized data types. r* needs its name parsed as a pattern and fixed_length_utf32 carries a configuration object, so data_type stops being a plain JSON string and the @JsonValue enum cannot model either. That is a separate refactor. - string and variable_length_bytes: these break the fixed-itemsize invariant of getByteCount() and belong with the vlen work. - ZarrPythonTests still drives its data type matrix from the hand-maintained dataTypeProviderV3/V2 lists, which FLOAT16 has been added to. If those are switched to DataType.values(), note that zarr_python_write.py builds test data with np.arange, whose float16 values above 2048 are quantized -- the Java side already accounts for this via ZarrTest.quantize. --- .../dev/zarr/zarrjava/core/ArrayMetadata.java | 14 +- .../java/dev/zarr/zarrjava/core/DataType.java | 15 + .../zarrjava/core/codec/core/BytesCodec.java | 28 + .../java/dev/zarr/zarrjava/utils/Float16.java | 155 ++++++ .../java/dev/zarr/zarrjava/v2/DataType.java | 14 + .../java/dev/zarr/zarrjava/v3/DataType.java | 12 + .../java/dev/zarr/zarrjava/Float16Test.java | 500 ++++++++++++++++++ src/test/java/dev/zarr/zarrjava/ZarrTest.java | 22 +- src/test/python-scripts/float16_interop.py | 106 ++++ 9 files changed, 860 insertions(+), 6 deletions(-) create mode 100644 src/main/java/dev/zarr/zarrjava/utils/Float16.java create mode 100644 src/test/java/dev/zarr/zarrjava/Float16Test.java create mode 100644 src/test/python-scripts/float16_interop.py diff --git a/src/main/java/dev/zarr/zarrjava/core/ArrayMetadata.java b/src/main/java/dev/zarr/zarrjava/core/ArrayMetadata.java index a2fb48f3..3c7db297 100644 --- a/src/main/java/dev/zarr/zarrjava/core/ArrayMetadata.java +++ b/src/main/java/dev/zarr/zarrjava/core/ArrayMetadata.java @@ -5,6 +5,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import dev.zarr.zarrjava.ZarrException; import dev.zarr.zarrjava.core.chunkkeyencoding.ChunkKeyEncoding; +import dev.zarr.zarrjava.utils.Float16; import dev.zarr.zarrjava.utils.MultiArrayUtils; import dev.zarr.zarrjava.utils.Utils; import ucar.ma2.Array; @@ -41,6 +42,8 @@ public static Object parseFillValue(Object fillValue, @Nonnull DataType dataType boolean dataTypeIsLong = dataType == dev.zarr.zarrjava.v3.DataType.INT64 || dataType == dev.zarr.zarrjava.v2.DataType.INT64 || dataType == dev.zarr.zarrjava.v2.DataType.INT64_BE || dataType == dev.zarr.zarrjava.v3.DataType.UINT64 || dataType == dev.zarr.zarrjava.v2.DataType.UINT64 || dataType == dev.zarr.zarrjava.v2.DataType.UINT64_BE; boolean dataTypeIsFloat = dataType == dev.zarr.zarrjava.v3.DataType.FLOAT32 || dataType == dev.zarr.zarrjava.v2.DataType.FLOAT32 || dataType == dev.zarr.zarrjava.v2.DataType.FLOAT32_BE; boolean dataTypeIsDouble = dataType == dev.zarr.zarrjava.v3.DataType.FLOAT64 || dataType == dev.zarr.zarrjava.v2.DataType.FLOAT64 || dataType == dev.zarr.zarrjava.v2.DataType.FLOAT64_BE; + // float16 fill values are parsed to float, matching the in-memory representation. + boolean dataTypeIsHalfFloat = dataType.isHalfPrecisionFloat(); if (fillValue instanceof Boolean) { Boolean fillValueBool = (Boolean) fillValue; @@ -60,7 +63,7 @@ public static Object parseFillValue(Object fillValue, @Nonnull DataType dataType return fillValueNumber.intValue(); } else if (dataTypeIsLong) { return fillValueNumber.longValue(); - } else if (dataTypeIsFloat) { + } else if (dataTypeIsFloat || dataTypeIsHalfFloat) { return fillValueNumber.floatValue(); } else if (dataTypeIsDouble) { return fillValueNumber.doubleValue(); @@ -69,7 +72,7 @@ public static Object parseFillValue(Object fillValue, @Nonnull DataType dataType } else if (fillValue instanceof String) { String fillValueString = (String) fillValue; if (fillValueString.equals("NaN")) { - if (dataTypeIsFloat) { + if (dataTypeIsFloat || dataTypeIsHalfFloat) { return Float.NaN; } else if (dataTypeIsDouble) { return Double.NaN; @@ -77,7 +80,7 @@ public static Object parseFillValue(Object fillValue, @Nonnull DataType dataType throw new ZarrException( "Invalid fill value '" + fillValueString + "' for data type '" + dataType + "'."); } else if (fillValueString.equals("+Infinity")) { - if (dataTypeIsFloat) { + if (dataTypeIsFloat || dataTypeIsHalfFloat) { return Float.POSITIVE_INFINITY; } else if (dataTypeIsDouble) { return Double.POSITIVE_INFINITY; @@ -85,7 +88,7 @@ public static Object parseFillValue(Object fillValue, @Nonnull DataType dataType throw new ZarrException( "Invalid fill value '" + fillValueString + "' for data type '" + dataType + "'."); } else if (fillValueString.equals("-Infinity")) { - if (dataTypeIsFloat) { + if (dataTypeIsFloat || dataTypeIsHalfFloat) { return Float.NEGATIVE_INFINITY; } else if (dataTypeIsDouble) { return Double.NEGATIVE_INFINITY; @@ -116,6 +119,9 @@ public static Object parseFillValue(Object fillValue, @Nonnull DataType dataType return buf.get() != 0; } else if (dataTypeIsByte) { return buf.get(); + } else if (dataTypeIsHalfFloat) { + // The 2 bytes are a binary16 bit pattern, not a truncated float32. + return Float16.halfBitsToFloat(buf.getShort()); } else if (dataTypeIsShort) { return buf.getShort(); } else if (dataTypeIsInt) { diff --git a/src/main/java/dev/zarr/zarrjava/core/DataType.java b/src/main/java/dev/zarr/zarrjava/core/DataType.java index d63de069..d7d2ed2e 100644 --- a/src/main/java/dev/zarr/zarrjava/core/DataType.java +++ b/src/main/java/dev/zarr/zarrjava/core/DataType.java @@ -4,4 +4,19 @@ public interface DataType { ucar.ma2.DataType getMA2DataType(); int getByteCount(); + + /** + * Whether elements are stored as IEEE 754 binary16 but held in memory as {@code float}. + * + *

For every other data type the encoded element width ({@link #getByteCount()}) matches the + * width of the in-memory {@link ucar.ma2.DataType}. Half precision is the exception: it encodes + * to 2 bytes but is held as a 4-byte {@code float}, because neither Java nor {@code ucar.ma2} + * has a 16-bit floating point type. The {@code bytes} codec uses this to convert at the + * encode/decode boundary via {@link dev.zarr.zarrjava.utils.Float16}. + * + * @return true for {@code float16}, false for every other data type + */ + default boolean isHalfPrecisionFloat() { + return false; + } } diff --git a/src/main/java/dev/zarr/zarrjava/core/codec/core/BytesCodec.java b/src/main/java/dev/zarr/zarrjava/core/codec/core/BytesCodec.java index 41382b5e..e0233eaa 100644 --- a/src/main/java/dev/zarr/zarrjava/core/codec/core/BytesCodec.java +++ b/src/main/java/dev/zarr/zarrjava/core/codec/core/BytesCodec.java @@ -3,6 +3,7 @@ import com.fasterxml.jackson.annotation.JsonValue; import dev.zarr.zarrjava.ZarrException; import dev.zarr.zarrjava.core.codec.ArrayBytesCodec; +import dev.zarr.zarrjava.utils.Float16; import ucar.ma2.*; import java.nio.ByteBuffer; @@ -31,6 +32,21 @@ public Array decode(ByteBuffer chunkBytes) throws ZarrException { Index index = Index.factory(shape); return Array.factory(DataType.BOOLEAN, index, bools); } + + // float16 encodes to 2 bytes per element but is held as a 4-byte float, so it cannot be + // read straight into an ma2 array; widen each element instead. + if (arrayMetadata.dataType.isHalfPrecisionFloat()) { + int base = chunkBytes.position(); + int size = chunkBytes.remaining() / Short.BYTES; + float[] floats = new float[size]; + for (int i = 0; i < size; i++) { + floats[i] = Float16.halfBitsToFloat(chunkBytes.getShort(base + i * Short.BYTES)); + } + + Index index = Index.factory(shape); + return Array.factory(DataType.FLOAT, index, floats); + } + return Array.factory(dtype, shape, chunkBytes); } @@ -51,6 +67,18 @@ public ByteBuffer encode(Array chunkArray) throws ZarrException { return bb; } + // Float16. Checked before ArrayFloat, because float16 is held in memory as an ArrayFloat + // and must be narrowed to 2 bytes per element rather than written as float32. + if (arrayMetadata.dataType.isHalfPrecisionFloat()) { + float[] data = (float[]) chunkArray.get1DJavaArray(DataType.FLOAT); + ByteBuffer bb = ByteBuffer.allocate(data.length * Short.BYTES).order(order); + for (float f : data) { + bb.putShort(Float16.floatToHalfBits(f)); + } + bb.flip(); + return bb; + } + // Float32 if (chunkArray instanceof ArrayFloat) { float[] data = (float[]) chunkArray.copyTo1DJavaArray(); diff --git a/src/main/java/dev/zarr/zarrjava/utils/Float16.java b/src/main/java/dev/zarr/zarrjava/utils/Float16.java new file mode 100644 index 00000000..9b094128 --- /dev/null +++ b/src/main/java/dev/zarr/zarrjava/utils/Float16.java @@ -0,0 +1,155 @@ +package dev.zarr.zarrjava.utils; + +/** + * Conversions between IEEE 754 binary16 ("half precision") bit patterns and Java {@code float}. + * + *

Zarr's {@code float16} data type stores 2 bytes per element, but there is no 16-bit floating + * point primitive in Java and no half precision type in {@code ucar.ma2.DataType}. Half precision + * arrays are therefore held in memory as {@code float} (binary32) and converted at the codec + * boundary. Widening binary16 to binary32 is always exact; narrowing is not, see + * {@link #floatToHalfBits(float)}. + * + *

{@code Float.float16ToFloat} and {@code Float.floatToFloat16} would do this for us, but they + * were added in Java 20 and this project targets Java 8, so the conversions are implemented here. + * The results match {@code numpy.float16} for all inputs, including subnormals, signed zeros, + * infinities, NaN payloads and round-half-to-even ties. + */ +public final class Float16 { + + /** + * Largest finite binary16 value, {@code 65504.0}. Finite {@code float} values greater than + * {@code MAX_VALUE + 16} round to infinity when narrowed. + */ + public static final float MAX_VALUE = 65504.0f; + + /** Smallest positive normal binary16 value, {@code 2^-14}. */ + public static final float MIN_NORMAL = 6.103515625E-5f; + + /** Smallest positive subnormal binary16 value, {@code 2^-24}. */ + public static final float MIN_VALUE = 5.9604645E-8f; + + private Float16() { + } + + /** + * Widens a binary16 bit pattern to the {@code float} of equal value. Exact for every input: + * every binary16 value is representable as a binary32. + * + *

Signed zeros and infinities are preserved. NaN payloads are shifted into the binary32 + * mantissa, so a NaN stays a NaN rather than becoming an infinity. + * + * @param halfBits the 16 bits of a binary16 value, in the low half of the short + * @return the equal-valued float + */ + public static float halfBitsToFloat(short halfBits) { + final int h = halfBits & 0xFFFF; + final int signBit = (h >>> 15) & 0x1; + final int exponent = (h >>> 10) & 0x1F; + int mantissa = h & 0x3FF; + + final int floatSign = signBit << 31; + + if (exponent == 0) { + if (mantissa == 0) { + // Signed zero. + return Float.intBitsToFloat(floatSign); + } + // Subnormal binary16, which is normal as a binary32: shift the mantissa up until the + // implicit leading bit is set, decrementing the exponent for each shift. + int normalizedExponent = 1; + while ((mantissa & 0x400) == 0) { + mantissa <<= 1; + normalizedExponent--; + } + mantissa &= 0x3FF; // Drop the leading bit; it is implicit in binary32 too. + return Float.intBitsToFloat( + floatSign | ((normalizedExponent - 15 + 127) << 23) | (mantissa << 13)); + } + + if (exponent == 0x1F) { + // Infinity (mantissa 0) or NaN (mantissa non-zero); shifting keeps the payload non-zero. + return Float.intBitsToFloat(floatSign | 0x7F800000 | (mantissa << 13)); + } + + // Normal binary16. + return Float.intBitsToFloat(floatSign | ((exponent - 15 + 127) << 23) | (mantissa << 13)); + } + + /** + * Narrows a {@code float} to the nearest binary16 bit pattern, rounding to nearest with ties to + * even -- the same rounding {@code numpy.float16} and IEEE 754 use. + * + *

This conversion is lossy. binary16 carries 11 bits of significand against binary32's 24, + * and its finite range stops at {@value #MAX_VALUE}: + *

+ * Callers writing data they cannot afford to round should not be using {@code float16}. + * + * @param value the float to narrow + * @return the 16 bits of the nearest binary16 value, in the low half of the short + */ + public static short floatToHalfBits(float value) { + final int f = Float.floatToRawIntBits(value); + final int sign = (f >>> 16) & 0x8000; + final int exponent = (f >>> 23) & 0xFF; + final int mantissa = f & 0x7FFFFF; + + if (exponent == 0xFF) { + if (mantissa == 0) { + return (short) (sign | 0x7C00); // Infinity. + } + // NaN. Keep the top payload bits, but never let the payload round down to zero, which + // would silently turn the NaN into an infinity. + int payload = mantissa >>> 13; + if (payload == 0) { + payload = 1; + } + return (short) (sign | 0x7C00 | payload); + } + + final int unbiasedExponent = exponent - 127; + + if (unbiasedExponent > 15) { + return (short) (sign | 0x7C00); // Overflows the binary16 range. + } + + if (unbiasedExponent >= -14) { + // Normal binary16. Keep the top 10 mantissa bits and round on the remaining 13. + final int halfMantissa = mantissa >>> 13; + final int roundBits = mantissa & 0x1FFF; + final int bits = sign | ((unbiasedExponent + 15) << 10) | halfMantissa; + if (roundBits > 0x1000 || (roundBits == 0x1000 && (halfMantissa & 1) != 0)) { + // A carry out of the mantissa increments the exponent, and a carry out of the + // maximum exponent yields infinity. Both are the correct IEEE 754 results. + return (short) (bits + 1); + } + return (short) bits; + } + + if (unbiasedExponent >= -25) { + // Subnormal binary16, or a value small enough that rounding may still reach the + // smallest subnormal. Restore the implicit leading bit, then shift into place so that + // the result is interpreted as halfMantissa * 2^-24. + final int significand = mantissa | 0x800000; + final int shift = -unbiasedExponent - 1; + final int halfMantissa = significand >>> shift; + final int roundBits = significand & ((1 << shift) - 1); + final int tie = 1 << (shift - 1); + if (roundBits > tie || (roundBits == tie && (halfMantissa & 1) != 0)) { + return (short) (sign | (halfMantissa + 1)); + } + return (short) (sign | halfMantissa); + } + + // Underflows to signed zero: strictly less than half of the smallest subnormal. + return (short) sign; + } +} diff --git a/src/main/java/dev/zarr/zarrjava/v2/DataType.java b/src/main/java/dev/zarr/zarrjava/v2/DataType.java index a0bbd731..973451b1 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/DataType.java +++ b/src/main/java/dev/zarr/zarrjava/v2/DataType.java @@ -12,6 +12,11 @@ public enum DataType implements dev.zarr.zarrjava.core.DataType { UINT16("u2", Endianness.LITTLE), UINT32("u4", Endianness.LITTLE), UINT64("u8", Endianness.LITTLE), + /** + * IEEE 754 binary16. Encoded as 2 bytes, but held in memory as {@code float}; see + * {@link dev.zarr.zarrjava.utils.Float16}. + */ + FLOAT16("f2", Endianness.LITTLE), FLOAT32("f4", Endianness.LITTLE), FLOAT64("f8", Endianness.LITTLE), INT16_BE("i2", Endianness.BIG), @@ -20,6 +25,7 @@ public enum DataType implements dev.zarr.zarrjava.core.DataType { UINT16_BE("u2", Endianness.BIG), UINT32_BE("u4", Endianness.BIG), UINT64_BE("u8", Endianness.BIG), + FLOAT16_BE("f2", Endianness.BIG), FLOAT32_BE("f4", Endianness.BIG), FLOAT64_BE("f8", Endianness.BIG); @@ -68,6 +74,9 @@ public ucar.ma2.DataType getMA2DataType() { case UINT64: case UINT64_BE: return ucar.ma2.DataType.ULONG; + case FLOAT16: + case FLOAT16_BE: + // No half precision type in ucar.ma2; widened to float, see Float16. case FLOAT32: case FLOAT32_BE: return ucar.ma2.DataType.FLOAT; @@ -84,4 +93,9 @@ public int getByteCount() { return Integer.parseInt(dtype.substring(1)); } + @Override + public boolean isHalfPrecisionFloat() { + return this == FLOAT16 || this == FLOAT16_BE; + } + } diff --git a/src/main/java/dev/zarr/zarrjava/v3/DataType.java b/src/main/java/dev/zarr/zarrjava/v3/DataType.java index 159663d5..7919a4e8 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/DataType.java +++ b/src/main/java/dev/zarr/zarrjava/v3/DataType.java @@ -15,6 +15,11 @@ public enum DataType implements dev.zarr.zarrjava.core.DataType { UINT16("uint16", 2), UINT32("uint32", 4), UINT64("uint64", 8), + /** + * IEEE 754 binary16. Encoded as 2 bytes, but held in memory as {@code float}; see + * {@link dev.zarr.zarrjava.utils.Float16}. + */ + FLOAT16("float16", 2), FLOAT32("float32", 4), FLOAT64( "float64", @@ -58,6 +63,8 @@ public ucar.ma2.DataType getMA2DataType() { return ucar.ma2.DataType.UINT; case UINT64: return ucar.ma2.DataType.ULONG; + case FLOAT16: + // No half precision type in ucar.ma2; widened to float, see Float16. case FLOAT32: return ucar.ma2.DataType.FLOAT; case FLOAT64: @@ -66,4 +73,9 @@ public ucar.ma2.DataType getMA2DataType() { throw new IllegalStateException("Unknown DataType: " + this); } } + + @Override + public boolean isHalfPrecisionFloat() { + return this == FLOAT16; + } } diff --git a/src/test/java/dev/zarr/zarrjava/Float16Test.java b/src/test/java/dev/zarr/zarrjava/Float16Test.java new file mode 100644 index 00000000..894937b0 --- /dev/null +++ b/src/test/java/dev/zarr/zarrjava/Float16Test.java @@ -0,0 +1,500 @@ +package dev.zarr.zarrjava; + +import dev.zarr.zarrjava.store.FilesystemStore; +import dev.zarr.zarrjava.store.StoreHandle; +import dev.zarr.zarrjava.utils.Float16; +import dev.zarr.zarrjava.v3.Array; +import dev.zarr.zarrjava.v3.DataType; +import dev.zarr.zarrjava.v3.codec.core.BytesCodec; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.EnumSource; +import org.junit.jupiter.params.provider.ValueSource; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Arrays; + +/** + * Tests for the {@code float16} data type. + * + *

The conversion cases are ground truth generated from {@code numpy.float16} (numpy 2.4.6); + * hex literals are exact bit patterns so the assertions do not depend on decimal parsing. + */ +public class Float16Test extends ZarrTest { + + final static Path PYTHON_TEST_PATH = Paths.get("src/test/python-scripts/"); + + /** + * The subset of the binary16 range that is exactly representable, so round-trips are exact. + * Mirrors VALUES in float16_interop.py. + */ + static final float[] EDGE_CASES = new float[]{ + 0.0f, + -0.0f, + 1.0f, + -2.0f, + 5.960464477539063E-8f, // smallest positive subnormal, 2^-24 + 6.097555160522461E-5f, // largest subnormal + 6.103515625E-5f, // smallest positive normal, 2^-14 + 65504.0f, // largest finite + -65504.0f, + Float.POSITIVE_INFINITY, + Float.NEGATIVE_INFINITY, + Float.NaN, + 0.333251953125f, // 1/3 rounded to binary16 + -0.0999755859375f, // 0.1 rounded to binary16, negated + }; + + static int runPython(String... args) throws IOException, InterruptedException { + String[] command = new String[args.length + 2]; + command[0] = "uv"; + command[1] = "run"; + System.arraycopy(args, 0, command, 2, args.length); + + ProcessBuilder pb = new ProcessBuilder(command); + Process process = pb.start(); + try (BufferedReader out = new BufferedReader(new InputStreamReader(process.getInputStream()))) { + String line; + while ((line = out.readLine()) != null) { + System.out.println(line); + } + } + try (BufferedReader err = new BufferedReader(new InputStreamReader(process.getErrorStream()))) { + String line; + while ((line = err.readLine()) != null) { + System.err.println(line); + } + } + return process.waitFor(); + } + + // ---------------------------------------------------------------- bit conversions + + /** + * Widening binary16 to float. Ground truth: numpy float32 bit patterns. + */ + @ParameterizedTest + @CsvSource({ + "0x0000, 0x00000000", // +0.0 + "0x8000, 0x80000000", // -0.0 + "0x0001, 0x33800000", // smallest subnormal, 2^-24 + "0x8001, 0xB3800000", // negative smallest subnormal + "0x0002, 0x34000000", // subnormal needing 9 normalization shifts + "0x03FF, 0x387FC000", // largest subnormal + "0x0400, 0x38800000", // smallest normal, 2^-14 + "0x3C00, 0x3F800000", // 1.0 + "0xBC00, 0xBF800000", // -1.0 + "0xC000, 0xC0000000", // -2.0 + "0x7BFF, 0x477FE000", // 65504.0, largest finite + "0xFBFF, 0xC77FE000", // -65504.0 + "0x7C00, 0x7F800000", // +Infinity + "0xFC00, 0xFF800000", // -Infinity + "0x7E00, 0x7FC00000", // NaN, payload preserved + "0x3555, 0x3EAAA000", // 1/3 rounded + }) + public void testHalfBitsToFloat(String halfHex, String floatHex) { + short halfBits = (short) Integer.parseInt(halfHex.substring(2), 16); + int expectedBits = (int) Long.parseLong(floatHex.substring(2), 16); + + float actual = Float16.halfBitsToFloat(halfBits); + + Assertions.assertEquals( + String.format("%08X", expectedBits), + String.format("%08X", Float.floatToRawIntBits(actual)), + "widening " + halfHex + " gave " + actual); + } + + /** + * Narrowing float to binary16, including round-half-to-even ties and overflow. Ground truth: + * numpy float16 bit patterns. + */ + @ParameterizedTest + @CsvSource({ + "1.0, 0x3C00", + "-2.0, 0xC000", + "0.0, 0x0000", + "-0.0, 0x8000", // signed zero survives + "65504.0, 0x7BFF", // largest finite + "65519.0, 0x7BFF", // just below the overflow midpoint + "65520.0, 0x7C00", // exactly the midpoint: ties-to-even overflows to Infinity + "65536.0, 0x7C00", + "1.0E8, 0x7C00", // far overflow + "-1.0E8, 0xFC00", + "5.9604645E-8, 0x0001", // smallest subnormal, exact + "3.0E-8, 0x0001", // rounds up to the smallest subnormal + "2.9802322E-8, 0x0000", // exactly 2^-25: ties-to-even rounds to zero + "1.5E-8, 0x0000", // underflows + "6.1035156E-5, 0x0400", // smallest normal + "6.0E-5, 0x03EF", // falls into the subnormal range + "0.1, 0x2E66", + "2048.0, 0x6800", // last exactly-representable integer + "2049.0, 0x6800", // tie rounds down to even + "2050.0, 0x6801", + "2051.0, 0x6802", // tie rounds up to even + "1.0009765625, 0x3C01", // one ulp above 1.0 + "1.00048828125, 0x3C00", // half an ulp: ties-to-even rounds down + }) + public void testFloatToHalfBits(float value, String expectedHex) { + short expected = (short) Integer.parseInt(expectedHex.substring(2), 16); + + short actual = Float16.floatToHalfBits(value); + + Assertions.assertEquals( + String.format("%04X", expected & 0xFFFF), + String.format("%04X", actual & 0xFFFF), + "narrowing " + value); + } + + @Test + public void testNaNNarrowsToNaN() { + // A NaN whose payload lives entirely in the low mantissa bits would round to a zero + // payload, turning it into an Infinity. It must stay a NaN. + float sneakyNaN = Float.intBitsToFloat(0x7F800001); + Assertions.assertTrue(Float.isNaN(sneakyNaN), "precondition: input is NaN"); + + short halfBits = Float16.floatToHalfBits(sneakyNaN); + + Assertions.assertTrue(Float.isNaN(Float16.halfBitsToFloat(halfBits)), + "NaN must not narrow to Infinity, got bits 0x" + String.format("%04X", halfBits & 0xFFFF)); + } + + @Test + public void testRoundTripIsIdentityForRepresentableValues() { + for (float value : EDGE_CASES) { + float actual = Float16.halfBitsToFloat(Float16.floatToHalfBits(value)); + if (Float.isNaN(value)) { + Assertions.assertTrue(Float.isNaN(actual), "NaN round-trip"); + } else { + Assertions.assertEquals( + Float.floatToRawIntBits(value), Float.floatToRawIntBits(actual), + "round-trip of " + value + " gave " + actual); + } + } + } + + @Test + public void testEveryHalfBitPatternRoundTrips() { + // Exhaustive over all 65536 patterns: widening then narrowing must be the identity on + // bits, except that NaN payloads are only required to stay NaN. + for (int bits = 0; bits <= 0xFFFF; bits++) { + short halfBits = (short) bits; + float widened = Float16.halfBitsToFloat(halfBits); + short narrowed = Float16.floatToHalfBits(widened); + if (Float.isNaN(widened)) { + Assertions.assertTrue(Float.isNaN(Float16.halfBitsToFloat(narrowed)), + "NaN pattern 0x" + String.format("%04X", bits) + " stopped being NaN"); + } else { + Assertions.assertEquals(bits, narrowed & 0xFFFF, + "pattern 0x" + String.format("%04X", bits) + " did not round-trip"); + } + } + } + + // ---------------------------------------------------------------- metadata + + @Test + public void testMetadataParsesFloat16() throws Exception { + String json = "{\"zarr_format\":3,\"node_type\":\"array\",\"shape\":[4]," + + "\"data_type\":\"float16\"," + + "\"chunk_grid\":{\"name\":\"regular\",\"configuration\":{\"chunk_shape\":[2]}}," + + "\"chunk_key_encoding\":{\"name\":\"default\"},\"fill_value\":0," + + "\"codecs\":[{\"name\":\"bytes\",\"configuration\":{\"endian\":\"little\"}}]}"; + + dev.zarr.zarrjava.v3.ArrayMetadata metadata = dev.zarr.zarrjava.v3.Node.makeObjectMapper() + .readValue(json, dev.zarr.zarrjava.v3.ArrayMetadata.class); + + Assertions.assertEquals(DataType.FLOAT16, metadata.dataType); + Assertions.assertEquals(2, metadata.dataType.getByteCount()); + Assertions.assertEquals(ucar.ma2.DataType.FLOAT, metadata.dataType.getMA2DataType()); + Assertions.assertTrue(metadata.dataType.isHalfPrecisionFloat()); + } + + @Test + public void testDataTypeSerializesToFloat16() throws Exception { + String json = dev.zarr.zarrjava.v3.Node.makeObjectMapper().writeValueAsString(DataType.FLOAT16); + Assertions.assertEquals("\"float16\"", json); + } + + /** + * Fill values, in every form the spec allows for a float. The hex and binary forms carry a + * binary16 bit pattern, not a truncated float32. + */ + @ParameterizedTest + @CsvSource({ + "0, 0.0", + "1, 1.0", + "-2.5, -2.5", + "'NaN', NaN", + "'+Infinity', Infinity", + "'-Infinity', -Infinity", + "'0x003C', 1.0", // 0x3C00 little-endian + "'0x00C0', -2.0", // 0xC000 little-endian + "'0b0000000000111100', 1.0", // same bits, binary form + }) + public void testFillValueForms(String fillValueJson, String expected) throws Exception { + String quoted = fillValueJson.startsWith("0x") || fillValueJson.startsWith("0b") + || fillValueJson.equals("NaN") || fillValueJson.endsWith("Infinity") + ? "\"" + fillValueJson + "\"" : fillValueJson; + String json = "{\"zarr_format\":3,\"node_type\":\"array\",\"shape\":[4]," + + "\"data_type\":\"float16\"," + + "\"chunk_grid\":{\"name\":\"regular\",\"configuration\":{\"chunk_shape\":[2]}}," + + "\"chunk_key_encoding\":{\"name\":\"default\"}," + + "\"fill_value\":" + quoted + "," + + "\"codecs\":[{\"name\":\"bytes\",\"configuration\":{\"endian\":\"little\"}}]}"; + + dev.zarr.zarrjava.v3.ArrayMetadata metadata = dev.zarr.zarrjava.v3.Node.makeObjectMapper() + .readValue(json, dev.zarr.zarrjava.v3.ArrayMetadata.class); + + Assertions.assertEquals(Float.class, metadata.parsedFillValue.getClass(), + "float16 fill values are parsed to Float, matching the in-memory type"); + Assertions.assertEquals(Float.parseFloat(expected), (Float) metadata.parsedFillValue, 1e-9f); + } + + @Test + public void testUnwrittenChunksReadAsFillValue() throws Exception { + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("float16_fill"); + Array array = Array.create(storeHandle, Array.metadataBuilder() + .withShape(4) + .withDataType(DataType.FLOAT16) + .withChunkShape(2) + .withFillValue(-2.5) + .withCodecs(c -> c.withBytes(BytesCodec.Endian.LITTLE)) + .build()); + + ucar.ma2.Array result = array.read(); + + Assertions.assertEquals(ucar.ma2.DataType.FLOAT, result.getDataType()); + for (int i = 0; i < 4; i++) { + Assertions.assertEquals(-2.5f, result.getFloat(i), 0.0f, "index " + i); + } + } + + // ---------------------------------------------------------------- zarr-java round-trip + + @ParameterizedTest + @EnumSource(BytesCodec.Endian.class) + public void testRoundTripBothEndiannesses(BytesCodec.Endian endian) throws Exception { + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT) + .resolve("float16_roundtrip", endian.name()); + + Array array = Array.create(storeHandle, Array.metadataBuilder() + .withShape(EDGE_CASES.length) + .withDataType(DataType.FLOAT16) + .withChunkShape(5) + .withFillValue(0) + .withCodecs(c -> c.withBytes(endian)) + .build()); + + array.write(ucar.ma2.Array.factory( + ucar.ma2.DataType.FLOAT, new int[]{EDGE_CASES.length}, EDGE_CASES.clone())); + + ucar.ma2.Array result = Array.open(storeHandle).read(); + + assertEdgeCases(result); + } + + @Test + public void testEncodedChunkIsTwoBytesPerElement() throws Exception { + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("float16_width"); + Array array = Array.create(storeHandle, Array.metadataBuilder() + .withShape(8) + .withDataType(DataType.FLOAT16) + .withChunkShape(8) + .withFillValue(0) + .withCodecs(c -> c.withBytes(BytesCodec.Endian.LITTLE)) + .build()); + + array.write(ucar.ma2.Array.factory(ucar.ma2.DataType.FLOAT, new int[]{8}, + new float[]{1, 2, 3, 4, 5, 6, 7, 8})); + + long chunkSize = storeHandle.resolve("c", "0").getSize(); + Assertions.assertEquals(16, chunkSize, + "8 float16 elements must encode to 16 bytes, not 32"); + } + + @Test + public void testEndiannessActuallySwapsBytes() throws Exception { + // 1.0 is 0x3C00, whose bytes differ between orders, so a byte-level check is meaningful. + long[] sizes = new long[2]; + byte[][] firstBytes = new byte[2][]; + BytesCodec.Endian[] orders = {BytesCodec.Endian.LITTLE, BytesCodec.Endian.BIG}; + + for (int i = 0; i < orders.length; i++) { + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT) + .resolve("float16_endian", orders[i].name()); + final BytesCodec.Endian endian = orders[i]; + Array array = Array.create(storeHandle, Array.metadataBuilder() + .withShape(1) + .withDataType(DataType.FLOAT16) + .withChunkShape(1) + .withFillValue(0) + .withCodecs(c -> c.withBytes(endian)) + .build()); + array.write(ucar.ma2.Array.factory(ucar.ma2.DataType.FLOAT, new int[]{1}, + new float[]{1.0f})); + + byte[] bytes = dev.zarr.zarrjava.utils.Utils.toArray( + storeHandle.resolve("c", "0").readNonNull()); + sizes[i] = bytes.length; + firstBytes[i] = bytes; + } + + Assertions.assertArrayEquals(new byte[]{0x00, 0x3C}, firstBytes[0], "little-endian 1.0"); + Assertions.assertArrayEquals(new byte[]{0x3C, 0x00}, firstBytes[1], "big-endian 1.0"); + Assertions.assertEquals(2, sizes[0]); + } + + @ParameterizedTest + @ValueSource(strings = {"blosc", "zstd", "gzip", "sharding", "crc32c"}) + public void testRoundTripThroughCompressingCodecs(String codec) throws Exception { + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("float16_codec", codec); + + Array array = Array.create(storeHandle, Array.metadataBuilder() + .withShape(EDGE_CASES.length) + .withDataType(DataType.FLOAT16) + .withChunkShape(EDGE_CASES.length) + .withFillValue(0) + .withCodecs(c -> { + switch (codec) { + case "blosc": + // typesize must follow the encoded width, 2 bytes, not the in-memory 4. + return c.withBlosc("zstd", "shuffle", 5); + case "zstd": + return c.withZstd(3); + case "gzip": + return c.withGzip(5); + case "sharding": + return c.withSharding(new int[]{7}, c1 -> c1.withBytes("LITTLE")); + case "crc32c": + return c.withBytes(BytesCodec.Endian.LITTLE).withCrc32c(); + default: + throw new IllegalArgumentException(codec); + } + }) + .build()); + + array.write(ucar.ma2.Array.factory( + ucar.ma2.DataType.FLOAT, new int[]{EDGE_CASES.length}, EDGE_CASES.clone())); + + assertEdgeCases(Array.open(storeHandle).read()); + } + + @Test + public void testBloscTypesizeIsEncodedWidth() { + Assertions.assertEquals(2, DataType.FLOAT16.getByteCount(), + "blosc typesize is derived from getByteCount, which must be the encoded width"); + } + + // ---------------------------------------------------------------- v2 + + @Test + public void testV2RoundTrip() throws Exception { + for (dev.zarr.zarrjava.v2.DataType dt : new dev.zarr.zarrjava.v2.DataType[]{ + dev.zarr.zarrjava.v2.DataType.FLOAT16, dev.zarr.zarrjava.v2.DataType.FLOAT16_BE}) { + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT) + .resolve("float16_v2", dt.name()); + + dev.zarr.zarrjava.v2.Array array = dev.zarr.zarrjava.v2.Array.create( + storeHandle, + dev.zarr.zarrjava.v2.Array.metadataBuilder() + .withShape(EDGE_CASES.length) + .withChunks(5) + .withDataType(dt) + .withFillValue(0) + .build()); + + array.write(ucar.ma2.Array.factory( + ucar.ma2.DataType.FLOAT, new int[]{EDGE_CASES.length}, EDGE_CASES.clone())); + + Assertions.assertEquals(2, dt.getByteCount(), dt + " byte count"); + assertEdgeCases(dev.zarr.zarrjava.v2.Array.open(storeHandle).read()); + } + } + + @Test + public void testV2DtypeStrings() { + Assertions.assertEquals("f2", dev.zarr.zarrjava.v2.DataType.FLOAT16_BE.getValue()); + } + + // ---------------------------------------------------------------- zarr-python interop + + @ParameterizedTest + @CsvSource({"3, LITTLE", "3, BIG", "2, LITTLE", "2, BIG"}) + public void testPythonWritesJavaReads(int zarrFormat, String endian) throws Exception { + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT) + .resolve("float16_py2java", "v" + zarrFormat, endian); + + int exit = runPython(PYTHON_TEST_PATH.resolve("float16_interop.py").toString(), + "write", storeHandle.toPath().toString(), String.valueOf(zarrFormat), endian); + Assertions.assertEquals(0, exit, "zarr-python writer failed"); + + ucar.ma2.Array result = zarrFormat == 3 + ? Array.open(storeHandle).read() + : dev.zarr.zarrjava.v2.Array.open(storeHandle).read(); + + assertEdgeCases(result); + } + + @ParameterizedTest + @CsvSource({"3, LITTLE", "3, BIG", "2, LITTLE", "2, BIG"}) + public void testJavaWritesPythonReads(int zarrFormat, String endian) throws Exception { + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT) + .resolve("float16_java2py", "v" + zarrFormat, endian); + + if (zarrFormat == 3) { + Array.create(storeHandle, Array.metadataBuilder() + .withShape(EDGE_CASES.length) + .withDataType(DataType.FLOAT16) + .withChunkShape(5) + .withFillValue(0) + .withCodecs(c -> c.withBytes(BytesCodec.Endian.valueOf(endian))) + .build()) + .write(ucar.ma2.Array.factory(ucar.ma2.DataType.FLOAT, + new int[]{EDGE_CASES.length}, EDGE_CASES.clone())); + } else { + dev.zarr.zarrjava.v2.DataType dt = endian.equals("BIG") + ? dev.zarr.zarrjava.v2.DataType.FLOAT16_BE + : dev.zarr.zarrjava.v2.DataType.FLOAT16; + dev.zarr.zarrjava.v2.Array.create(storeHandle, + dev.zarr.zarrjava.v2.Array.metadataBuilder() + .withShape(EDGE_CASES.length) + .withChunks(5) + .withDataType(dt) + .withFillValue(0) + .build()) + .write(ucar.ma2.Array.factory(ucar.ma2.DataType.FLOAT, + new int[]{EDGE_CASES.length}, EDGE_CASES.clone())); + } + + int exit = runPython(PYTHON_TEST_PATH.resolve("float16_interop.py").toString(), + "verify", storeHandle.toPath().toString(), String.valueOf(zarrFormat)); + Assertions.assertEquals(0, exit, "zarr-python could not read what zarr-java wrote"); + } + + // ---------------------------------------------------------------- helper + + private static void assertEdgeCases(ucar.ma2.Array result) { + Assertions.assertEquals(EDGE_CASES.length, result.getSize(), "element count"); + for (int i = 0; i < EDGE_CASES.length; i++) { + float expected = EDGE_CASES[i]; + float actual = result.getFloat(i); + if (Float.isNaN(expected)) { + Assertions.assertTrue(Float.isNaN(actual), "index " + i + " should be NaN"); + } else { + // Raw bits, so that +0.0 and -0.0 are told apart. + Assertions.assertEquals( + Float.floatToRawIntBits(expected), Float.floatToRawIntBits(actual), + "index " + i + ": expected " + expected + " got " + actual + + " (all: " + Arrays.toString( + (float[]) result.get1DJavaArray(ucar.ma2.DataType.FLOAT)) + ")"); + } + } + } +} diff --git a/src/test/java/dev/zarr/zarrjava/ZarrTest.java b/src/test/java/dev/zarr/zarrjava/ZarrTest.java index fdb8ec17..c8fd44e0 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrTest.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrTest.java @@ -46,6 +46,7 @@ static Stream dataTypeProviderV3() { DataType.UINT32, DataType.INT64, DataType.UINT64, + DataType.FLOAT16, DataType.FLOAT32, DataType.FLOAT64 ); @@ -62,6 +63,7 @@ static Stream dataTypeProviderV2() { dev.zarr.zarrjava.v2.DataType.UINT32, dev.zarr.zarrjava.v2.DataType.INT64, dev.zarr.zarrjava.v2.DataType.UINT64, + dev.zarr.zarrjava.v2.DataType.FLOAT16, dev.zarr.zarrjava.v2.DataType.FLOAT32, dev.zarr.zarrjava.v2.DataType.FLOAT64, dev.zarr.zarrjava.v2.DataType.UINT16_BE, @@ -70,6 +72,7 @@ static Stream dataTypeProviderV2() { dev.zarr.zarrjava.v2.DataType.INT16_BE, dev.zarr.zarrjava.v2.DataType.INT32_BE, dev.zarr.zarrjava.v2.DataType.INT64_BE, + dev.zarr.zarrjava.v2.DataType.FLOAT16_BE, dev.zarr.zarrjava.v2.DataType.FLOAT32_BE, dev.zarr.zarrjava.v2.DataType.FLOAT64_BE ); @@ -154,6 +157,21 @@ protected void assertContainsTestAttributes(Attributes attributes) throws ZarrEx } + /** + * Quantizes a test value the way the data type would store it. Only float16 loses anything: it + * carries 11 significand bits, so 1024 of the 4096 values below are not exactly representable + * (2049 stores as 2048, 2051 as 2052, 4095 as 4096). Pre-quantizing here keeps + * {@link #testdata} and {@link #assertIsTestdata} in agreement, and matches + * {@code np.arange(16 * 16 * 16, dtype='float16')} element for element. + */ + private static float quantize(dev.zarr.zarrjava.core.DataType dt, int i) { + if (dt.isHalfPrecisionFloat()) { + return dev.zarr.zarrjava.utils.Float16.halfBitsToFloat( + dev.zarr.zarrjava.utils.Float16.floatToHalfBits((float) i)); + } + return (float) i; + } + protected ucar.ma2.Array testdata(dev.zarr.zarrjava.core.DataType dt) { ucar.ma2.DataType ma2Type = dt.getMA2DataType(); ucar.ma2.Array array = ucar.ma2.Array.factory(ma2Type, new int[]{16, 16, 16}); @@ -181,7 +199,7 @@ protected ucar.ma2.Array testdata(dev.zarr.zarrjava.core.DataType dt) { array.setLong(i, i); break; case FLOAT: - array.setFloat(i, (float) i); + array.setFloat(i, quantize(dt, i)); break; case DOUBLE: array.setDouble(i, i); @@ -220,7 +238,7 @@ protected void assertIsTestdata(ucar.ma2.Array result, dev.zarr.zarrjava.core.Da Assertions.assertEquals(i, result.getLong(i)); break; case FLOAT: - Assertions.assertEquals((float) i, result.getFloat(i), 1e-6); + Assertions.assertEquals(quantize(dt, i), result.getFloat(i), 1e-6); break; case DOUBLE: Assertions.assertEquals(i, result.getDouble(i), 1e-12); diff --git a/src/test/python-scripts/float16_interop.py b/src/test/python-scripts/float16_interop.py new file mode 100644 index 00000000..dfd8bede --- /dev/null +++ b/src/test/python-scripts/float16_interop.py @@ -0,0 +1,106 @@ +"""Cross-implementation check for the float16 data type. + +Usage: + float16_interop.py write + float16_interop.py verify + +`write` produces a float16 array for zarr-java to read; `verify` reads an array +zarr-java produced and asserts it holds the same values. Both directions are needed: +a zarr-java round-trip cannot catch a symmetric encode/decode bug. + +Every value below is exactly representable in binary16, so the comparison is exact +and never depends on rounding. +""" + +import sys + +import numpy as np +import zarr +from zarr.storage import LocalStore + +# Ordered edge cases: signed zeros, min/max subnormal, min normal, max finite, +# infinities, NaN, and a few rounded values. +VALUES = np.array( + [ + 0.0, + -0.0, + 1.0, + -2.0, + 5.960464477539063e-08, # smallest positive subnormal, 2**-24 + 6.097555160522461e-05, # largest subnormal + 6.103515625e-05, # smallest positive normal, 2**-14 + 65504.0, # largest finite + -65504.0, + np.inf, + -np.inf, + np.nan, + 0.333251953125, # 1/3 rounded to binary16 + -0.0999755859375, # 0.1 rounded to binary16, negated + ], + dtype=np.float16, +) + +SHAPE = (len(VALUES),) +CHUNKS = (5,) # Deliberately not a divisor of 14, so the last chunk is partial. + + +def assert_matches(actual): + actual = np.asarray(actual, dtype=np.float16) + assert actual.shape == SHAPE, f"shape {actual.shape} != {SHAPE}" + for i, (got, want) in enumerate(zip(actual, VALUES)): + if np.isnan(want): + assert np.isnan(got), f"index {i}: got {got!r}, expected NaN" + else: + # Exact bit comparison, so that +0.0 and -0.0 are told apart. + got_bits = np.frombuffer(np.float16(got).tobytes(), dtype=np.uint16)[0] + want_bits = np.frombuffer(np.float16(want).tobytes(), dtype=np.uint16)[0] + assert got_bits == want_bits, ( + f"index {i}: got {got!r} (0x{got_bits:04X}), " + f"expected {want!r} (0x{want_bits:04X})" + ) + + +def main(): + mode = sys.argv[1] + store_path = sys.argv[2] + zarr_format = int(sys.argv[3]) + + if mode == "write": + endian = sys.argv[4] + dtype = ">f2" if endian == "BIG" else "f2", which is not == to native "float16". + assert a.dtype.kind == "f" and a.dtype.itemsize == 2, ( + f"dtype is {a.dtype}, expected a 2-byte float" + ) + assert_matches(a[:]) + print(f"verified float16 v{zarr_format} at {store_path}") + + else: + raise SystemExit(f"unknown mode {mode!r}") + + +if __name__ == "__main__": + main()