diff --git a/src/main/java/dev/zarr/zarrjava/core/ArrayMetadata.java b/src/main/java/dev/zarr/zarrjava/core/ArrayMetadata.java index a2fb48f3..3c7db297 100644 --- a/src/main/java/dev/zarr/zarrjava/core/ArrayMetadata.java +++ b/src/main/java/dev/zarr/zarrjava/core/ArrayMetadata.java @@ -5,6 +5,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import dev.zarr.zarrjava.ZarrException; import dev.zarr.zarrjava.core.chunkkeyencoding.ChunkKeyEncoding; +import dev.zarr.zarrjava.utils.Float16; import dev.zarr.zarrjava.utils.MultiArrayUtils; import dev.zarr.zarrjava.utils.Utils; import ucar.ma2.Array; @@ -41,6 +42,8 @@ public static Object parseFillValue(Object fillValue, @Nonnull DataType dataType boolean dataTypeIsLong = dataType == dev.zarr.zarrjava.v3.DataType.INT64 || dataType == dev.zarr.zarrjava.v2.DataType.INT64 || dataType == dev.zarr.zarrjava.v2.DataType.INT64_BE || dataType == dev.zarr.zarrjava.v3.DataType.UINT64 || dataType == dev.zarr.zarrjava.v2.DataType.UINT64 || dataType == dev.zarr.zarrjava.v2.DataType.UINT64_BE; boolean dataTypeIsFloat = dataType == dev.zarr.zarrjava.v3.DataType.FLOAT32 || dataType == dev.zarr.zarrjava.v2.DataType.FLOAT32 || dataType == dev.zarr.zarrjava.v2.DataType.FLOAT32_BE; boolean dataTypeIsDouble = dataType == dev.zarr.zarrjava.v3.DataType.FLOAT64 || dataType == dev.zarr.zarrjava.v2.DataType.FLOAT64 || dataType == dev.zarr.zarrjava.v2.DataType.FLOAT64_BE; + // float16 fill values are parsed to float, matching the in-memory representation. + boolean dataTypeIsHalfFloat = dataType.isHalfPrecisionFloat(); if (fillValue instanceof Boolean) { Boolean fillValueBool = (Boolean) fillValue; @@ -60,7 +63,7 @@ public static Object parseFillValue(Object fillValue, @Nonnull DataType dataType return fillValueNumber.intValue(); } else if (dataTypeIsLong) { return fillValueNumber.longValue(); - } else if (dataTypeIsFloat) { + } else if (dataTypeIsFloat || dataTypeIsHalfFloat) { return fillValueNumber.floatValue(); } else if (dataTypeIsDouble) { return fillValueNumber.doubleValue(); @@ -69,7 +72,7 @@ public static Object parseFillValue(Object fillValue, @Nonnull DataType dataType } else if (fillValue instanceof String) { String fillValueString = (String) fillValue; if (fillValueString.equals("NaN")) { - if (dataTypeIsFloat) { + if (dataTypeIsFloat || dataTypeIsHalfFloat) { return Float.NaN; } else if (dataTypeIsDouble) { return Double.NaN; @@ -77,7 +80,7 @@ public static Object parseFillValue(Object fillValue, @Nonnull DataType dataType throw new ZarrException( "Invalid fill value '" + fillValueString + "' for data type '" + dataType + "'."); } else if (fillValueString.equals("+Infinity")) { - if (dataTypeIsFloat) { + if (dataTypeIsFloat || dataTypeIsHalfFloat) { return Float.POSITIVE_INFINITY; } else if (dataTypeIsDouble) { return Double.POSITIVE_INFINITY; @@ -85,7 +88,7 @@ public static Object parseFillValue(Object fillValue, @Nonnull DataType dataType throw new ZarrException( "Invalid fill value '" + fillValueString + "' for data type '" + dataType + "'."); } else if (fillValueString.equals("-Infinity")) { - if (dataTypeIsFloat) { + if (dataTypeIsFloat || dataTypeIsHalfFloat) { return Float.NEGATIVE_INFINITY; } else if (dataTypeIsDouble) { return Double.NEGATIVE_INFINITY; @@ -116,6 +119,9 @@ public static Object parseFillValue(Object fillValue, @Nonnull DataType dataType return buf.get() != 0; } else if (dataTypeIsByte) { return buf.get(); + } else if (dataTypeIsHalfFloat) { + // The 2 bytes are a binary16 bit pattern, not a truncated float32. + return Float16.halfBitsToFloat(buf.getShort()); } else if (dataTypeIsShort) { return buf.getShort(); } else if (dataTypeIsInt) { diff --git a/src/main/java/dev/zarr/zarrjava/core/DataType.java b/src/main/java/dev/zarr/zarrjava/core/DataType.java index d63de069..d7d2ed2e 100644 --- a/src/main/java/dev/zarr/zarrjava/core/DataType.java +++ b/src/main/java/dev/zarr/zarrjava/core/DataType.java @@ -4,4 +4,19 @@ public interface DataType { ucar.ma2.DataType getMA2DataType(); int getByteCount(); + + /** + * Whether elements are stored as IEEE 754 binary16 but held in memory as {@code float}. + * + *
For every other data type the encoded element width ({@link #getByteCount()}) matches the + * width of the in-memory {@link ucar.ma2.DataType}. Half precision is the exception: it encodes + * to 2 bytes but is held as a 4-byte {@code float}, because neither Java nor {@code ucar.ma2} + * has a 16-bit floating point type. The {@code bytes} codec uses this to convert at the + * encode/decode boundary via {@link dev.zarr.zarrjava.utils.Float16}. + * + * @return true for {@code float16}, false for every other data type + */ + default boolean isHalfPrecisionFloat() { + return false; + } } diff --git a/src/main/java/dev/zarr/zarrjava/core/codec/core/BytesCodec.java b/src/main/java/dev/zarr/zarrjava/core/codec/core/BytesCodec.java index 41382b5e..e0233eaa 100644 --- a/src/main/java/dev/zarr/zarrjava/core/codec/core/BytesCodec.java +++ b/src/main/java/dev/zarr/zarrjava/core/codec/core/BytesCodec.java @@ -3,6 +3,7 @@ import com.fasterxml.jackson.annotation.JsonValue; import dev.zarr.zarrjava.ZarrException; import dev.zarr.zarrjava.core.codec.ArrayBytesCodec; +import dev.zarr.zarrjava.utils.Float16; import ucar.ma2.*; import java.nio.ByteBuffer; @@ -31,6 +32,21 @@ public Array decode(ByteBuffer chunkBytes) throws ZarrException { Index index = Index.factory(shape); return Array.factory(DataType.BOOLEAN, index, bools); } + + // float16 encodes to 2 bytes per element but is held as a 4-byte float, so it cannot be + // read straight into an ma2 array; widen each element instead. + if (arrayMetadata.dataType.isHalfPrecisionFloat()) { + int base = chunkBytes.position(); + int size = chunkBytes.remaining() / Short.BYTES; + float[] floats = new float[size]; + for (int i = 0; i < size; i++) { + floats[i] = Float16.halfBitsToFloat(chunkBytes.getShort(base + i * Short.BYTES)); + } + + Index index = Index.factory(shape); + return Array.factory(DataType.FLOAT, index, floats); + } + return Array.factory(dtype, shape, chunkBytes); } @@ -51,6 +67,18 @@ public ByteBuffer encode(Array chunkArray) throws ZarrException { return bb; } + // Float16. Checked before ArrayFloat, because float16 is held in memory as an ArrayFloat + // and must be narrowed to 2 bytes per element rather than written as float32. + if (arrayMetadata.dataType.isHalfPrecisionFloat()) { + float[] data = (float[]) chunkArray.get1DJavaArray(DataType.FLOAT); + ByteBuffer bb = ByteBuffer.allocate(data.length * Short.BYTES).order(order); + for (float f : data) { + bb.putShort(Float16.floatToHalfBits(f)); + } + bb.flip(); + return bb; + } + // Float32 if (chunkArray instanceof ArrayFloat) { float[] data = (float[]) chunkArray.copyTo1DJavaArray(); diff --git a/src/main/java/dev/zarr/zarrjava/utils/Float16.java b/src/main/java/dev/zarr/zarrjava/utils/Float16.java new file mode 100644 index 00000000..9b094128 --- /dev/null +++ b/src/main/java/dev/zarr/zarrjava/utils/Float16.java @@ -0,0 +1,155 @@ +package dev.zarr.zarrjava.utils; + +/** + * Conversions between IEEE 754 binary16 ("half precision") bit patterns and Java {@code float}. + * + *
Zarr's {@code float16} data type stores 2 bytes per element, but there is no 16-bit floating + * point primitive in Java and no half precision type in {@code ucar.ma2.DataType}. Half precision + * arrays are therefore held in memory as {@code float} (binary32) and converted at the codec + * boundary. Widening binary16 to binary32 is always exact; narrowing is not, see + * {@link #floatToHalfBits(float)}. + * + *
{@code Float.float16ToFloat} and {@code Float.floatToFloat16} would do this for us, but they + * were added in Java 20 and this project targets Java 8, so the conversions are implemented here. + * The results match {@code numpy.float16} for all inputs, including subnormals, signed zeros, + * infinities, NaN payloads and round-half-to-even ties. + */ +public final class Float16 { + + /** + * Largest finite binary16 value, {@code 65504.0}. Finite {@code float} values greater than + * {@code MAX_VALUE + 16} round to infinity when narrowed. + */ + public static final float MAX_VALUE = 65504.0f; + + /** Smallest positive normal binary16 value, {@code 2^-14}. */ + public static final float MIN_NORMAL = 6.103515625E-5f; + + /** Smallest positive subnormal binary16 value, {@code 2^-24}. */ + public static final float MIN_VALUE = 5.9604645E-8f; + + private Float16() { + } + + /** + * Widens a binary16 bit pattern to the {@code float} of equal value. Exact for every input: + * every binary16 value is representable as a binary32. + * + *
Signed zeros and infinities are preserved. NaN payloads are shifted into the binary32 + * mantissa, so a NaN stays a NaN rather than becoming an infinity. + * + * @param halfBits the 16 bits of a binary16 value, in the low half of the short + * @return the equal-valued float + */ + public static float halfBitsToFloat(short halfBits) { + final int h = halfBits & 0xFFFF; + final int signBit = (h >>> 15) & 0x1; + final int exponent = (h >>> 10) & 0x1F; + int mantissa = h & 0x3FF; + + final int floatSign = signBit << 31; + + if (exponent == 0) { + if (mantissa == 0) { + // Signed zero. + return Float.intBitsToFloat(floatSign); + } + // Subnormal binary16, which is normal as a binary32: shift the mantissa up until the + // implicit leading bit is set, decrementing the exponent for each shift. + int normalizedExponent = 1; + while ((mantissa & 0x400) == 0) { + mantissa <<= 1; + normalizedExponent--; + } + mantissa &= 0x3FF; // Drop the leading bit; it is implicit in binary32 too. + return Float.intBitsToFloat( + floatSign | ((normalizedExponent - 15 + 127) << 23) | (mantissa << 13)); + } + + if (exponent == 0x1F) { + // Infinity (mantissa 0) or NaN (mantissa non-zero); shifting keeps the payload non-zero. + return Float.intBitsToFloat(floatSign | 0x7F800000 | (mantissa << 13)); + } + + // Normal binary16. + return Float.intBitsToFloat(floatSign | ((exponent - 15 + 127) << 23) | (mantissa << 13)); + } + + /** + * Narrows a {@code float} to the nearest binary16 bit pattern, rounding to nearest with ties to + * even -- the same rounding {@code numpy.float16} and IEEE 754 use. + * + *
This conversion is lossy. binary16 carries 11 bits of significand against binary32's 24, + * and its finite range stops at {@value #MAX_VALUE}: + *
The conversion cases are ground truth generated from {@code numpy.float16} (numpy 2.4.6);
+ * hex literals are exact bit patterns so the assertions do not depend on decimal parsing.
+ */
+public class Float16Test extends ZarrTest {
+
+ final static Path PYTHON_TEST_PATH = Paths.get("src/test/python-scripts/");
+
+ /**
+ * The subset of the binary16 range that is exactly representable, so round-trips are exact.
+ * Mirrors VALUES in float16_interop.py.
+ */
+ static final float[] EDGE_CASES = new float[]{
+ 0.0f,
+ -0.0f,
+ 1.0f,
+ -2.0f,
+ 5.960464477539063E-8f, // smallest positive subnormal, 2^-24
+ 6.097555160522461E-5f, // largest subnormal
+ 6.103515625E-5f, // smallest positive normal, 2^-14
+ 65504.0f, // largest finite
+ -65504.0f,
+ Float.POSITIVE_INFINITY,
+ Float.NEGATIVE_INFINITY,
+ Float.NaN,
+ 0.333251953125f, // 1/3 rounded to binary16
+ -0.0999755859375f, // 0.1 rounded to binary16, negated
+ };
+
+ static int runPython(String... args) throws IOException, InterruptedException {
+ String[] command = new String[args.length + 2];
+ command[0] = "uv";
+ command[1] = "run";
+ System.arraycopy(args, 0, command, 2, args.length);
+
+ ProcessBuilder pb = new ProcessBuilder(command);
+ Process process = pb.start();
+ try (BufferedReader out = new BufferedReader(new InputStreamReader(process.getInputStream()))) {
+ String line;
+ while ((line = out.readLine()) != null) {
+ System.out.println(line);
+ }
+ }
+ try (BufferedReader err = new BufferedReader(new InputStreamReader(process.getErrorStream()))) {
+ String line;
+ while ((line = err.readLine()) != null) {
+ System.err.println(line);
+ }
+ }
+ return process.waitFor();
+ }
+
+ // ---------------------------------------------------------------- bit conversions
+
+ /**
+ * Widening binary16 to float. Ground truth: numpy float32 bit patterns.
+ */
+ @ParameterizedTest
+ @CsvSource({
+ "0x0000, 0x00000000", // +0.0
+ "0x8000, 0x80000000", // -0.0
+ "0x0001, 0x33800000", // smallest subnormal, 2^-24
+ "0x8001, 0xB3800000", // negative smallest subnormal
+ "0x0002, 0x34000000", // subnormal needing 9 normalization shifts
+ "0x03FF, 0x387FC000", // largest subnormal
+ "0x0400, 0x38800000", // smallest normal, 2^-14
+ "0x3C00, 0x3F800000", // 1.0
+ "0xBC00, 0xBF800000", // -1.0
+ "0xC000, 0xC0000000", // -2.0
+ "0x7BFF, 0x477FE000", // 65504.0, largest finite
+ "0xFBFF, 0xC77FE000", // -65504.0
+ "0x7C00, 0x7F800000", // +Infinity
+ "0xFC00, 0xFF800000", // -Infinity
+ "0x7E00, 0x7FC00000", // NaN, payload preserved
+ "0x3555, 0x3EAAA000", // 1/3 rounded
+ })
+ public void testHalfBitsToFloat(String halfHex, String floatHex) {
+ short halfBits = (short) Integer.parseInt(halfHex.substring(2), 16);
+ int expectedBits = (int) Long.parseLong(floatHex.substring(2), 16);
+
+ float actual = Float16.halfBitsToFloat(halfBits);
+
+ Assertions.assertEquals(
+ String.format("%08X", expectedBits),
+ String.format("%08X", Float.floatToRawIntBits(actual)),
+ "widening " + halfHex + " gave " + actual);
+ }
+
+ /**
+ * Narrowing float to binary16, including round-half-to-even ties and overflow. Ground truth:
+ * numpy float16 bit patterns.
+ */
+ @ParameterizedTest
+ @CsvSource({
+ "1.0, 0x3C00",
+ "-2.0, 0xC000",
+ "0.0, 0x0000",
+ "-0.0, 0x8000", // signed zero survives
+ "65504.0, 0x7BFF", // largest finite
+ "65519.0, 0x7BFF", // just below the overflow midpoint
+ "65520.0, 0x7C00", // exactly the midpoint: ties-to-even overflows to Infinity
+ "65536.0, 0x7C00",
+ "1.0E8, 0x7C00", // far overflow
+ "-1.0E8, 0xFC00",
+ "5.9604645E-8, 0x0001", // smallest subnormal, exact
+ "3.0E-8, 0x0001", // rounds up to the smallest subnormal
+ "2.9802322E-8, 0x0000", // exactly 2^-25: ties-to-even rounds to zero
+ "1.5E-8, 0x0000", // underflows
+ "6.1035156E-5, 0x0400", // smallest normal
+ "6.0E-5, 0x03EF", // falls into the subnormal range
+ "0.1, 0x2E66",
+ "2048.0, 0x6800", // last exactly-representable integer
+ "2049.0, 0x6800", // tie rounds down to even
+ "2050.0, 0x6801",
+ "2051.0, 0x6802", // tie rounds up to even
+ "1.0009765625, 0x3C01", // one ulp above 1.0
+ "1.00048828125, 0x3C00", // half an ulp: ties-to-even rounds down
+ })
+ public void testFloatToHalfBits(float value, String expectedHex) {
+ short expected = (short) Integer.parseInt(expectedHex.substring(2), 16);
+
+ short actual = Float16.floatToHalfBits(value);
+
+ Assertions.assertEquals(
+ String.format("%04X", expected & 0xFFFF),
+ String.format("%04X", actual & 0xFFFF),
+ "narrowing " + value);
+ }
+
+ @Test
+ public void testNaNNarrowsToNaN() {
+ // A NaN whose payload lives entirely in the low mantissa bits would round to a zero
+ // payload, turning it into an Infinity. It must stay a NaN.
+ float sneakyNaN = Float.intBitsToFloat(0x7F800001);
+ Assertions.assertTrue(Float.isNaN(sneakyNaN), "precondition: input is NaN");
+
+ short halfBits = Float16.floatToHalfBits(sneakyNaN);
+
+ Assertions.assertTrue(Float.isNaN(Float16.halfBitsToFloat(halfBits)),
+ "NaN must not narrow to Infinity, got bits 0x" + String.format("%04X", halfBits & 0xFFFF));
+ }
+
+ @Test
+ public void testRoundTripIsIdentityForRepresentableValues() {
+ for (float value : EDGE_CASES) {
+ float actual = Float16.halfBitsToFloat(Float16.floatToHalfBits(value));
+ if (Float.isNaN(value)) {
+ Assertions.assertTrue(Float.isNaN(actual), "NaN round-trip");
+ } else {
+ Assertions.assertEquals(
+ Float.floatToRawIntBits(value), Float.floatToRawIntBits(actual),
+ "round-trip of " + value + " gave " + actual);
+ }
+ }
+ }
+
+ @Test
+ public void testEveryHalfBitPatternRoundTrips() {
+ // Exhaustive over all 65536 patterns: widening then narrowing must be the identity on
+ // bits, except that NaN payloads are only required to stay NaN.
+ for (int bits = 0; bits <= 0xFFFF; bits++) {
+ short halfBits = (short) bits;
+ float widened = Float16.halfBitsToFloat(halfBits);
+ short narrowed = Float16.floatToHalfBits(widened);
+ if (Float.isNaN(widened)) {
+ Assertions.assertTrue(Float.isNaN(Float16.halfBitsToFloat(narrowed)),
+ "NaN pattern 0x" + String.format("%04X", bits) + " stopped being NaN");
+ } else {
+ Assertions.assertEquals(bits, narrowed & 0xFFFF,
+ "pattern 0x" + String.format("%04X", bits) + " did not round-trip");
+ }
+ }
+ }
+
+ // ---------------------------------------------------------------- metadata
+
+ @Test
+ public void testMetadataParsesFloat16() throws Exception {
+ String json = "{\"zarr_format\":3,\"node_type\":\"array\",\"shape\":[4],"
+ + "\"data_type\":\"float16\","
+ + "\"chunk_grid\":{\"name\":\"regular\",\"configuration\":{\"chunk_shape\":[2]}},"
+ + "\"chunk_key_encoding\":{\"name\":\"default\"},\"fill_value\":0,"
+ + "\"codecs\":[{\"name\":\"bytes\",\"configuration\":{\"endian\":\"little\"}}]}";
+
+ dev.zarr.zarrjava.v3.ArrayMetadata metadata = dev.zarr.zarrjava.v3.Node.makeObjectMapper()
+ .readValue(json, dev.zarr.zarrjava.v3.ArrayMetadata.class);
+
+ Assertions.assertEquals(DataType.FLOAT16, metadata.dataType);
+ Assertions.assertEquals(2, metadata.dataType.getByteCount());
+ Assertions.assertEquals(ucar.ma2.DataType.FLOAT, metadata.dataType.getMA2DataType());
+ Assertions.assertTrue(metadata.dataType.isHalfPrecisionFloat());
+ }
+
+ @Test
+ public void testDataTypeSerializesToFloat16() throws Exception {
+ String json = dev.zarr.zarrjava.v3.Node.makeObjectMapper().writeValueAsString(DataType.FLOAT16);
+ Assertions.assertEquals("\"float16\"", json);
+ }
+
+ /**
+ * Fill values, in every form the spec allows for a float. The hex and binary forms carry a
+ * binary16 bit pattern, not a truncated float32.
+ */
+ @ParameterizedTest
+ @CsvSource({
+ "0, 0.0",
+ "1, 1.0",
+ "-2.5, -2.5",
+ "'NaN', NaN",
+ "'+Infinity', Infinity",
+ "'-Infinity', -Infinity",
+ "'0x003C', 1.0", // 0x3C00 little-endian
+ "'0x00C0', -2.0", // 0xC000 little-endian
+ "'0b0000000000111100', 1.0", // same bits, binary form
+ })
+ public void testFillValueForms(String fillValueJson, String expected) throws Exception {
+ String quoted = fillValueJson.startsWith("0x") || fillValueJson.startsWith("0b")
+ || fillValueJson.equals("NaN") || fillValueJson.endsWith("Infinity")
+ ? "\"" + fillValueJson + "\"" : fillValueJson;
+ String json = "{\"zarr_format\":3,\"node_type\":\"array\",\"shape\":[4],"
+ + "\"data_type\":\"float16\","
+ + "\"chunk_grid\":{\"name\":\"regular\",\"configuration\":{\"chunk_shape\":[2]}},"
+ + "\"chunk_key_encoding\":{\"name\":\"default\"},"
+ + "\"fill_value\":" + quoted + ","
+ + "\"codecs\":[{\"name\":\"bytes\",\"configuration\":{\"endian\":\"little\"}}]}";
+
+ dev.zarr.zarrjava.v3.ArrayMetadata metadata = dev.zarr.zarrjava.v3.Node.makeObjectMapper()
+ .readValue(json, dev.zarr.zarrjava.v3.ArrayMetadata.class);
+
+ Assertions.assertEquals(Float.class, metadata.parsedFillValue.getClass(),
+ "float16 fill values are parsed to Float, matching the in-memory type");
+ Assertions.assertEquals(Float.parseFloat(expected), (Float) metadata.parsedFillValue, 1e-9f);
+ }
+
+ @Test
+ public void testUnwrittenChunksReadAsFillValue() throws Exception {
+ StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("float16_fill");
+ Array array = Array.create(storeHandle, Array.metadataBuilder()
+ .withShape(4)
+ .withDataType(DataType.FLOAT16)
+ .withChunkShape(2)
+ .withFillValue(-2.5)
+ .withCodecs(c -> c.withBytes(BytesCodec.Endian.LITTLE))
+ .build());
+
+ ucar.ma2.Array result = array.read();
+
+ Assertions.assertEquals(ucar.ma2.DataType.FLOAT, result.getDataType());
+ for (int i = 0; i < 4; i++) {
+ Assertions.assertEquals(-2.5f, result.getFloat(i), 0.0f, "index " + i);
+ }
+ }
+
+ // ---------------------------------------------------------------- zarr-java round-trip
+
+ @ParameterizedTest
+ @EnumSource(BytesCodec.Endian.class)
+ public void testRoundTripBothEndiannesses(BytesCodec.Endian endian) throws Exception {
+ StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT)
+ .resolve("float16_roundtrip", endian.name());
+
+ Array array = Array.create(storeHandle, Array.metadataBuilder()
+ .withShape(EDGE_CASES.length)
+ .withDataType(DataType.FLOAT16)
+ .withChunkShape(5)
+ .withFillValue(0)
+ .withCodecs(c -> c.withBytes(endian))
+ .build());
+
+ array.write(ucar.ma2.Array.factory(
+ ucar.ma2.DataType.FLOAT, new int[]{EDGE_CASES.length}, EDGE_CASES.clone()));
+
+ ucar.ma2.Array result = Array.open(storeHandle).read();
+
+ assertEdgeCases(result);
+ }
+
+ @Test
+ public void testEncodedChunkIsTwoBytesPerElement() throws Exception {
+ StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("float16_width");
+ Array array = Array.create(storeHandle, Array.metadataBuilder()
+ .withShape(8)
+ .withDataType(DataType.FLOAT16)
+ .withChunkShape(8)
+ .withFillValue(0)
+ .withCodecs(c -> c.withBytes(BytesCodec.Endian.LITTLE))
+ .build());
+
+ array.write(ucar.ma2.Array.factory(ucar.ma2.DataType.FLOAT, new int[]{8},
+ new float[]{1, 2, 3, 4, 5, 6, 7, 8}));
+
+ long chunkSize = storeHandle.resolve("c", "0").getSize();
+ Assertions.assertEquals(16, chunkSize,
+ "8 float16 elements must encode to 16 bytes, not 32");
+ }
+
+ @Test
+ public void testEndiannessActuallySwapsBytes() throws Exception {
+ // 1.0 is 0x3C00, whose bytes differ between orders, so a byte-level check is meaningful.
+ long[] sizes = new long[2];
+ byte[][] firstBytes = new byte[2][];
+ BytesCodec.Endian[] orders = {BytesCodec.Endian.LITTLE, BytesCodec.Endian.BIG};
+
+ for (int i = 0; i < orders.length; i++) {
+ StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT)
+ .resolve("float16_endian", orders[i].name());
+ final BytesCodec.Endian endian = orders[i];
+ Array array = Array.create(storeHandle, Array.metadataBuilder()
+ .withShape(1)
+ .withDataType(DataType.FLOAT16)
+ .withChunkShape(1)
+ .withFillValue(0)
+ .withCodecs(c -> c.withBytes(endian))
+ .build());
+ array.write(ucar.ma2.Array.factory(ucar.ma2.DataType.FLOAT, new int[]{1},
+ new float[]{1.0f}));
+
+ byte[] bytes = dev.zarr.zarrjava.utils.Utils.toArray(
+ storeHandle.resolve("c", "0").readNonNull());
+ sizes[i] = bytes.length;
+ firstBytes[i] = bytes;
+ }
+
+ Assertions.assertArrayEquals(new byte[]{0x00, 0x3C}, firstBytes[0], "little-endian 1.0");
+ Assertions.assertArrayEquals(new byte[]{0x3C, 0x00}, firstBytes[1], "big-endian 1.0");
+ Assertions.assertEquals(2, sizes[0]);
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"blosc", "zstd", "gzip", "sharding", "crc32c"})
+ public void testRoundTripThroughCompressingCodecs(String codec) throws Exception {
+ StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("float16_codec", codec);
+
+ Array array = Array.create(storeHandle, Array.metadataBuilder()
+ .withShape(EDGE_CASES.length)
+ .withDataType(DataType.FLOAT16)
+ .withChunkShape(EDGE_CASES.length)
+ .withFillValue(0)
+ .withCodecs(c -> {
+ switch (codec) {
+ case "blosc":
+ // typesize must follow the encoded width, 2 bytes, not the in-memory 4.
+ return c.withBlosc("zstd", "shuffle", 5);
+ case "zstd":
+ return c.withZstd(3);
+ case "gzip":
+ return c.withGzip(5);
+ case "sharding":
+ return c.withSharding(new int[]{7}, c1 -> c1.withBytes("LITTLE"));
+ case "crc32c":
+ return c.withBytes(BytesCodec.Endian.LITTLE).withCrc32c();
+ default:
+ throw new IllegalArgumentException(codec);
+ }
+ })
+ .build());
+
+ array.write(ucar.ma2.Array.factory(
+ ucar.ma2.DataType.FLOAT, new int[]{EDGE_CASES.length}, EDGE_CASES.clone()));
+
+ assertEdgeCases(Array.open(storeHandle).read());
+ }
+
+ @Test
+ public void testBloscTypesizeIsEncodedWidth() {
+ Assertions.assertEquals(2, DataType.FLOAT16.getByteCount(),
+ "blosc typesize is derived from getByteCount, which must be the encoded width");
+ }
+
+ // ---------------------------------------------------------------- v2
+
+ @Test
+ public void testV2RoundTrip() throws Exception {
+ for (dev.zarr.zarrjava.v2.DataType dt : new dev.zarr.zarrjava.v2.DataType[]{
+ dev.zarr.zarrjava.v2.DataType.FLOAT16, dev.zarr.zarrjava.v2.DataType.FLOAT16_BE}) {
+ StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT)
+ .resolve("float16_v2", dt.name());
+
+ dev.zarr.zarrjava.v2.Array array = dev.zarr.zarrjava.v2.Array.create(
+ storeHandle,
+ dev.zarr.zarrjava.v2.Array.metadataBuilder()
+ .withShape(EDGE_CASES.length)
+ .withChunks(5)
+ .withDataType(dt)
+ .withFillValue(0)
+ .build());
+
+ array.write(ucar.ma2.Array.factory(
+ ucar.ma2.DataType.FLOAT, new int[]{EDGE_CASES.length}, EDGE_CASES.clone()));
+
+ Assertions.assertEquals(2, dt.getByteCount(), dt + " byte count");
+ assertEdgeCases(dev.zarr.zarrjava.v2.Array.open(storeHandle).read());
+ }
+ }
+
+ @Test
+ public void testV2DtypeStrings() {
+ Assertions.assertEquals("