diff --git a/src/main/java/dev/zarr/zarrjava/core/ArrayMetadata.java b/src/main/java/dev/zarr/zarrjava/core/ArrayMetadata.java index a2fb48f3..3c7db297 100644 --- a/src/main/java/dev/zarr/zarrjava/core/ArrayMetadata.java +++ b/src/main/java/dev/zarr/zarrjava/core/ArrayMetadata.java @@ -5,6 +5,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import dev.zarr.zarrjava.ZarrException; import dev.zarr.zarrjava.core.chunkkeyencoding.ChunkKeyEncoding; +import dev.zarr.zarrjava.utils.Float16; import dev.zarr.zarrjava.utils.MultiArrayUtils; import dev.zarr.zarrjava.utils.Utils; import ucar.ma2.Array; @@ -41,6 +42,8 @@ public static Object parseFillValue(Object fillValue, @Nonnull DataType dataType boolean dataTypeIsLong = dataType == dev.zarr.zarrjava.v3.DataType.INT64 || dataType == dev.zarr.zarrjava.v2.DataType.INT64 || dataType == dev.zarr.zarrjava.v2.DataType.INT64_BE || dataType == dev.zarr.zarrjava.v3.DataType.UINT64 || dataType == dev.zarr.zarrjava.v2.DataType.UINT64 || dataType == dev.zarr.zarrjava.v2.DataType.UINT64_BE; boolean dataTypeIsFloat = dataType == dev.zarr.zarrjava.v3.DataType.FLOAT32 || dataType == dev.zarr.zarrjava.v2.DataType.FLOAT32 || dataType == dev.zarr.zarrjava.v2.DataType.FLOAT32_BE; boolean dataTypeIsDouble = dataType == dev.zarr.zarrjava.v3.DataType.FLOAT64 || dataType == dev.zarr.zarrjava.v2.DataType.FLOAT64 || dataType == dev.zarr.zarrjava.v2.DataType.FLOAT64_BE; + // float16 fill values are parsed to float, matching the in-memory representation. + boolean dataTypeIsHalfFloat = dataType.isHalfPrecisionFloat(); if (fillValue instanceof Boolean) { Boolean fillValueBool = (Boolean) fillValue; @@ -60,7 +63,7 @@ public static Object parseFillValue(Object fillValue, @Nonnull DataType dataType return fillValueNumber.intValue(); } else if (dataTypeIsLong) { return fillValueNumber.longValue(); - } else if (dataTypeIsFloat) { + } else if (dataTypeIsFloat || dataTypeIsHalfFloat) { return fillValueNumber.floatValue(); } else if (dataTypeIsDouble) { return fillValueNumber.doubleValue(); @@ -69,7 +72,7 @@ public static Object parseFillValue(Object fillValue, @Nonnull DataType dataType } else if (fillValue instanceof String) { String fillValueString = (String) fillValue; if (fillValueString.equals("NaN")) { - if (dataTypeIsFloat) { + if (dataTypeIsFloat || dataTypeIsHalfFloat) { return Float.NaN; } else if (dataTypeIsDouble) { return Double.NaN; @@ -77,7 +80,7 @@ public static Object parseFillValue(Object fillValue, @Nonnull DataType dataType throw new ZarrException( "Invalid fill value '" + fillValueString + "' for data type '" + dataType + "'."); } else if (fillValueString.equals("+Infinity")) { - if (dataTypeIsFloat) { + if (dataTypeIsFloat || dataTypeIsHalfFloat) { return Float.POSITIVE_INFINITY; } else if (dataTypeIsDouble) { return Double.POSITIVE_INFINITY; @@ -85,7 +88,7 @@ public static Object parseFillValue(Object fillValue, @Nonnull DataType dataType throw new ZarrException( "Invalid fill value '" + fillValueString + "' for data type '" + dataType + "'."); } else if (fillValueString.equals("-Infinity")) { - if (dataTypeIsFloat) { + if (dataTypeIsFloat || dataTypeIsHalfFloat) { return Float.NEGATIVE_INFINITY; } else if (dataTypeIsDouble) { return Double.NEGATIVE_INFINITY; @@ -116,6 +119,9 @@ public static Object parseFillValue(Object fillValue, @Nonnull DataType dataType return buf.get() != 0; } else if (dataTypeIsByte) { return buf.get(); + } else if (dataTypeIsHalfFloat) { + // The 2 bytes are a binary16 bit pattern, not a truncated float32. + return Float16.halfBitsToFloat(buf.getShort()); } else if (dataTypeIsShort) { return buf.getShort(); } else if (dataTypeIsInt) { diff --git a/src/main/java/dev/zarr/zarrjava/core/DataType.java b/src/main/java/dev/zarr/zarrjava/core/DataType.java index d63de069..d7d2ed2e 100644 --- a/src/main/java/dev/zarr/zarrjava/core/DataType.java +++ b/src/main/java/dev/zarr/zarrjava/core/DataType.java @@ -4,4 +4,19 @@ public interface DataType { ucar.ma2.DataType getMA2DataType(); int getByteCount(); + + /** + * Whether elements are stored as IEEE 754 binary16 but held in memory as {@code float}. + * + *

For every other data type the encoded element width ({@link #getByteCount()}) matches the + * width of the in-memory {@link ucar.ma2.DataType}. Half precision is the exception: it encodes + * to 2 bytes but is held as a 4-byte {@code float}, because neither Java nor {@code ucar.ma2} + * has a 16-bit floating point type. The {@code bytes} codec uses this to convert at the + * encode/decode boundary via {@link dev.zarr.zarrjava.utils.Float16}. + * + * @return true for {@code float16}, false for every other data type + */ + default boolean isHalfPrecisionFloat() { + return false; + } } diff --git a/src/main/java/dev/zarr/zarrjava/core/codec/core/BytesCodec.java b/src/main/java/dev/zarr/zarrjava/core/codec/core/BytesCodec.java index 41382b5e..e0233eaa 100644 --- a/src/main/java/dev/zarr/zarrjava/core/codec/core/BytesCodec.java +++ b/src/main/java/dev/zarr/zarrjava/core/codec/core/BytesCodec.java @@ -3,6 +3,7 @@ import com.fasterxml.jackson.annotation.JsonValue; import dev.zarr.zarrjava.ZarrException; import dev.zarr.zarrjava.core.codec.ArrayBytesCodec; +import dev.zarr.zarrjava.utils.Float16; import ucar.ma2.*; import java.nio.ByteBuffer; @@ -31,6 +32,21 @@ public Array decode(ByteBuffer chunkBytes) throws ZarrException { Index index = Index.factory(shape); return Array.factory(DataType.BOOLEAN, index, bools); } + + // float16 encodes to 2 bytes per element but is held as a 4-byte float, so it cannot be + // read straight into an ma2 array; widen each element instead. + if (arrayMetadata.dataType.isHalfPrecisionFloat()) { + int base = chunkBytes.position(); + int size = chunkBytes.remaining() / Short.BYTES; + float[] floats = new float[size]; + for (int i = 0; i < size; i++) { + floats[i] = Float16.halfBitsToFloat(chunkBytes.getShort(base + i * Short.BYTES)); + } + + Index index = Index.factory(shape); + return Array.factory(DataType.FLOAT, index, floats); + } + return Array.factory(dtype, shape, chunkBytes); } @@ -51,6 +67,18 @@ public ByteBuffer encode(Array chunkArray) throws ZarrException { return bb; } + // Float16. Checked before ArrayFloat, because float16 is held in memory as an ArrayFloat + // and must be narrowed to 2 bytes per element rather than written as float32. + if (arrayMetadata.dataType.isHalfPrecisionFloat()) { + float[] data = (float[]) chunkArray.get1DJavaArray(DataType.FLOAT); + ByteBuffer bb = ByteBuffer.allocate(data.length * Short.BYTES).order(order); + for (float f : data) { + bb.putShort(Float16.floatToHalfBits(f)); + } + bb.flip(); + return bb; + } + // Float32 if (chunkArray instanceof ArrayFloat) { float[] data = (float[]) chunkArray.copyTo1DJavaArray(); diff --git a/src/main/java/dev/zarr/zarrjava/utils/Float16.java b/src/main/java/dev/zarr/zarrjava/utils/Float16.java new file mode 100644 index 00000000..9b094128 --- /dev/null +++ b/src/main/java/dev/zarr/zarrjava/utils/Float16.java @@ -0,0 +1,155 @@ +package dev.zarr.zarrjava.utils; + +/** + * Conversions between IEEE 754 binary16 ("half precision") bit patterns and Java {@code float}. + * + *

Zarr's {@code float16} data type stores 2 bytes per element, but there is no 16-bit floating + * point primitive in Java and no half precision type in {@code ucar.ma2.DataType}. Half precision + * arrays are therefore held in memory as {@code float} (binary32) and converted at the codec + * boundary. Widening binary16 to binary32 is always exact; narrowing is not, see + * {@link #floatToHalfBits(float)}. + * + *

{@code Float.float16ToFloat} and {@code Float.floatToFloat16} would do this for us, but they + * were added in Java 20 and this project targets Java 8, so the conversions are implemented here. + * The results match {@code numpy.float16} for all inputs, including subnormals, signed zeros, + * infinities, NaN payloads and round-half-to-even ties. + */ +public final class Float16 { + + /** + * Largest finite binary16 value, {@code 65504.0}. Finite {@code float} values greater than + * {@code MAX_VALUE + 16} round to infinity when narrowed. + */ + public static final float MAX_VALUE = 65504.0f; + + /** Smallest positive normal binary16 value, {@code 2^-14}. */ + public static final float MIN_NORMAL = 6.103515625E-5f; + + /** Smallest positive subnormal binary16 value, {@code 2^-24}. */ + public static final float MIN_VALUE = 5.9604645E-8f; + + private Float16() { + } + + /** + * Widens a binary16 bit pattern to the {@code float} of equal value. Exact for every input: + * every binary16 value is representable as a binary32. + * + *

Signed zeros and infinities are preserved. NaN payloads are shifted into the binary32 + * mantissa, so a NaN stays a NaN rather than becoming an infinity. + * + * @param halfBits the 16 bits of a binary16 value, in the low half of the short + * @return the equal-valued float + */ + public static float halfBitsToFloat(short halfBits) { + final int h = halfBits & 0xFFFF; + final int signBit = (h >>> 15) & 0x1; + final int exponent = (h >>> 10) & 0x1F; + int mantissa = h & 0x3FF; + + final int floatSign = signBit << 31; + + if (exponent == 0) { + if (mantissa == 0) { + // Signed zero. + return Float.intBitsToFloat(floatSign); + } + // Subnormal binary16, which is normal as a binary32: shift the mantissa up until the + // implicit leading bit is set, decrementing the exponent for each shift. + int normalizedExponent = 1; + while ((mantissa & 0x400) == 0) { + mantissa <<= 1; + normalizedExponent--; + } + mantissa &= 0x3FF; // Drop the leading bit; it is implicit in binary32 too. + return Float.intBitsToFloat( + floatSign | ((normalizedExponent - 15 + 127) << 23) | (mantissa << 13)); + } + + if (exponent == 0x1F) { + // Infinity (mantissa 0) or NaN (mantissa non-zero); shifting keeps the payload non-zero. + return Float.intBitsToFloat(floatSign | 0x7F800000 | (mantissa << 13)); + } + + // Normal binary16. + return Float.intBitsToFloat(floatSign | ((exponent - 15 + 127) << 23) | (mantissa << 13)); + } + + /** + * Narrows a {@code float} to the nearest binary16 bit pattern, rounding to nearest with ties to + * even -- the same rounding {@code numpy.float16} and IEEE 754 use. + * + *

This conversion is lossy. binary16 carries 11 bits of significand against binary32's 24, + * and its finite range stops at {@value #MAX_VALUE}: + *

+ * Callers writing data they cannot afford to round should not be using {@code float16}. + * + * @param value the float to narrow + * @return the 16 bits of the nearest binary16 value, in the low half of the short + */ + public static short floatToHalfBits(float value) { + final int f = Float.floatToRawIntBits(value); + final int sign = (f >>> 16) & 0x8000; + final int exponent = (f >>> 23) & 0xFF; + final int mantissa = f & 0x7FFFFF; + + if (exponent == 0xFF) { + if (mantissa == 0) { + return (short) (sign | 0x7C00); // Infinity. + } + // NaN. Keep the top payload bits, but never let the payload round down to zero, which + // would silently turn the NaN into an infinity. + int payload = mantissa >>> 13; + if (payload == 0) { + payload = 1; + } + return (short) (sign | 0x7C00 | payload); + } + + final int unbiasedExponent = exponent - 127; + + if (unbiasedExponent > 15) { + return (short) (sign | 0x7C00); // Overflows the binary16 range. + } + + if (unbiasedExponent >= -14) { + // Normal binary16. Keep the top 10 mantissa bits and round on the remaining 13. + final int halfMantissa = mantissa >>> 13; + final int roundBits = mantissa & 0x1FFF; + final int bits = sign | ((unbiasedExponent + 15) << 10) | halfMantissa; + if (roundBits > 0x1000 || (roundBits == 0x1000 && (halfMantissa & 1) != 0)) { + // A carry out of the mantissa increments the exponent, and a carry out of the + // maximum exponent yields infinity. Both are the correct IEEE 754 results. + return (short) (bits + 1); + } + return (short) bits; + } + + if (unbiasedExponent >= -25) { + // Subnormal binary16, or a value small enough that rounding may still reach the + // smallest subnormal. Restore the implicit leading bit, then shift into place so that + // the result is interpreted as halfMantissa * 2^-24. + final int significand = mantissa | 0x800000; + final int shift = -unbiasedExponent - 1; + final int halfMantissa = significand >>> shift; + final int roundBits = significand & ((1 << shift) - 1); + final int tie = 1 << (shift - 1); + if (roundBits > tie || (roundBits == tie && (halfMantissa & 1) != 0)) { + return (short) (sign | (halfMantissa + 1)); + } + return (short) (sign | halfMantissa); + } + + // Underflows to signed zero: strictly less than half of the smallest subnormal. + return (short) sign; + } +} diff --git a/src/main/java/dev/zarr/zarrjava/v2/DataType.java b/src/main/java/dev/zarr/zarrjava/v2/DataType.java index a0bbd731..973451b1 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/DataType.java +++ b/src/main/java/dev/zarr/zarrjava/v2/DataType.java @@ -12,6 +12,11 @@ public enum DataType implements dev.zarr.zarrjava.core.DataType { UINT16("u2", Endianness.LITTLE), UINT32("u4", Endianness.LITTLE), UINT64("u8", Endianness.LITTLE), + /** + * IEEE 754 binary16. Encoded as 2 bytes, but held in memory as {@code float}; see + * {@link dev.zarr.zarrjava.utils.Float16}. + */ + FLOAT16("f2", Endianness.LITTLE), FLOAT32("f4", Endianness.LITTLE), FLOAT64("f8", Endianness.LITTLE), INT16_BE("i2", Endianness.BIG), @@ -20,6 +25,7 @@ public enum DataType implements dev.zarr.zarrjava.core.DataType { UINT16_BE("u2", Endianness.BIG), UINT32_BE("u4", Endianness.BIG), UINT64_BE("u8", Endianness.BIG), + FLOAT16_BE("f2", Endianness.BIG), FLOAT32_BE("f4", Endianness.BIG), FLOAT64_BE("f8", Endianness.BIG); @@ -68,6 +74,9 @@ public ucar.ma2.DataType getMA2DataType() { case UINT64: case UINT64_BE: return ucar.ma2.DataType.ULONG; + case FLOAT16: + case FLOAT16_BE: + // No half precision type in ucar.ma2; widened to float, see Float16. case FLOAT32: case FLOAT32_BE: return ucar.ma2.DataType.FLOAT; @@ -84,4 +93,9 @@ public int getByteCount() { return Integer.parseInt(dtype.substring(1)); } + @Override + public boolean isHalfPrecisionFloat() { + return this == FLOAT16 || this == FLOAT16_BE; + } + } diff --git a/src/main/java/dev/zarr/zarrjava/v3/DataType.java b/src/main/java/dev/zarr/zarrjava/v3/DataType.java index 159663d5..7919a4e8 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/DataType.java +++ b/src/main/java/dev/zarr/zarrjava/v3/DataType.java @@ -15,6 +15,11 @@ public enum DataType implements dev.zarr.zarrjava.core.DataType { UINT16("uint16", 2), UINT32("uint32", 4), UINT64("uint64", 8), + /** + * IEEE 754 binary16. Encoded as 2 bytes, but held in memory as {@code float}; see + * {@link dev.zarr.zarrjava.utils.Float16}. + */ + FLOAT16("float16", 2), FLOAT32("float32", 4), FLOAT64( "float64", @@ -58,6 +63,8 @@ public ucar.ma2.DataType getMA2DataType() { return ucar.ma2.DataType.UINT; case UINT64: return ucar.ma2.DataType.ULONG; + case FLOAT16: + // No half precision type in ucar.ma2; widened to float, see Float16. case FLOAT32: return ucar.ma2.DataType.FLOAT; case FLOAT64: @@ -66,4 +73,9 @@ public ucar.ma2.DataType getMA2DataType() { throw new IllegalStateException("Unknown DataType: " + this); } } + + @Override + public boolean isHalfPrecisionFloat() { + return this == FLOAT16; + } } diff --git a/src/test/java/dev/zarr/zarrjava/Float16Test.java b/src/test/java/dev/zarr/zarrjava/Float16Test.java new file mode 100644 index 00000000..894937b0 --- /dev/null +++ b/src/test/java/dev/zarr/zarrjava/Float16Test.java @@ -0,0 +1,500 @@ +package dev.zarr.zarrjava; + +import dev.zarr.zarrjava.store.FilesystemStore; +import dev.zarr.zarrjava.store.StoreHandle; +import dev.zarr.zarrjava.utils.Float16; +import dev.zarr.zarrjava.v3.Array; +import dev.zarr.zarrjava.v3.DataType; +import dev.zarr.zarrjava.v3.codec.core.BytesCodec; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.EnumSource; +import org.junit.jupiter.params.provider.ValueSource; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Arrays; + +/** + * Tests for the {@code float16} data type. + * + *

The conversion cases are ground truth generated from {@code numpy.float16} (numpy 2.4.6); + * hex literals are exact bit patterns so the assertions do not depend on decimal parsing. + */ +public class Float16Test extends ZarrTest { + + final static Path PYTHON_TEST_PATH = Paths.get("src/test/python-scripts/"); + + /** + * The subset of the binary16 range that is exactly representable, so round-trips are exact. + * Mirrors VALUES in float16_interop.py. + */ + static final float[] EDGE_CASES = new float[]{ + 0.0f, + -0.0f, + 1.0f, + -2.0f, + 5.960464477539063E-8f, // smallest positive subnormal, 2^-24 + 6.097555160522461E-5f, // largest subnormal + 6.103515625E-5f, // smallest positive normal, 2^-14 + 65504.0f, // largest finite + -65504.0f, + Float.POSITIVE_INFINITY, + Float.NEGATIVE_INFINITY, + Float.NaN, + 0.333251953125f, // 1/3 rounded to binary16 + -0.0999755859375f, // 0.1 rounded to binary16, negated + }; + + static int runPython(String... args) throws IOException, InterruptedException { + String[] command = new String[args.length + 2]; + command[0] = "uv"; + command[1] = "run"; + System.arraycopy(args, 0, command, 2, args.length); + + ProcessBuilder pb = new ProcessBuilder(command); + Process process = pb.start(); + try (BufferedReader out = new BufferedReader(new InputStreamReader(process.getInputStream()))) { + String line; + while ((line = out.readLine()) != null) { + System.out.println(line); + } + } + try (BufferedReader err = new BufferedReader(new InputStreamReader(process.getErrorStream()))) { + String line; + while ((line = err.readLine()) != null) { + System.err.println(line); + } + } + return process.waitFor(); + } + + // ---------------------------------------------------------------- bit conversions + + /** + * Widening binary16 to float. Ground truth: numpy float32 bit patterns. + */ + @ParameterizedTest + @CsvSource({ + "0x0000, 0x00000000", // +0.0 + "0x8000, 0x80000000", // -0.0 + "0x0001, 0x33800000", // smallest subnormal, 2^-24 + "0x8001, 0xB3800000", // negative smallest subnormal + "0x0002, 0x34000000", // subnormal needing 9 normalization shifts + "0x03FF, 0x387FC000", // largest subnormal + "0x0400, 0x38800000", // smallest normal, 2^-14 + "0x3C00, 0x3F800000", // 1.0 + "0xBC00, 0xBF800000", // -1.0 + "0xC000, 0xC0000000", // -2.0 + "0x7BFF, 0x477FE000", // 65504.0, largest finite + "0xFBFF, 0xC77FE000", // -65504.0 + "0x7C00, 0x7F800000", // +Infinity + "0xFC00, 0xFF800000", // -Infinity + "0x7E00, 0x7FC00000", // NaN, payload preserved + "0x3555, 0x3EAAA000", // 1/3 rounded + }) + public void testHalfBitsToFloat(String halfHex, String floatHex) { + short halfBits = (short) Integer.parseInt(halfHex.substring(2), 16); + int expectedBits = (int) Long.parseLong(floatHex.substring(2), 16); + + float actual = Float16.halfBitsToFloat(halfBits); + + Assertions.assertEquals( + String.format("%08X", expectedBits), + String.format("%08X", Float.floatToRawIntBits(actual)), + "widening " + halfHex + " gave " + actual); + } + + /** + * Narrowing float to binary16, including round-half-to-even ties and overflow. Ground truth: + * numpy float16 bit patterns. + */ + @ParameterizedTest + @CsvSource({ + "1.0, 0x3C00", + "-2.0, 0xC000", + "0.0, 0x0000", + "-0.0, 0x8000", // signed zero survives + "65504.0, 0x7BFF", // largest finite + "65519.0, 0x7BFF", // just below the overflow midpoint + "65520.0, 0x7C00", // exactly the midpoint: ties-to-even overflows to Infinity + "65536.0, 0x7C00", + "1.0E8, 0x7C00", // far overflow + "-1.0E8, 0xFC00", + "5.9604645E-8, 0x0001", // smallest subnormal, exact + "3.0E-8, 0x0001", // rounds up to the smallest subnormal + "2.9802322E-8, 0x0000", // exactly 2^-25: ties-to-even rounds to zero + "1.5E-8, 0x0000", // underflows + "6.1035156E-5, 0x0400", // smallest normal + "6.0E-5, 0x03EF", // falls into the subnormal range + "0.1, 0x2E66", + "2048.0, 0x6800", // last exactly-representable integer + "2049.0, 0x6800", // tie rounds down to even + "2050.0, 0x6801", + "2051.0, 0x6802", // tie rounds up to even + "1.0009765625, 0x3C01", // one ulp above 1.0 + "1.00048828125, 0x3C00", // half an ulp: ties-to-even rounds down + }) + public void testFloatToHalfBits(float value, String expectedHex) { + short expected = (short) Integer.parseInt(expectedHex.substring(2), 16); + + short actual = Float16.floatToHalfBits(value); + + Assertions.assertEquals( + String.format("%04X", expected & 0xFFFF), + String.format("%04X", actual & 0xFFFF), + "narrowing " + value); + } + + @Test + public void testNaNNarrowsToNaN() { + // A NaN whose payload lives entirely in the low mantissa bits would round to a zero + // payload, turning it into an Infinity. It must stay a NaN. + float sneakyNaN = Float.intBitsToFloat(0x7F800001); + Assertions.assertTrue(Float.isNaN(sneakyNaN), "precondition: input is NaN"); + + short halfBits = Float16.floatToHalfBits(sneakyNaN); + + Assertions.assertTrue(Float.isNaN(Float16.halfBitsToFloat(halfBits)), + "NaN must not narrow to Infinity, got bits 0x" + String.format("%04X", halfBits & 0xFFFF)); + } + + @Test + public void testRoundTripIsIdentityForRepresentableValues() { + for (float value : EDGE_CASES) { + float actual = Float16.halfBitsToFloat(Float16.floatToHalfBits(value)); + if (Float.isNaN(value)) { + Assertions.assertTrue(Float.isNaN(actual), "NaN round-trip"); + } else { + Assertions.assertEquals( + Float.floatToRawIntBits(value), Float.floatToRawIntBits(actual), + "round-trip of " + value + " gave " + actual); + } + } + } + + @Test + public void testEveryHalfBitPatternRoundTrips() { + // Exhaustive over all 65536 patterns: widening then narrowing must be the identity on + // bits, except that NaN payloads are only required to stay NaN. + for (int bits = 0; bits <= 0xFFFF; bits++) { + short halfBits = (short) bits; + float widened = Float16.halfBitsToFloat(halfBits); + short narrowed = Float16.floatToHalfBits(widened); + if (Float.isNaN(widened)) { + Assertions.assertTrue(Float.isNaN(Float16.halfBitsToFloat(narrowed)), + "NaN pattern 0x" + String.format("%04X", bits) + " stopped being NaN"); + } else { + Assertions.assertEquals(bits, narrowed & 0xFFFF, + "pattern 0x" + String.format("%04X", bits) + " did not round-trip"); + } + } + } + + // ---------------------------------------------------------------- metadata + + @Test + public void testMetadataParsesFloat16() throws Exception { + String json = "{\"zarr_format\":3,\"node_type\":\"array\",\"shape\":[4]," + + "\"data_type\":\"float16\"," + + "\"chunk_grid\":{\"name\":\"regular\",\"configuration\":{\"chunk_shape\":[2]}}," + + "\"chunk_key_encoding\":{\"name\":\"default\"},\"fill_value\":0," + + "\"codecs\":[{\"name\":\"bytes\",\"configuration\":{\"endian\":\"little\"}}]}"; + + dev.zarr.zarrjava.v3.ArrayMetadata metadata = dev.zarr.zarrjava.v3.Node.makeObjectMapper() + .readValue(json, dev.zarr.zarrjava.v3.ArrayMetadata.class); + + Assertions.assertEquals(DataType.FLOAT16, metadata.dataType); + Assertions.assertEquals(2, metadata.dataType.getByteCount()); + Assertions.assertEquals(ucar.ma2.DataType.FLOAT, metadata.dataType.getMA2DataType()); + Assertions.assertTrue(metadata.dataType.isHalfPrecisionFloat()); + } + + @Test + public void testDataTypeSerializesToFloat16() throws Exception { + String json = dev.zarr.zarrjava.v3.Node.makeObjectMapper().writeValueAsString(DataType.FLOAT16); + Assertions.assertEquals("\"float16\"", json); + } + + /** + * Fill values, in every form the spec allows for a float. The hex and binary forms carry a + * binary16 bit pattern, not a truncated float32. + */ + @ParameterizedTest + @CsvSource({ + "0, 0.0", + "1, 1.0", + "-2.5, -2.5", + "'NaN', NaN", + "'+Infinity', Infinity", + "'-Infinity', -Infinity", + "'0x003C', 1.0", // 0x3C00 little-endian + "'0x00C0', -2.0", // 0xC000 little-endian + "'0b0000000000111100', 1.0", // same bits, binary form + }) + public void testFillValueForms(String fillValueJson, String expected) throws Exception { + String quoted = fillValueJson.startsWith("0x") || fillValueJson.startsWith("0b") + || fillValueJson.equals("NaN") || fillValueJson.endsWith("Infinity") + ? "\"" + fillValueJson + "\"" : fillValueJson; + String json = "{\"zarr_format\":3,\"node_type\":\"array\",\"shape\":[4]," + + "\"data_type\":\"float16\"," + + "\"chunk_grid\":{\"name\":\"regular\",\"configuration\":{\"chunk_shape\":[2]}}," + + "\"chunk_key_encoding\":{\"name\":\"default\"}," + + "\"fill_value\":" + quoted + "," + + "\"codecs\":[{\"name\":\"bytes\",\"configuration\":{\"endian\":\"little\"}}]}"; + + dev.zarr.zarrjava.v3.ArrayMetadata metadata = dev.zarr.zarrjava.v3.Node.makeObjectMapper() + .readValue(json, dev.zarr.zarrjava.v3.ArrayMetadata.class); + + Assertions.assertEquals(Float.class, metadata.parsedFillValue.getClass(), + "float16 fill values are parsed to Float, matching the in-memory type"); + Assertions.assertEquals(Float.parseFloat(expected), (Float) metadata.parsedFillValue, 1e-9f); + } + + @Test + public void testUnwrittenChunksReadAsFillValue() throws Exception { + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("float16_fill"); + Array array = Array.create(storeHandle, Array.metadataBuilder() + .withShape(4) + .withDataType(DataType.FLOAT16) + .withChunkShape(2) + .withFillValue(-2.5) + .withCodecs(c -> c.withBytes(BytesCodec.Endian.LITTLE)) + .build()); + + ucar.ma2.Array result = array.read(); + + Assertions.assertEquals(ucar.ma2.DataType.FLOAT, result.getDataType()); + for (int i = 0; i < 4; i++) { + Assertions.assertEquals(-2.5f, result.getFloat(i), 0.0f, "index " + i); + } + } + + // ---------------------------------------------------------------- zarr-java round-trip + + @ParameterizedTest + @EnumSource(BytesCodec.Endian.class) + public void testRoundTripBothEndiannesses(BytesCodec.Endian endian) throws Exception { + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT) + .resolve("float16_roundtrip", endian.name()); + + Array array = Array.create(storeHandle, Array.metadataBuilder() + .withShape(EDGE_CASES.length) + .withDataType(DataType.FLOAT16) + .withChunkShape(5) + .withFillValue(0) + .withCodecs(c -> c.withBytes(endian)) + .build()); + + array.write(ucar.ma2.Array.factory( + ucar.ma2.DataType.FLOAT, new int[]{EDGE_CASES.length}, EDGE_CASES.clone())); + + ucar.ma2.Array result = Array.open(storeHandle).read(); + + assertEdgeCases(result); + } + + @Test + public void testEncodedChunkIsTwoBytesPerElement() throws Exception { + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("float16_width"); + Array array = Array.create(storeHandle, Array.metadataBuilder() + .withShape(8) + .withDataType(DataType.FLOAT16) + .withChunkShape(8) + .withFillValue(0) + .withCodecs(c -> c.withBytes(BytesCodec.Endian.LITTLE)) + .build()); + + array.write(ucar.ma2.Array.factory(ucar.ma2.DataType.FLOAT, new int[]{8}, + new float[]{1, 2, 3, 4, 5, 6, 7, 8})); + + long chunkSize = storeHandle.resolve("c", "0").getSize(); + Assertions.assertEquals(16, chunkSize, + "8 float16 elements must encode to 16 bytes, not 32"); + } + + @Test + public void testEndiannessActuallySwapsBytes() throws Exception { + // 1.0 is 0x3C00, whose bytes differ between orders, so a byte-level check is meaningful. + long[] sizes = new long[2]; + byte[][] firstBytes = new byte[2][]; + BytesCodec.Endian[] orders = {BytesCodec.Endian.LITTLE, BytesCodec.Endian.BIG}; + + for (int i = 0; i < orders.length; i++) { + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT) + .resolve("float16_endian", orders[i].name()); + final BytesCodec.Endian endian = orders[i]; + Array array = Array.create(storeHandle, Array.metadataBuilder() + .withShape(1) + .withDataType(DataType.FLOAT16) + .withChunkShape(1) + .withFillValue(0) + .withCodecs(c -> c.withBytes(endian)) + .build()); + array.write(ucar.ma2.Array.factory(ucar.ma2.DataType.FLOAT, new int[]{1}, + new float[]{1.0f})); + + byte[] bytes = dev.zarr.zarrjava.utils.Utils.toArray( + storeHandle.resolve("c", "0").readNonNull()); + sizes[i] = bytes.length; + firstBytes[i] = bytes; + } + + Assertions.assertArrayEquals(new byte[]{0x00, 0x3C}, firstBytes[0], "little-endian 1.0"); + Assertions.assertArrayEquals(new byte[]{0x3C, 0x00}, firstBytes[1], "big-endian 1.0"); + Assertions.assertEquals(2, sizes[0]); + } + + @ParameterizedTest + @ValueSource(strings = {"blosc", "zstd", "gzip", "sharding", "crc32c"}) + public void testRoundTripThroughCompressingCodecs(String codec) throws Exception { + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("float16_codec", codec); + + Array array = Array.create(storeHandle, Array.metadataBuilder() + .withShape(EDGE_CASES.length) + .withDataType(DataType.FLOAT16) + .withChunkShape(EDGE_CASES.length) + .withFillValue(0) + .withCodecs(c -> { + switch (codec) { + case "blosc": + // typesize must follow the encoded width, 2 bytes, not the in-memory 4. + return c.withBlosc("zstd", "shuffle", 5); + case "zstd": + return c.withZstd(3); + case "gzip": + return c.withGzip(5); + case "sharding": + return c.withSharding(new int[]{7}, c1 -> c1.withBytes("LITTLE")); + case "crc32c": + return c.withBytes(BytesCodec.Endian.LITTLE).withCrc32c(); + default: + throw new IllegalArgumentException(codec); + } + }) + .build()); + + array.write(ucar.ma2.Array.factory( + ucar.ma2.DataType.FLOAT, new int[]{EDGE_CASES.length}, EDGE_CASES.clone())); + + assertEdgeCases(Array.open(storeHandle).read()); + } + + @Test + public void testBloscTypesizeIsEncodedWidth() { + Assertions.assertEquals(2, DataType.FLOAT16.getByteCount(), + "blosc typesize is derived from getByteCount, which must be the encoded width"); + } + + // ---------------------------------------------------------------- v2 + + @Test + public void testV2RoundTrip() throws Exception { + for (dev.zarr.zarrjava.v2.DataType dt : new dev.zarr.zarrjava.v2.DataType[]{ + dev.zarr.zarrjava.v2.DataType.FLOAT16, dev.zarr.zarrjava.v2.DataType.FLOAT16_BE}) { + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT) + .resolve("float16_v2", dt.name()); + + dev.zarr.zarrjava.v2.Array array = dev.zarr.zarrjava.v2.Array.create( + storeHandle, + dev.zarr.zarrjava.v2.Array.metadataBuilder() + .withShape(EDGE_CASES.length) + .withChunks(5) + .withDataType(dt) + .withFillValue(0) + .build()); + + array.write(ucar.ma2.Array.factory( + ucar.ma2.DataType.FLOAT, new int[]{EDGE_CASES.length}, EDGE_CASES.clone())); + + Assertions.assertEquals(2, dt.getByteCount(), dt + " byte count"); + assertEdgeCases(dev.zarr.zarrjava.v2.Array.open(storeHandle).read()); + } + } + + @Test + public void testV2DtypeStrings() { + Assertions.assertEquals("f2", dev.zarr.zarrjava.v2.DataType.FLOAT16_BE.getValue()); + } + + // ---------------------------------------------------------------- zarr-python interop + + @ParameterizedTest + @CsvSource({"3, LITTLE", "3, BIG", "2, LITTLE", "2, BIG"}) + public void testPythonWritesJavaReads(int zarrFormat, String endian) throws Exception { + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT) + .resolve("float16_py2java", "v" + zarrFormat, endian); + + int exit = runPython(PYTHON_TEST_PATH.resolve("float16_interop.py").toString(), + "write", storeHandle.toPath().toString(), String.valueOf(zarrFormat), endian); + Assertions.assertEquals(0, exit, "zarr-python writer failed"); + + ucar.ma2.Array result = zarrFormat == 3 + ? Array.open(storeHandle).read() + : dev.zarr.zarrjava.v2.Array.open(storeHandle).read(); + + assertEdgeCases(result); + } + + @ParameterizedTest + @CsvSource({"3, LITTLE", "3, BIG", "2, LITTLE", "2, BIG"}) + public void testJavaWritesPythonReads(int zarrFormat, String endian) throws Exception { + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT) + .resolve("float16_java2py", "v" + zarrFormat, endian); + + if (zarrFormat == 3) { + Array.create(storeHandle, Array.metadataBuilder() + .withShape(EDGE_CASES.length) + .withDataType(DataType.FLOAT16) + .withChunkShape(5) + .withFillValue(0) + .withCodecs(c -> c.withBytes(BytesCodec.Endian.valueOf(endian))) + .build()) + .write(ucar.ma2.Array.factory(ucar.ma2.DataType.FLOAT, + new int[]{EDGE_CASES.length}, EDGE_CASES.clone())); + } else { + dev.zarr.zarrjava.v2.DataType dt = endian.equals("BIG") + ? dev.zarr.zarrjava.v2.DataType.FLOAT16_BE + : dev.zarr.zarrjava.v2.DataType.FLOAT16; + dev.zarr.zarrjava.v2.Array.create(storeHandle, + dev.zarr.zarrjava.v2.Array.metadataBuilder() + .withShape(EDGE_CASES.length) + .withChunks(5) + .withDataType(dt) + .withFillValue(0) + .build()) + .write(ucar.ma2.Array.factory(ucar.ma2.DataType.FLOAT, + new int[]{EDGE_CASES.length}, EDGE_CASES.clone())); + } + + int exit = runPython(PYTHON_TEST_PATH.resolve("float16_interop.py").toString(), + "verify", storeHandle.toPath().toString(), String.valueOf(zarrFormat)); + Assertions.assertEquals(0, exit, "zarr-python could not read what zarr-java wrote"); + } + + // ---------------------------------------------------------------- helper + + private static void assertEdgeCases(ucar.ma2.Array result) { + Assertions.assertEquals(EDGE_CASES.length, result.getSize(), "element count"); + for (int i = 0; i < EDGE_CASES.length; i++) { + float expected = EDGE_CASES[i]; + float actual = result.getFloat(i); + if (Float.isNaN(expected)) { + Assertions.assertTrue(Float.isNaN(actual), "index " + i + " should be NaN"); + } else { + // Raw bits, so that +0.0 and -0.0 are told apart. + Assertions.assertEquals( + Float.floatToRawIntBits(expected), Float.floatToRawIntBits(actual), + "index " + i + ": expected " + expected + " got " + actual + + " (all: " + Arrays.toString( + (float[]) result.get1DJavaArray(ucar.ma2.DataType.FLOAT)) + ")"); + } + } + } +} diff --git a/src/test/java/dev/zarr/zarrjava/ZarrTest.java b/src/test/java/dev/zarr/zarrjava/ZarrTest.java index fdb8ec17..c8fd44e0 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrTest.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrTest.java @@ -46,6 +46,7 @@ static Stream dataTypeProviderV3() { DataType.UINT32, DataType.INT64, DataType.UINT64, + DataType.FLOAT16, DataType.FLOAT32, DataType.FLOAT64 ); @@ -62,6 +63,7 @@ static Stream dataTypeProviderV2() { dev.zarr.zarrjava.v2.DataType.UINT32, dev.zarr.zarrjava.v2.DataType.INT64, dev.zarr.zarrjava.v2.DataType.UINT64, + dev.zarr.zarrjava.v2.DataType.FLOAT16, dev.zarr.zarrjava.v2.DataType.FLOAT32, dev.zarr.zarrjava.v2.DataType.FLOAT64, dev.zarr.zarrjava.v2.DataType.UINT16_BE, @@ -70,6 +72,7 @@ static Stream dataTypeProviderV2() { dev.zarr.zarrjava.v2.DataType.INT16_BE, dev.zarr.zarrjava.v2.DataType.INT32_BE, dev.zarr.zarrjava.v2.DataType.INT64_BE, + dev.zarr.zarrjava.v2.DataType.FLOAT16_BE, dev.zarr.zarrjava.v2.DataType.FLOAT32_BE, dev.zarr.zarrjava.v2.DataType.FLOAT64_BE ); @@ -154,6 +157,21 @@ protected void assertContainsTestAttributes(Attributes attributes) throws ZarrEx } + /** + * Quantizes a test value the way the data type would store it. Only float16 loses anything: it + * carries 11 significand bits, so 1024 of the 4096 values below are not exactly representable + * (2049 stores as 2048, 2051 as 2052, 4095 as 4096). Pre-quantizing here keeps + * {@link #testdata} and {@link #assertIsTestdata} in agreement, and matches + * {@code np.arange(16 * 16 * 16, dtype='float16')} element for element. + */ + private static float quantize(dev.zarr.zarrjava.core.DataType dt, int i) { + if (dt.isHalfPrecisionFloat()) { + return dev.zarr.zarrjava.utils.Float16.halfBitsToFloat( + dev.zarr.zarrjava.utils.Float16.floatToHalfBits((float) i)); + } + return (float) i; + } + protected ucar.ma2.Array testdata(dev.zarr.zarrjava.core.DataType dt) { ucar.ma2.DataType ma2Type = dt.getMA2DataType(); ucar.ma2.Array array = ucar.ma2.Array.factory(ma2Type, new int[]{16, 16, 16}); @@ -181,7 +199,7 @@ protected ucar.ma2.Array testdata(dev.zarr.zarrjava.core.DataType dt) { array.setLong(i, i); break; case FLOAT: - array.setFloat(i, (float) i); + array.setFloat(i, quantize(dt, i)); break; case DOUBLE: array.setDouble(i, i); @@ -220,7 +238,7 @@ protected void assertIsTestdata(ucar.ma2.Array result, dev.zarr.zarrjava.core.Da Assertions.assertEquals(i, result.getLong(i)); break; case FLOAT: - Assertions.assertEquals((float) i, result.getFloat(i), 1e-6); + Assertions.assertEquals(quantize(dt, i), result.getFloat(i), 1e-6); break; case DOUBLE: Assertions.assertEquals(i, result.getDouble(i), 1e-12); diff --git a/src/test/python-scripts/float16_interop.py b/src/test/python-scripts/float16_interop.py new file mode 100644 index 00000000..dfd8bede --- /dev/null +++ b/src/test/python-scripts/float16_interop.py @@ -0,0 +1,106 @@ +"""Cross-implementation check for the float16 data type. + +Usage: + float16_interop.py write + float16_interop.py verify + +`write` produces a float16 array for zarr-java to read; `verify` reads an array +zarr-java produced and asserts it holds the same values. Both directions are needed: +a zarr-java round-trip cannot catch a symmetric encode/decode bug. + +Every value below is exactly representable in binary16, so the comparison is exact +and never depends on rounding. +""" + +import sys + +import numpy as np +import zarr +from zarr.storage import LocalStore + +# Ordered edge cases: signed zeros, min/max subnormal, min normal, max finite, +# infinities, NaN, and a few rounded values. +VALUES = np.array( + [ + 0.0, + -0.0, + 1.0, + -2.0, + 5.960464477539063e-08, # smallest positive subnormal, 2**-24 + 6.097555160522461e-05, # largest subnormal + 6.103515625e-05, # smallest positive normal, 2**-14 + 65504.0, # largest finite + -65504.0, + np.inf, + -np.inf, + np.nan, + 0.333251953125, # 1/3 rounded to binary16 + -0.0999755859375, # 0.1 rounded to binary16, negated + ], + dtype=np.float16, +) + +SHAPE = (len(VALUES),) +CHUNKS = (5,) # Deliberately not a divisor of 14, so the last chunk is partial. + + +def assert_matches(actual): + actual = np.asarray(actual, dtype=np.float16) + assert actual.shape == SHAPE, f"shape {actual.shape} != {SHAPE}" + for i, (got, want) in enumerate(zip(actual, VALUES)): + if np.isnan(want): + assert np.isnan(got), f"index {i}: got {got!r}, expected NaN" + else: + # Exact bit comparison, so that +0.0 and -0.0 are told apart. + got_bits = np.frombuffer(np.float16(got).tobytes(), dtype=np.uint16)[0] + want_bits = np.frombuffer(np.float16(want).tobytes(), dtype=np.uint16)[0] + assert got_bits == want_bits, ( + f"index {i}: got {got!r} (0x{got_bits:04X}), " + f"expected {want!r} (0x{want_bits:04X})" + ) + + +def main(): + mode = sys.argv[1] + store_path = sys.argv[2] + zarr_format = int(sys.argv[3]) + + if mode == "write": + endian = sys.argv[4] + dtype = ">f2" if endian == "BIG" else "f2", which is not == to native "float16". + assert a.dtype.kind == "f" and a.dtype.itemsize == 2, ( + f"dtype is {a.dtype}, expected a 2-byte float" + ) + assert_matches(a[:]) + print(f"verified float16 v{zarr_format} at {store_path}") + + else: + raise SystemExit(f"unknown mode {mode!r}") + + +if __name__ == "__main__": + main()