diff --git a/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadata.java b/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadata.java index 58d5733..dbabf86 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadata.java +++ b/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadata.java @@ -1,5 +1,7 @@ package dev.zarr.zarrjava.v3; +import com.fasterxml.jackson.annotation.JsonAnyGetter; +import com.fasterxml.jackson.annotation.JsonAnySetter; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; @@ -48,6 +50,14 @@ public final class ArrayMetadata extends dev.zarr.zarrjava.core.ArrayMetadata { @JsonProperty("storage_transformers") public final Map[] storageTransformers; + /** + * Members of the metadata document that zarr-java does not know about. The Zarr v3 specification + * requires that these are ignored when they declare {@code "must_understand": false}, and that + * they are rejected otherwise. They are kept here so that rewriting the metadata does not drop + * extensions written by another implementation. + */ + private final Map extraFields; + @JsonIgnore public CoreArrayMetadata coreArrayMetadata; @@ -65,6 +75,39 @@ public ArrayMetadata( ); } + public ArrayMetadata( + long[] shape, DataType dataType, ChunkGrid chunkGrid, ChunkKeyEncoding chunkKeyEncoding, + Object fillValue, + @Nonnull Codec[] codecs, + @Nullable String[] dimensionNames, + @Nullable Attributes attributes, + @Nullable Map[] storageTransformers, + @Nullable Map extraFields + ) throws ZarrException { + this(ZARR_FORMAT, NODE_TYPE, shape, dataType, chunkGrid, chunkKeyEncoding, fillValue, codecs, + dimensionNames, + attributes, storageTransformers, extraFields + ); + } + + public ArrayMetadata( + int zarrFormat, + String nodeType, + long[] shape, + DataType dataType, + ChunkGrid chunkGrid, + ChunkKeyEncoding chunkKeyEncoding, + Object fillValue, + @Nonnull Codec[] codecs, + @Nullable String[] dimensionNames, + @Nullable Attributes attributes, + @Nullable Map[] storageTransformers + ) throws ZarrException { + this(zarrFormat, nodeType, shape, dataType, chunkGrid, chunkKeyEncoding, fillValue, codecs, + dimensionNames, attributes, storageTransformers, null + ); + } + @JsonCreator(mode = JsonCreator.Mode.PROPERTIES) public ArrayMetadata( @JsonProperty(value = "zarr_format", required = true) int zarrFormat, @@ -77,9 +120,11 @@ public ArrayMetadata( @Nonnull @JsonProperty(value = "codecs") Codec[] codecs, @Nullable @JsonProperty(value = "dimension_names") String[] dimensionNames, @Nullable @JsonProperty(value = "attributes") Attributes attributes, - @Nullable @JsonProperty(value = "storage_transformers") Map[] storageTransformers + @Nullable @JsonProperty(value = "storage_transformers") Map[] storageTransformers, + @Nullable @JsonAnySetter Map extraFields ) throws ZarrException { super(shape, fillValue, dataType); + this.extraFields = ExtraFields.validated(extraFields, ExtraFields.ARRAY_METADATA_KEYS); if (zarrFormat != this.zarrFormat) { throw new ZarrException( "Expected zarr format '" + this.zarrFormat + "', got '" + zarrFormat + "'."); @@ -129,6 +174,18 @@ public ArrayMetadata( this.storageTransformers = storageTransformers; } + /** + * The members of the metadata document that zarr-java does not know about, but that declared + * {@code "must_understand": false} and could therefore be ignored. They are written back out + * unchanged, so that extensions written by another implementation survive a metadata rewrite. + * + * @return the extra fields, never {@code null} + */ + @JsonAnyGetter + public Map extraFields() { + return extraFields; + } + public static Optional getShardingIndexedCodec(Codec[] codecs) { return Arrays.stream(codecs).filter(codec -> codec instanceof ShardingIndexedCodec).findFirst(); } diff --git a/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadataBuilder.java b/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadataBuilder.java index 5d5f494..fd6b5c3 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadataBuilder.java +++ b/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadataBuilder.java @@ -31,6 +31,7 @@ public class ArrayMetadataBuilder { Attributes attributes = new Attributes(); Map[] storageTransformers = new HashMap[]{}; String[] dimensionNames = null; + Map extraFields = null; protected ArrayMetadataBuilder() { } @@ -49,6 +50,7 @@ protected static ArrayMetadataBuilder fromArrayMetadata(ArrayMetadata arrayMetad builder.codecs = arrayMetadata.codecs; builder.dimensionNames = arrayMetadata.dimensionNames; builder.storageTransformers = arrayMetadata.storageTransformers; + builder.extraFields = arrayMetadata.extraFields(); if (withAttributes) { builder.attributes = arrayMetadata.attributes; } @@ -155,6 +157,17 @@ public ArrayMetadataBuilder withStorageTransformers(Map[] storag return this; } + /** + * Sets members of the metadata document that zarr-java itself does not interpret. Every value + * needs to be a map carrying {@code "must_understand": false}, otherwise {@link #build()} fails. + * + * @param extraFields the extra fields to write into the metadata document + */ + public ArrayMetadataBuilder withExtraFields(Map extraFields) { + this.extraFields = extraFields; + return this; + } + public ArrayMetadata build() throws ZarrException { if (shape == null) { throw new ZarrException("Shape needs to be provided. Please call `.withShape`."); @@ -172,7 +185,8 @@ public ArrayMetadata build() throws ZarrException { return new ArrayMetadata(shape, dataType, chunkGrid, chunkKeyEncoding, fillValue, codecs, dimensionNames, attributes, - storageTransformers + storageTransformers, + extraFields ); } } diff --git a/src/main/java/dev/zarr/zarrjava/v3/ExtraFields.java b/src/main/java/dev/zarr/zarrjava/v3/ExtraFields.java new file mode 100644 index 0000000..6c1b843 --- /dev/null +++ b/src/main/java/dev/zarr/zarrjava/v3/ExtraFields.java @@ -0,0 +1,104 @@ +package dev.zarr.zarrjava.v3; + +import dev.zarr.zarrjava.ZarrException; + +import javax.annotation.Nullable; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * Handling of unknown ("extra") members of a Zarr v3 metadata document. + * + *

The Zarr v3 specification allows a writer to add members that a reader may not know about. Such + * a member has to be a JSON object carrying {@code "must_understand": false}, which declares that a + * reader that does not know the member may safely ignore it. Any other unknown member has to be + * rejected, because it may change how the array is to be interpreted. + * + *

These semantics match zarr-python 3.1.6, see {@code zarr/core/metadata/v3.py}. + */ +final class ExtraFields { + + static final String MUST_UNDERSTAND = "must_understand"; + + /** + * Members of a v3 array metadata document that zarr-java knows about. Anything else read from a + * {@code zarr.json} array document is an extra field. + */ + static final Set ARRAY_METADATA_KEYS = unmodifiableSetOf( + "zarr_format", "node_type", "shape", "data_type", "chunk_grid", "chunk_key_encoding", + "fill_value", "codecs", "attributes", "dimension_names", "storage_transformers" + ); + + /** + * Members of a v3 group metadata document that zarr-java knows about. Anything else read from a + * {@code zarr.json} group document is an extra field. + */ + static final Set GROUP_METADATA_KEYS = unmodifiableSetOf( + "zarr_format", "node_type", "attributes", "consolidated_metadata" + ); + + private ExtraFields() { + } + + private static Set unmodifiableSetOf(String... keys) { + return Collections.unmodifiableSet(new HashSet<>(Arrays.asList(keys))); + } + + /** + * Whether an unknown metadata member may be ignored, i.e. whether it is a JSON object with a + * {@code must_understand} member that is set to {@code false}. + */ + static boolean isIgnorable(@Nullable Object value) { + return value instanceof Map + && Boolean.FALSE.equals(((Map) value).get(MUST_UNDERSTAND)); + } + + /** + * Validates unknown members of a metadata document and returns them so that they can be written + * back out unchanged. + * + * @param extraFields the unknown members, may be {@code null} + * @param reservedKeys the members that the metadata document defines itself, either + * {@link #ARRAY_METADATA_KEYS} or {@link #GROUP_METADATA_KEYS} + * @return the extra fields, never {@code null} + * @throws ZarrException if a member collides with a reserved key, or if a member may not be + * ignored because it is not a JSON object carrying + * {@code "must_understand": false} + */ + static Map validated( + @Nullable Map extraFields, Set reservedKeys + ) throws ZarrException { + if (extraFields == null || extraFields.isEmpty()) { + return Collections.emptyMap(); + } + final List reservedCollisions = new ArrayList<>(); + final List notIgnorable = new ArrayList<>(); + for (Map.Entry entry : extraFields.entrySet()) { + if (reservedKeys.contains(entry.getKey())) { + reservedCollisions.add(entry.getKey()); + } else if (!isIgnorable(entry.getValue())) { + notIgnorable.add(entry.getKey()); + } + } + if (!reservedCollisions.isEmpty()) { + Collections.sort(reservedCollisions); + throw new ZarrException( + "Invalid extra fields. The following keys: " + reservedCollisions + " are invalid " + + "because they collide with keys reserved for use by the metadata document."); + } + if (!notIgnorable.isEmpty()) { + Collections.sort(notIgnorable); + throw new ZarrException( + "Got a Zarr v3 metadata document with the following disallowed extra fields: " + + notIgnorable + ". Extra fields are not allowed unless they are a JSON object " + + "with a \"" + MUST_UNDERSTAND + "\" key which is assigned the value `false`."); + } + return Collections.unmodifiableMap(new LinkedHashMap<>(extraFields)); + } +} diff --git a/src/main/java/dev/zarr/zarrjava/v3/Group.java b/src/main/java/dev/zarr/zarrjava/v3/Group.java index 8b1a81b..6a3547f 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/Group.java +++ b/src/main/java/dev/zarr/zarrjava/v3/Group.java @@ -302,7 +302,8 @@ public Group updateAttributes(Function attributeMapper) * @throws IOException if the metadata cannot be serialized */ public Group setAttributes(Attributes newAttributes) throws ZarrException, IOException { - GroupMetadata newGroupMetadata = new GroupMetadata(newAttributes); + GroupMetadata newGroupMetadata = + new GroupMetadata(newAttributes, metadata.extraFields()); return writeMetadata(newGroupMetadata); } diff --git a/src/main/java/dev/zarr/zarrjava/v3/GroupMetadata.java b/src/main/java/dev/zarr/zarrjava/v3/GroupMetadata.java index 56414a2..6283bc6 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/GroupMetadata.java +++ b/src/main/java/dev/zarr/zarrjava/v3/GroupMetadata.java @@ -1,5 +1,7 @@ package dev.zarr.zarrjava.v3; +import com.fasterxml.jackson.annotation.JsonAnyGetter; +import com.fasterxml.jackson.annotation.JsonAnySetter; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import dev.zarr.zarrjava.ZarrException; @@ -7,6 +9,7 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; +import java.util.Map; public final class GroupMetadata extends dev.zarr.zarrjava.core.GroupMetadata { @@ -22,15 +25,35 @@ public final class GroupMetadata extends dev.zarr.zarrjava.core.GroupMetadata { @Nullable public final Attributes attributes; + /** + * Members of the metadata document that zarr-java does not know about. The Zarr v3 specification + * requires that these are ignored when they declare {@code "must_understand": false}, and that + * they are rejected otherwise. They are kept here so that rewriting the metadata does not drop + * extensions written by another implementation. + */ + private final Map extraFields; + public GroupMetadata(@Nullable Attributes attributes) throws ZarrException { - this(ZARR_FORMAT, NODE_TYPE, attributes); + this(ZARR_FORMAT, NODE_TYPE, attributes, null); + } + + public GroupMetadata( + @Nullable Attributes attributes, @Nullable Map extraFields + ) throws ZarrException { + this(ZARR_FORMAT, NODE_TYPE, attributes, extraFields); + } + + public GroupMetadata(int zarrFormat, String nodeType, @Nullable Attributes attributes) + throws ZarrException { + this(zarrFormat, nodeType, attributes, null); } @JsonCreator(mode = JsonCreator.Mode.PROPERTIES) public GroupMetadata( @JsonProperty(value = "zarr_format", required = true) int zarrFormat, @JsonProperty(value = "node_type", required = true) String nodeType, - @Nullable @JsonProperty(value = "attributes") Attributes attributes + @Nullable @JsonProperty(value = "attributes") Attributes attributes, + @Nullable @JsonAnySetter Map extraFields ) throws ZarrException { if (zarrFormat != this.zarrFormat) { throw new ZarrException( @@ -41,6 +64,7 @@ public GroupMetadata( "Expected node type '" + this.nodeType + "', got '" + nodeType + "'."); } this.attributes = attributes; + this.extraFields = ExtraFields.validated(extraFields, ExtraFields.GROUP_METADATA_KEYS); } public static GroupMetadata defaultValue() { @@ -53,6 +77,18 @@ public static GroupMetadata defaultValue() { } } + /** + * The members of the metadata document that zarr-java does not know about, but that declared + * {@code "must_understand": false} and could therefore be ignored. They are written back out + * unchanged, so that extensions written by another implementation survive a metadata rewrite. + * + * @return the extra fields, never {@code null} + */ + @JsonAnyGetter + public Map extraFields() { + return extraFields; + } + @Override public @Nonnull Attributes attributes() throws ZarrException { if (attributes == null) { diff --git a/src/test/java/dev/zarr/zarrjava/ExtraFieldsTest.java b/src/test/java/dev/zarr/zarrjava/ExtraFieldsTest.java new file mode 100644 index 0000000..5d42dca --- /dev/null +++ b/src/test/java/dev/zarr/zarrjava/ExtraFieldsTest.java @@ -0,0 +1,321 @@ +package dev.zarr.zarrjava; + +import com.fasterxml.jackson.databind.ObjectMapper; +import dev.zarr.zarrjava.core.Attributes; +import dev.zarr.zarrjava.store.FilesystemStore; +import dev.zarr.zarrjava.store.StoreHandle; +import dev.zarr.zarrjava.v3.Array; +import dev.zarr.zarrjava.v3.ArrayMetadata; +import dev.zarr.zarrjava.v3.DataType; +import dev.zarr.zarrjava.v3.Group; +import dev.zarr.zarrjava.v3.GroupMetadata; +import dev.zarr.zarrjava.v3.Node; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.Map; + +/** + * Unknown members of a Zarr v3 metadata document. + * + *

The specification requires a reader to ignore an unknown member that declares + * {@code "must_understand": false}, and to reject any other unknown member. These tests pin that + * behaviour, including that an ignored member survives a metadata rewrite. The semantics match + * zarr-python 3.1.6, see {@code zarr/core/metadata/v3.py}. + */ +public class ExtraFieldsTest extends ZarrTest { + + /** + * A minimal but valid v3 array metadata document. {@code %s} is a placeholder for additional + * members, so that a test can inject an extra field. + */ + private static final String ARRAY_METADATA_TEMPLATE = "{" + + "\"zarr_format\":3," + + "\"node_type\":\"array\"," + + "\"shape\":[4]," + + "\"data_type\":\"uint8\"," + + "\"chunk_grid\":{\"name\":\"regular\",\"configuration\":{\"chunk_shape\":[2]}}," + + "\"chunk_key_encoding\":{\"name\":\"default\"}," + + "\"fill_value\":0," + + "\"codecs\":[{\"name\":\"bytes\",\"configuration\":{\"endian\":\"little\"}}]" + + "%s}"; + + private static final String GROUP_METADATA_TEMPLATE = + "{\"zarr_format\":3,\"node_type\":\"group\",\"attributes\":{}%s}"; + + private static String arrayMetadata(String extraMembers) { + return String.format(ARRAY_METADATA_TEMPLATE, extraMembers); + } + + private static String groupMetadata(String extraMembers) { + return String.format(GROUP_METADATA_TEMPLATE, extraMembers); + } + + private static ObjectMapper objectMapper() { + return Node.makeObjectMapper(); + } + + // --------------------------------------------------------------------------------------------- + // (a) an ignorable extra field is accepted and round-trips + // --------------------------------------------------------------------------------------------- + + @Test + public void testArrayAcceptsIgnorableExtraField() throws Exception { + ArrayMetadata metadata = objectMapper().readValue( + arrayMetadata(",\"some_future_field\":{\"must_understand\":false,\"detail\":42}"), + ArrayMetadata.class); + + Assertions.assertEquals(1, metadata.extraFields().size(), + "the unknown member should have been captured as an extra field"); + Object extra = metadata.extraFields().get("some_future_field"); + Assertions.assertTrue(extra instanceof Map); + Assertions.assertEquals(Boolean.FALSE, ((Map) extra).get("must_understand")); + Assertions.assertEquals(42, ((Map) extra).get("detail")); + } + + @Test + public void testArrayExtraFieldIsWrittenBackOut() throws Exception { + ObjectMapper objectMapper = objectMapper(); + ArrayMetadata metadata = objectMapper.readValue( + arrayMetadata(",\"some_future_field\":{\"must_understand\":false,\"detail\":42}"), + ArrayMetadata.class); + + String serialized = objectMapper.writeValueAsString(metadata); + + // The extra field has to be written back as a top-level member, not nested under a property + // named after the Java field, and not dropped. + Map reparsed = objectMapper.readValue(serialized, Map.class); + Assertions.assertTrue(reparsed.containsKey("some_future_field"), + "extra field was dropped on write, serialized document: " + serialized); + Assertions.assertFalse(reparsed.containsKey("extraFields"), + "extra fields leaked as their own property, serialized document: " + serialized); + Assertions.assertEquals(Boolean.FALSE, + ((Map) reparsed.get("some_future_field")).get("must_understand")); + Assertions.assertEquals(42, ((Map) reparsed.get("some_future_field")).get("detail")); + + // And it survives a second parse, i.e. the round-trip is stable. + ArrayMetadata roundTripped = objectMapper.readValue(serialized, ArrayMetadata.class); + Assertions.assertEquals(metadata.extraFields(), roundTripped.extraFields()); + } + + @Test + public void testGroupAcceptsAndRoundTripsIgnorableExtraField() throws Exception { + ObjectMapper objectMapper = objectMapper(); + GroupMetadata metadata = objectMapper.readValue( + groupMetadata(",\"some_future_field\":{\"must_understand\":false}"), + GroupMetadata.class); + + Assertions.assertEquals(1, metadata.extraFields().size()); + + Map reparsed = objectMapper.readValue( + objectMapper.writeValueAsString(metadata), Map.class); + Assertions.assertTrue(reparsed.containsKey("some_future_field")); + } + + /** + * An extra field has to survive a metadata rewrite, otherwise updating an array's attributes + * would silently discard another implementation's extension. + */ + @Test + public void testArrayExtraFieldSurvivesMetadataRewrite() throws Exception { + StoreHandle storeHandle = + new FilesystemStore(TESTOUTPUT).resolve("extraFields", "arrayRewrite"); + Path zarrJson = storeHandle.resolve("zarr.json").toPath(); + + // Write a document with an extra field directly, then open it through the public API. + storeHandle.resolve("zarr.json").set(ByteBuffer.wrap( + arrayMetadata(",\"some_future_field\":{\"must_understand\":false,\"detail\":42}") + .getBytes(StandardCharsets.UTF_8))); + + Array array = Array.open(storeHandle); + Assertions.assertEquals(1, array.metadata().extraFields().size()); + + Attributes attributes = new Attributes(); + attributes.put("answer", 7); + Array updated = array.setAttributes(attributes); + + Assertions.assertEquals(1, updated.metadata().extraFields().size(), + "the extra field was dropped when the metadata was rewritten"); + + // Also assert against the bytes actually on disk. + String onDisk = new String( + java.nio.file.Files.readAllBytes(zarrJson), StandardCharsets.UTF_8); + Assertions.assertTrue(onDisk.contains("some_future_field"), + "the extra field is missing from the rewritten document: " + onDisk); + } + + @Test + public void testGroupExtraFieldSurvivesMetadataRewrite() throws Exception { + StoreHandle storeHandle = + new FilesystemStore(TESTOUTPUT).resolve("extraFields", "groupRewrite"); + storeHandle.resolve("zarr.json").set(ByteBuffer.wrap( + groupMetadata(",\"some_future_field\":{\"must_understand\":false}") + .getBytes(StandardCharsets.UTF_8))); + + Group group = Group.open(storeHandle); + Assertions.assertEquals(1, group.metadata().extraFields().size()); + + Attributes attributes = new Attributes(); + attributes.put("answer", 7); + Group updated = group.setAttributes(attributes); + + Assertions.assertEquals(1, updated.metadata().extraFields().size(), + "the extra field was dropped when the group metadata was rewritten"); + } + + // --------------------------------------------------------------------------------------------- + // (b) (c) (d) every other unknown member is rejected + // --------------------------------------------------------------------------------------------- + + /** + * @param extraMember an unknown member that the reader may not ignore: a scalar carries no + * {@code must_understand} declaration at all, an object may omit it or set it + * to {@code true}, and {@code must_understand} has to be exactly {@code false} + * rather than a truthy or stringly-typed stand-in. + */ + @ParameterizedTest + @ValueSource(strings = { + ",\"some_future_field\":5", + ",\"some_future_field\":\"a string\"", + ",\"some_future_field\":null", + ",\"some_future_field\":[1,2,3]", + ",\"some_future_field\":{}", + ",\"some_future_field\":{\"detail\":42}", + ",\"some_future_field\":{\"must_understand\":true}", + ",\"some_future_field\":{\"must_understand\":\"false\"}", + ",\"some_future_field\":{\"must_understand\":0}", + ",\"some_future_field\":{\"must_understand\":null}", + }) + public void testArrayRejectsNonIgnorableExtraField(String extraMember) { + Exception exception = Assertions.assertThrows(Exception.class, + () -> objectMapper().readValue(arrayMetadata(extraMember), ArrayMetadata.class)); + + assertCausedByZarrExceptionMentioning(exception, "some_future_field"); + } + + @ParameterizedTest + @ValueSource(strings = { + ",\"some_future_field\":5", + ",\"some_future_field\":{\"detail\":42}", + ",\"some_future_field\":{\"must_understand\":true}", + }) + public void testGroupRejectsNonIgnorableExtraField(String extraMember) { + Exception exception = Assertions.assertThrows(Exception.class, + () -> objectMapper().readValue(groupMetadata(extraMember), GroupMetadata.class)); + + assertCausedByZarrExceptionMentioning(exception, "some_future_field"); + } + + /** + * A disallowed extra field has to be rejected when opening an array, not silently accepted. + */ + @Test + public void testOpenRejectsNonIgnorableExtraField() throws Exception { + StoreHandle storeHandle = + new FilesystemStore(TESTOUTPUT).resolve("extraFields", "rejectedOnOpen"); + storeHandle.resolve("zarr.json").set(ByteBuffer.wrap( + arrayMetadata(",\"some_future_field\":{\"must_understand\":true}") + .getBytes(StandardCharsets.UTF_8))); + + Exception exception = + Assertions.assertThrows(Exception.class, () -> Array.open(storeHandle)); + assertCausedByZarrExceptionMentioning(exception, "some_future_field"); + } + + /** + * Constructing metadata with an extra field that collides with a member the metadata document + * defines itself has to fail, mirroring zarr-python's {@code parse_extra_fields}. + */ + @Test + public void testExtraFieldCollidingWithReservedKeyIsRejected() { + Map ignorable = new HashMap<>(); + ignorable.put("must_understand", false); + Map extraFields = new HashMap<>(); + extraFields.put("shape", ignorable); + + ZarrException exception = Assertions.assertThrows(ZarrException.class, + () -> Array.metadataBuilder() + .withShape(4) + .withDataType(DataType.UINT8) + .withChunkShape(2) + .withExtraFields(extraFields) + .build()); + + Assertions.assertTrue(exception.getMessage().contains("shape"), exception.getMessage()); + Assertions.assertTrue(exception.getMessage().contains("collide"), exception.getMessage()); + } + + // --------------------------------------------------------------------------------------------- + // (e) must_understand does not apply to codecs + // --------------------------------------------------------------------------------------------- + + /** + * An unknown codec name always has to fail: a codec cannot be skipped and still leave the chunk + * bytes decodable, so there is no {@code must_understand} escape hatch for it. + */ + @Test + public void testUnknownCodecNameThrows() { + Assertions.assertThrows(Exception.class, () -> objectMapper().readValue( + arrayMetadata("").replace("\"name\":\"bytes\"", "\"name\":\"not_a_real_codec\""), + ArrayMetadata.class)); + } + + /** + * Not even declaring {@code must_understand: false} inside a codec object makes an unknown codec + * skippable. + */ + @Test + public void testUnknownCodecNameThrowsEvenWithMustUnderstandFalse() { + Assertions.assertThrows(Exception.class, () -> objectMapper().readValue( + arrayMetadata("").replace( + "\"codecs\":[{\"name\":\"bytes\",\"configuration\":{\"endian\":\"little\"}}]", + "\"codecs\":[{\"name\":\"not_a_real_codec\",\"must_understand\":false}]"), + ArrayMetadata.class)); + } + + /** + * An unknown chunk grid name also has to fail, for the same reason. + */ + @Test + public void testUnknownChunkGridNameThrows() { + Assertions.assertThrows(Exception.class, () -> objectMapper().readValue( + arrayMetadata("").replace("\"name\":\"regular\"", "\"name\":\"not_a_real_grid\""), + ArrayMetadata.class)); + } + + /** + * An unknown chunk key encoding name also has to fail. + */ + @Test + public void testUnknownChunkKeyEncodingNameThrows() { + Assertions.assertThrows(Exception.class, () -> objectMapper().readValue( + arrayMetadata("").replace("\"name\":\"default\"", "\"name\":\"not_a_real_encoding\""), + ArrayMetadata.class)); + } + + // --------------------------------------------------------------------------------------------- + // helpers + // --------------------------------------------------------------------------------------------- + + /** + * Jackson wraps an exception thrown from a creator, so the {@link ZarrException} carrying the + * explanation shows up as a cause rather than as the thrown exception itself. + */ + private static void assertCausedByZarrExceptionMentioning(Throwable thrown, String needle) { + for (Throwable cause = thrown; cause != null; cause = cause.getCause()) { + if (cause instanceof ZarrException && cause.getMessage() != null + && cause.getMessage().contains(needle)) { + return; + } + } + Assertions.fail("expected a ZarrException mentioning '" + needle + "', but got: " + thrown, + thrown); + } +}