Ignore v3 metadata members declaring must_understand: false

src/main/java/dev/zarr/zarrjava/v3/ArrayMetadata.java

@@ -1,5 +1,7 @@
 package dev.zarr.zarrjava.v3;
 
+import com.fasterxml.jackson.annotation.JsonAnyGetter;
+import com.fasterxml.jackson.annotation.JsonAnySetter;
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonProperty;
@@ -48,6 +50,14 @@ public final class ArrayMetadata extends dev.zarr.zarrjava.core.ArrayMetadata {
     @JsonProperty("storage_transformers")
     public final Map<String, Object>[] storageTransformers;
 
+    /**
+     * Members of the metadata document that zarr-java does not know about. The Zarr v3 specification
+     * requires that these are ignored when they declare {@code "must_understand": false}, and that
+     * they are rejected otherwise. They are kept here so that rewriting the metadata does not drop
+     * extensions written by another implementation.
+     */
+    private final Map<String, Object> extraFields;
+
     @JsonIgnore
     public CoreArrayMetadata coreArrayMetadata;
 
@@ -65,6 +75,39 @@ public ArrayMetadata(
         );
     }
 
+    public ArrayMetadata(
+            long[] shape, DataType dataType, ChunkGrid chunkGrid, ChunkKeyEncoding chunkKeyEncoding,
+            Object fillValue,
+            @Nonnull Codec[] codecs,
+            @Nullable String[] dimensionNames,
+            @Nullable Attributes attributes,
+            @Nullable Map<String, Object>[] storageTransformers,
+            @Nullable Map<String, Object> extraFields
+    ) throws ZarrException {
+        this(ZARR_FORMAT, NODE_TYPE, shape, dataType, chunkGrid, chunkKeyEncoding, fillValue, codecs,
+                dimensionNames,
+                attributes, storageTransformers, extraFields
+        );
+    }
+
+    public ArrayMetadata(
+            int zarrFormat,
+            String nodeType,
+            long[] shape,
+            DataType dataType,
+            ChunkGrid chunkGrid,
+            ChunkKeyEncoding chunkKeyEncoding,
+            Object fillValue,
+            @Nonnull Codec[] codecs,
+            @Nullable String[] dimensionNames,
+            @Nullable Attributes attributes,
+            @Nullable Map<String, Object>[] storageTransformers
+    ) throws ZarrException {
+        this(zarrFormat, nodeType, shape, dataType, chunkGrid, chunkKeyEncoding, fillValue, codecs,
+                dimensionNames, attributes, storageTransformers, null
+        );
+    }
+
     @JsonCreator(mode = JsonCreator.Mode.PROPERTIES)
     public ArrayMetadata(
             @JsonProperty(value = "zarr_format", required = true) int zarrFormat,
@@ -77,9 +120,11 @@ public ArrayMetadata(
             @Nonnull @JsonProperty(value = "codecs") Codec[] codecs,
             @Nullable @JsonProperty(value = "dimension_names") String[] dimensionNames,
             @Nullable @JsonProperty(value = "attributes") Attributes attributes,
-            @Nullable @JsonProperty(value = "storage_transformers") Map<String, Object>[] storageTransformers
+            @Nullable @JsonProperty(value = "storage_transformers") Map<String, Object>[] storageTransformers,
+            @Nullable @JsonAnySetter Map<String, Object> extraFields
     ) throws ZarrException {
         super(shape, fillValue, dataType);
+        this.extraFields = ExtraFields.validated(extraFields, ExtraFields.ARRAY_METADATA_KEYS);
         if (zarrFormat != this.zarrFormat) {
             throw new ZarrException(
                     "Expected zarr format '" + this.zarrFormat + "', got '" + zarrFormat + "'.");
@@ -129,6 +174,18 @@ public ArrayMetadata(
         this.storageTransformers = storageTransformers;
     }
 
+    /**
+     * The members of the metadata document that zarr-java does not know about, but that declared
+     * {@code "must_understand": false} and could therefore be ignored. They are written back out
+     * unchanged, so that extensions written by another implementation survive a metadata rewrite.
+     *
+     * @return the extra fields, never {@code null}
+     */
+    @JsonAnyGetter
+    public Map<String, Object> extraFields() {
+        return extraFields;
+    }
+
     public static Optional<Codec> getShardingIndexedCodec(Codec[] codecs) {
         return Arrays.stream(codecs).filter(codec -> codec instanceof ShardingIndexedCodec).findFirst();
     }

src/main/java/dev/zarr/zarrjava/v3/ArrayMetadataBuilder.java

@@ -31,6 +31,7 @@ public class ArrayMetadataBuilder {
     Attributes attributes = new Attributes();
     Map<String, Object>[] storageTransformers = new HashMap[]{};
     String[] dimensionNames = null;
+    Map<String, Object> extraFields = null;
 
     protected ArrayMetadataBuilder() {
     }
@@ -49,6 +50,7 @@ protected static ArrayMetadataBuilder fromArrayMetadata(ArrayMetadata arrayMetad
         builder.codecs = arrayMetadata.codecs;
         builder.dimensionNames = arrayMetadata.dimensionNames;
         builder.storageTransformers = arrayMetadata.storageTransformers;
+        builder.extraFields = arrayMetadata.extraFields();
         if (withAttributes) {
             builder.attributes = arrayMetadata.attributes;
         }
@@ -155,6 +157,17 @@ public ArrayMetadataBuilder withStorageTransformers(Map<String, Object>[] storag
         return this;
     }
 
+    /**
+     * Sets members of the metadata document that zarr-java itself does not interpret. Every value
+     * needs to be a map carrying {@code "must_understand": false}, otherwise {@link #build()} fails.
+     *
+     * @param extraFields the extra fields to write into the metadata document
+     */
+    public ArrayMetadataBuilder withExtraFields(Map<String, Object> extraFields) {
+        this.extraFields = extraFields;
+        return this;
+    }
+
     public ArrayMetadata build() throws ZarrException {
         if (shape == null) {
             throw new ZarrException("Shape needs to be provided. Please call `.withShape`.");
@@ -172,7 +185,8 @@ public ArrayMetadata build() throws ZarrException {
         return new ArrayMetadata(shape, dataType, chunkGrid, chunkKeyEncoding, fillValue, codecs,
                 dimensionNames,
                 attributes,
-                storageTransformers
+                storageTransformers,
+                extraFields
         );
     }
 }

src/main/java/dev/zarr/zarrjava/v3/ExtraFields.java

@@ -0,0 +1,104 @@
+package dev.zarr.zarrjava.v3;
+
+import dev.zarr.zarrjava.ZarrException;
+
+import javax.annotation.Nullable;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * Handling of unknown ("extra") members of a Zarr v3 metadata document.
+ *
+ * <p>The Zarr v3 specification allows a writer to add members that a reader may not know about. Such
+ * a member has to be a JSON object carrying {@code "must_understand": false}, which declares that a
+ * reader that does not know the member may safely ignore it. Any other unknown member has to be
+ * rejected, because it may change how the array is to be interpreted.
+ *
+ * <p>These semantics match zarr-python 3.1.6, see {@code zarr/core/metadata/v3.py}.
+ */
+final class ExtraFields {
+
+    static final String MUST_UNDERSTAND = "must_understand";
+
+    /**
+     * Members of a v3 array metadata document that zarr-java knows about. Anything else read from a
+     * {@code zarr.json} array document is an extra field.
+     */
+    static final Set<String> ARRAY_METADATA_KEYS = unmodifiableSetOf(
+            "zarr_format", "node_type", "shape", "data_type", "chunk_grid", "chunk_key_encoding",
+            "fill_value", "codecs", "attributes", "dimension_names", "storage_transformers"
+    );
+
+    /**
+     * Members of a v3 group metadata document that zarr-java knows about. Anything else read from a
+     * {@code zarr.json} group document is an extra field.
+     */
+    static final Set<String> GROUP_METADATA_KEYS = unmodifiableSetOf(
+            "zarr_format", "node_type", "attributes", "consolidated_metadata"
+    );
+
+    private ExtraFields() {
+    }
+
+    private static Set<String> unmodifiableSetOf(String... keys) {
+        return Collections.unmodifiableSet(new HashSet<>(Arrays.asList(keys)));
+    }
+
+    /**
+     * Whether an unknown metadata member may be ignored, i.e. whether it is a JSON object with a
+     * {@code must_understand} member that is set to {@code false}.
+     */
+    static boolean isIgnorable(@Nullable Object value) {
+        return value instanceof Map
+                && Boolean.FALSE.equals(((Map<?, ?>) value).get(MUST_UNDERSTAND));
+    }
+
+    /**
+     * Validates unknown members of a metadata document and returns them so that they can be written
+     * back out unchanged.
+     *
+     * @param extraFields  the unknown members, may be {@code null}
+     * @param reservedKeys the members that the metadata document defines itself, either
+     *                     {@link #ARRAY_METADATA_KEYS} or {@link #GROUP_METADATA_KEYS}
+     * @return the extra fields, never {@code null}
+     * @throws ZarrException if a member collides with a reserved key, or if a member may not be
+     *                       ignored because it is not a JSON object carrying
+     *                       {@code "must_understand": false}
+     */
+    static Map<String, Object> validated(
+            @Nullable Map<String, Object> extraFields, Set<String> reservedKeys
+    ) throws ZarrException {
+        if (extraFields == null || extraFields.isEmpty()) {
+            return Collections.emptyMap();
+        }
+        final List<String> reservedCollisions = new ArrayList<>();
+        final List<String> notIgnorable = new ArrayList<>();
+        for (Map.Entry<String, Object> entry : extraFields.entrySet()) {
+            if (reservedKeys.contains(entry.getKey())) {
+                reservedCollisions.add(entry.getKey());
+            } else if (!isIgnorable(entry.getValue())) {
+                notIgnorable.add(entry.getKey());
+            }
+        }
+        if (!reservedCollisions.isEmpty()) {
+            Collections.sort(reservedCollisions);
+            throw new ZarrException(
+                    "Invalid extra fields. The following keys: " + reservedCollisions + " are invalid " +
+                            "because they collide with keys reserved for use by the metadata document.");
+        }
+        if (!notIgnorable.isEmpty()) {
+            Collections.sort(notIgnorable);
+            throw new ZarrException(
+                    "Got a Zarr v3 metadata document with the following disallowed extra fields: " +
+                            notIgnorable + ". Extra fields are not allowed unless they are a JSON object " +
+                            "with a \"" + MUST_UNDERSTAND + "\" key which is assigned the value `false`.");
+        }
+        return Collections.unmodifiableMap(new LinkedHashMap<>(extraFields));
+    }
+}

src/main/java/dev/zarr/zarrjava/v3/Group.java

@@ -302,7 +302,8 @@ public Group updateAttributes(Function<Attributes, Attributes> attributeMapper)
      * @throws IOException   if the metadata cannot be serialized
      */
     public Group setAttributes(Attributes newAttributes) throws ZarrException, IOException {
-        GroupMetadata newGroupMetadata = new GroupMetadata(newAttributes);
+        GroupMetadata newGroupMetadata =
+                new GroupMetadata(newAttributes, metadata.extraFields());
         return writeMetadata(newGroupMetadata);
     }
 

src/main/java/dev/zarr/zarrjava/v3/GroupMetadata.java

@@ -1,12 +1,15 @@
 package dev.zarr.zarrjava.v3;
 
+import com.fasterxml.jackson.annotation.JsonAnyGetter;
+import com.fasterxml.jackson.annotation.JsonAnySetter;
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import dev.zarr.zarrjava.ZarrException;
 import dev.zarr.zarrjava.core.Attributes;
 
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
+import java.util.Map;
 
 public final class GroupMetadata extends dev.zarr.zarrjava.core.GroupMetadata {
 
@@ -22,15 +25,35 @@ public final class GroupMetadata extends dev.zarr.zarrjava.core.GroupMetadata {
     @Nullable
     public final Attributes attributes;
 
+    /**
+     * Members of the metadata document that zarr-java does not know about. The Zarr v3 specification
+     * requires that these are ignored when they declare {@code "must_understand": false}, and that
+     * they are rejected otherwise. They are kept here so that rewriting the metadata does not drop
+     * extensions written by another implementation.
+     */
+    private final Map<String, Object> extraFields;
+
     public GroupMetadata(@Nullable Attributes attributes) throws ZarrException {
-        this(ZARR_FORMAT, NODE_TYPE, attributes);
+        this(ZARR_FORMAT, NODE_TYPE, attributes, null);
+    }
+
+    public GroupMetadata(
+            @Nullable Attributes attributes, @Nullable Map<String, Object> extraFields
+    ) throws ZarrException {
+        this(ZARR_FORMAT, NODE_TYPE, attributes, extraFields);
+    }
+
+    public GroupMetadata(int zarrFormat, String nodeType, @Nullable Attributes attributes)
+            throws ZarrException {
+        this(zarrFormat, nodeType, attributes, null);
     }
 
     @JsonCreator(mode = JsonCreator.Mode.PROPERTIES)
     public GroupMetadata(
             @JsonProperty(value = "zarr_format", required = true) int zarrFormat,
             @JsonProperty(value = "node_type", required = true) String nodeType,
-            @Nullable @JsonProperty(value = "attributes") Attributes attributes
+            @Nullable @JsonProperty(value = "attributes") Attributes attributes,
+            @Nullable @JsonAnySetter Map<String, Object> extraFields
     ) throws ZarrException {
         if (zarrFormat != this.zarrFormat) {
             throw new ZarrException(
@@ -41,6 +64,7 @@ public GroupMetadata(
                     "Expected node type '" + this.nodeType + "', got '" + nodeType + "'.");
         }
         this.attributes = attributes;
+        this.extraFields = ExtraFields.validated(extraFields, ExtraFields.GROUP_METADATA_KEYS);
     }
 
     public static GroupMetadata defaultValue() {
@@ -53,6 +77,18 @@ public static GroupMetadata defaultValue() {
         }
     }
 
+    /**
+     * The members of the metadata document that zarr-java does not know about, but that declared
+     * {@code "must_understand": false} and could therefore be ignored. They are written back out
+     * unchanged, so that extensions written by another implementation survive a metadata rewrite.
+     *
+     * @return the extra fields, never {@code null}
+     */
+    @JsonAnyGetter
+    public Map<String, Object> extraFields() {
+        return extraFields;
+    }
+
     @Override
     public @Nonnull Attributes attributes() throws ZarrException {
         if (attributes == null) {