From f535335032762c643919b72dc84b1f90661e922c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Mej=C3=ADa?= Date: Sun, 12 Jul 2026 12:24:49 +0200 Subject: [PATCH 01/25] AVRO-4300: [java] Bound zero-byte array element allocation An array whose element schema encodes to zero bytes (null, a zero-length fixed, or a record with only zero-byte fields) consumes no input per element, so the number of elements a block declares cannot be bounded by the bytes remaining in the stream. ensureAvailableCollectionBytes therefore skips the check for such elements, and the collection-length cap is Integer.MAX_VALUE-8 (a VM array-size ceiling, not a memory budget). A tiny payload declaring a huge block count of such elements (e.g. {"type":"array","items":"null"} with a count of 200,000,000) drives an unbounded backing-array allocation and exhausts the heap. This affects both the classic GenericDatumReader.readArray path and the default fast-reader path (FastReaderBuilder), which had no collection guard at all. Add SystemLimitException.checkMaxCollectionAllocation, a heap-aware cumulative cap (default: maxMemory()/4/8 elements, overridable via the org.apache.avro.limits.collectionItems.maxAllocation system property, mirroring the existing decompression limit). Enforce it before allocating in both reader paths, keyed on GenericDatumReader.isZeroByteSchema so only the unbounded zero-byte case is affected; all other element types remain bounded by ensureAvailableCollectionBytes and are unchanged. Maps are already bounded because each entry carries a string key of at least one byte. Assisted-by: GitHub Copilot:claude-opus-4.8 --- .../org/apache/avro/SystemLimitException.java | 81 +++++++++++++++++++ .../avro/generic/GenericDatumReader.java | 30 ++++++- .../org/apache/avro/io/FastReaderBuilder.java | 24 ++++++ .../apache/avro/TestSystemLimitException.java | 41 ++++++++++ .../avro/generic/TestGenericDatumReader.java | 80 ++++++++++++++++++ 5 files changed, 255 insertions(+), 1 deletion(-) diff --git a/lang/java/avro/src/main/java/org/apache/avro/SystemLimitException.java b/lang/java/avro/src/main/java/org/apache/avro/SystemLimitException.java index 886a939735c..33fd97531bf 100644 --- a/lang/java/avro/src/main/java/org/apache/avro/SystemLimitException.java +++ b/lang/java/avro/src/main/java/org/apache/avro/SystemLimitException.java @@ -35,6 +35,12 @@ * once single sequence. *
  • org.apache.avro.limits.string.maxLength
    • limits the maximum * size of string types. + *
  • org.apache.avro.limits.collectionItems.maxAllocation
    • limits + * the number of array elements whose schema encodes to zero bytes + * (such as null or a self-referencing record) that may be allocated at + * once. Unlike other element types, these cannot be bounded by the number of + * bytes remaining in the stream, so the limit defaults to a fraction of the + * maximum heap. * * * The default is to permit sizes up to {@link #MAX_ARRAY_VM_LIMIT}. @@ -83,6 +89,32 @@ public class SystemLimitException extends AvroRuntimeException { public static final long MAX_DECOMPRESS_LENGTH = getLongLimitFromProperty(MAX_DECOMPRESS_LENGTH_PROPERTY, defaultMaxDecompressLength()); + /** + * System property declaring the maximum number of zero-byte-encoded array + * elements (e.g. {@code null} or a self-referencing record) to allocate at + * once: {@value}. + */ + public static final String MAX_COLLECTION_ALLOCATION_PROPERTY = "org.apache.avro.limits.collectionItems.maxAllocation"; + + /** + * Fraction of the maximum heap a single decoded collection of zero-byte + * elements may occupy by default. Keeps the backing allocation below the heap + * so a small payload declaring a huge block count cannot exhaust the JVM. + */ + private static final long DEFAULT_MAX_COLLECTION_ALLOCATION_HEAP_FRACTION = 4; + + /** + * Estimated bytes retained per pre-allocated collection slot (a single object + * reference), used to translate the heap budget into an element count. + */ + private static final long BYTES_PER_COLLECTION_SLOT = 8; + + /** + * Maximum number of zero-byte-encoded array elements to allocate at once. + * Recomputed from the system property (or the heap) by {@link #resetLimits()}. + */ + private static long maxCollectionAllocation = defaultMaxCollectionAllocation(); + static { resetLimits(); } @@ -153,6 +185,19 @@ private static long defaultMaxDecompressLength() { Math.max(1L, Runtime.getRuntime().maxMemory() / DEFAULT_MAX_DECOMPRESS_HEAP_FRACTION)); } + /** + * Calculate the default maximum number of zero-byte-encoded array elements to + * allocate at once, as a fraction of the maximum heap. Such elements consume no + * input bytes, so the usual "bytes remaining" bound does not apply and the + * allocation must instead be capped relative to the available memory. + * + * @return the calculated default max zero-byte element count. + */ + private static long defaultMaxCollectionAllocation() { + long heapBudget = Math.max(1L, Runtime.getRuntime().maxMemory() / DEFAULT_MAX_COLLECTION_ALLOCATION_HEAP_FRACTION); + return Math.max(1L, heapBudget / BYTES_PER_COLLECTION_SLOT); + } + /** * Check to ensure that reading the bytes is within the specified limits. * @@ -246,6 +291,40 @@ public static int checkMaxCollectionLength(long items) { return (int) items; } + /** + * Check to ensure that allocating storage for the specified number of + * zero-byte-encoded array elements remains within the heap-aware limit. + *

    + * Elements whose schema encodes to zero bytes (e.g. {@code null} or a + * self-referencing record) consume no input bytes, so the number that may be + * declared is not bounded by the bytes remaining in the stream. Without a cap, + * a tiny payload can declare an enormous block count and drive an unbounded + * backing-array allocation. This limit is derived from the maximum heap (see + * {@link #MAX_COLLECTION_ALLOCATION_PROPERTY}). + * + * @param existing The number of elements already allocated for the collection. + * @param items The next number of elements to allocate. + * @return The cumulative element count if and only if it is within the limit. + * @throws SystemLimitException if the cumulative allocation would exceed the + * limit. + * @throws AvroRuntimeException if either argument is negative. + */ + public static long checkMaxCollectionAllocation(long existing, long items) { + if (existing < 0) { + throw new AvroRuntimeException("Malformed data. Length is negative: " + existing); + } + if (items < 0) { + throw new AvroRuntimeException("Malformed data. Length is negative: " + items); + } + long total = existing + items; + if (total < existing || total > maxCollectionAllocation) { + throw new SystemLimitException("Cannot allocate " + (total < existing ? "more than Long.MAX_VALUE" : total) + + " zero-byte collection elements: exceeds the maximum allowed of " + maxCollectionAllocation + + " (configure with the system property " + MAX_COLLECTION_ALLOCATION_PROPERTY + ")"); + } + return total; + } + /** * Check to ensure that reading the string size is within the specified limits. * @@ -294,5 +373,7 @@ static void resetLimits() { maxBytesLength = getLimitFromProperty(MAX_BYTES_LENGTH_PROPERTY, MAX_ARRAY_VM_LIMIT); maxCollectionLength = getLimitFromProperty(MAX_COLLECTION_LENGTH_PROPERTY, MAX_ARRAY_VM_LIMIT); maxStringLength = getLimitFromProperty(MAX_STRING_LENGTH_PROPERTY, MAX_ARRAY_VM_LIMIT); + maxCollectionAllocation = getLongLimitFromProperty(MAX_COLLECTION_ALLOCATION_PROPERTY, + defaultMaxCollectionAllocation()); } } diff --git a/lang/java/avro/src/main/java/org/apache/avro/generic/GenericDatumReader.java b/lang/java/avro/src/main/java/org/apache/avro/generic/GenericDatumReader.java index 178ca50ccad..8773904460f 100644 --- a/lang/java/avro/src/main/java/org/apache/avro/generic/GenericDatumReader.java +++ b/lang/java/avro/src/main/java/org/apache/avro/generic/GenericDatumReader.java @@ -37,6 +37,7 @@ import org.apache.avro.LogicalType; import org.apache.avro.Schema; import org.apache.avro.Schema.Field; +import org.apache.avro.SystemLimitException; import org.apache.avro.io.DatumReader; import org.apache.avro.io.Decoder; import org.apache.avro.io.DecoderFactory; @@ -296,6 +297,15 @@ protected Object readArray(Object old, Schema expected, ResolvingDecoder in) thr long base = 0; if (l > 0) { ensureAvailableCollectionBytes(in, l, expectedType); + // Elements whose schema encodes to zero bytes (null, or a self-referencing + // record) consume no input, so ensureAvailableCollectionBytes cannot bound + // their count from the bytes remaining. Cap such collections against a + // heap-aware limit so a tiny payload cannot declare a huge block count and + // drive an unbounded backing-array allocation. + boolean zeroByteElements = minBytesPerElement(expectedType) == 0; + if (zeroByteElements) { + SystemLimitException.checkMaxCollectionAllocation(base, l); + } LogicalType logicalType = expectedType.getLogicalType(); Conversion conversion = getData().getConversionFor(logicalType); Object array = newArray(old, (int) l, expected); @@ -311,7 +321,11 @@ protected Object readArray(Object old, Schema expected, ResolvingDecoder in) thr } } base += l; - } while ((l = arrayNext(in, expectedType)) > 0); + l = arrayNext(in, expectedType); + if (zeroByteElements && l > 0) { + SystemLimitException.checkMaxCollectionAllocation(base, l); + } + } while (l > 0); return pruneArray(array); } else { return pruneArray(newArray(old, 0, expected)); @@ -441,6 +455,20 @@ static int minBytesPerElement(Schema schema) { return minBytesPerElement(schema, Collections.newSetFromMap(new IdentityHashMap<>())); } + /** + * Whether values of the given schema encode to zero bytes (e.g. {@code null}, + * zero-length {@code fixed}, or a record whose fields are all zero-byte). Such + * elements cannot be bounded by the number of bytes remaining in the stream, so + * a collection of them must be bounded by a heap-aware allocation limit + * instead. + * + * @param schema the element (or map value) schema + * @return {@code true} if the schema encodes to zero bytes + */ + public static boolean isZeroByteSchema(Schema schema) { + return minBytesPerElement(schema) == 0; + } + private static int minBytesPerElement(Schema schema, Set visited) { switch (schema.getType()) { case NULL: diff --git a/lang/java/avro/src/main/java/org/apache/avro/io/FastReaderBuilder.java b/lang/java/avro/src/main/java/org/apache/avro/io/FastReaderBuilder.java index 512c9ebf34f..7c03d76eec6 100644 --- a/lang/java/avro/src/main/java/org/apache/avro/io/FastReaderBuilder.java +++ b/lang/java/avro/src/main/java/org/apache/avro/io/FastReaderBuilder.java @@ -40,6 +40,7 @@ import org.apache.avro.Resolver.WriterUnion; import org.apache.avro.Schema; import org.apache.avro.Schema.Field; +import org.apache.avro.SystemLimitException; import org.apache.avro.generic.GenericArray; import org.apache.avro.generic.GenericData; import org.apache.avro.generic.GenericData.InstanceSupplier; @@ -470,21 +471,40 @@ private Optional> findClass(String clazz) { private FieldReader createArrayReader(Schema readerSchema, Container action) throws IOException { FieldReader elementReader = getReaderFor(action.elementAction, null); + // Elements whose schema encodes to zero bytes (null, or a record with only + // zero-byte fields) consume no input, so the block count cannot be bounded by + // the bytes remaining in the stream. Cap such collections against a heap-aware + // limit so a tiny payload cannot declare a huge block count and drive an + // unbounded backing-array allocation (AVRO-4300). + boolean zeroByteElements = GenericDatumReader.isZeroByteSchema(readerSchema.getElementType()); + return reusingReader((reuse, decoder) -> { if (reuse instanceof GenericArray) { GenericArray reuseArray = (GenericArray) reuse; long l = decoder.readArrayStart(); + long total = 0; + if (zeroByteElements && l > 0) { + SystemLimitException.checkMaxCollectionAllocation(total, l); + } reuseArray.clear(); while (l > 0) { for (long i = 0; i < l; i++) { reuseArray.add(elementReader.read(reuseArray.peek(), decoder)); } + total += l; l = decoder.arrayNext(); + if (zeroByteElements && l > 0) { + SystemLimitException.checkMaxCollectionAllocation(total, l); + } } return reuseArray; } else { long l = decoder.readArrayStart(); + long total = 0; + if (zeroByteElements && l > 0) { + SystemLimitException.checkMaxCollectionAllocation(total, l); + } List array = (reuse instanceof List) ? (List) reuse : new GenericData.Array<>((int) l, readerSchema); array.clear(); @@ -492,7 +512,11 @@ private FieldReader createArrayReader(Schema readerSchema, Container action) thr for (long i = 0; i < l; i++) { array.add(elementReader.read(null, decoder)); } + total += l; l = decoder.arrayNext(); + if (zeroByteElements && l > 0) { + SystemLimitException.checkMaxCollectionAllocation(total, l); + } } return array; } diff --git a/lang/java/avro/src/test/java/org/apache/avro/TestSystemLimitException.java b/lang/java/avro/src/test/java/org/apache/avro/TestSystemLimitException.java index 0da39179506..7477ce58dc4 100644 --- a/lang/java/avro/src/test/java/org/apache/avro/TestSystemLimitException.java +++ b/lang/java/avro/src/test/java/org/apache/avro/TestSystemLimitException.java @@ -46,6 +46,7 @@ void reset() { System.clearProperty(MAX_BYTES_LENGTH_PROPERTY); System.clearProperty(MAX_COLLECTION_LENGTH_PROPERTY); System.clearProperty(MAX_STRING_LENGTH_PROPERTY); + System.clearProperty(MAX_COLLECTION_ALLOCATION_PROPERTY); resetLimits(); } @@ -110,6 +111,46 @@ void testCheckMaxStringLength() { "String length 1024 exceeds maximum allowed"); } + @Test + void testCheckMaxCollectionAllocation() { + // With a small custom limit, cumulative allocations beyond it are rejected. + System.setProperty(MAX_COLLECTION_ALLOCATION_PROPERTY, "1000"); + resetLimits(); + + // Values within the limit pass through and return the running total. + assertEquals(0L, checkMaxCollectionAllocation(0L, 0L)); + assertEquals(1000L, checkMaxCollectionAllocation(0L, 1000L)); + assertEquals(1000L, checkMaxCollectionAllocation(400L, 600L)); + + // A single block over the limit is rejected. + SystemLimitException ex = assertThrows(SystemLimitException.class, () -> checkMaxCollectionAllocation(0L, 1001L)); + assertTrue(ex.getMessage().contains("exceeds the maximum allowed of 1000"), ex.getMessage()); + + // Cumulative blocks that cross the limit are rejected. + ex = assertThrows(SystemLimitException.class, () -> checkMaxCollectionAllocation(600L, 401L)); + assertTrue(ex.getMessage().contains("exceeds the maximum allowed of 1000"), ex.getMessage()); + + // Negative arguments are rejected as malformed. + Exception nex = assertThrows(AvroRuntimeException.class, () -> checkMaxCollectionAllocation(-1L, 10L)); + assertEquals(ERROR_NEGATIVE, nex.getMessage()); + nex = assertThrows(AvroRuntimeException.class, () -> checkMaxCollectionAllocation(10L, -1L)); + assertEquals(ERROR_NEGATIVE, nex.getMessage()); + + // Additive overflow is rejected rather than wrapping to a small value. + assertThrows(SystemLimitException.class, () -> checkMaxCollectionAllocation(Long.MAX_VALUE, 10L)); + } + + @Test + void testCheckMaxCollectionAllocationDefaultsToHeapFraction() { + // With no property set, the default is derived from the heap and is well + // below Integer.MAX_VALUE on a normally sized JVM, yet generous enough for + // legitimate small collections. + resetLimits(); + assertEquals(1024L, checkMaxCollectionAllocation(0L, 1024L)); + // A pathologically large zero-byte collection is rejected without allocating. + assertThrows(SystemLimitException.class, () -> checkMaxCollectionAllocation(0L, (long) Integer.MAX_VALUE - 8)); + } + @Test void testCheckMaxCollectionLengthFromNonZero() { // Correct values pass through diff --git a/lang/java/avro/src/test/java/org/apache/avro/generic/TestGenericDatumReader.java b/lang/java/avro/src/test/java/org/apache/avro/generic/TestGenericDatumReader.java index 5586b828999..db91bb08b6e 100644 --- a/lang/java/avro/src/test/java/org/apache/avro/generic/TestGenericDatumReader.java +++ b/lang/java/avro/src/test/java/org/apache/avro/generic/TestGenericDatumReader.java @@ -32,6 +32,7 @@ import java.util.stream.IntStream; import org.apache.avro.Schema; +import org.apache.avro.SystemLimitException; import org.apache.avro.io.BinaryDecoder; import org.apache.avro.io.BinaryEncoder; import org.apache.avro.io.DecoderFactory; @@ -303,4 +304,83 @@ void mapOfRecordsRejectsHugeCountUsingFullRecordSize() throws Exception { BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(data, null); assertThrows(EOFException.class, () -> reader.read(null, decoder)); } + + // --- Zero-byte element collection allocation limit (AVRO-4300) --- + + /** + * An array of {@code null} elements encodes each element as 0 bytes, so the + * bytes-remaining check cannot bound the block count. A huge count must be + * rejected by the heap-aware allocation limit rather than driving an unbounded + * backing-array allocation. + */ + @Test + void arrayOfNullsRejectsCountAboveAllocationLimit() throws Exception { + System.setProperty(SystemLimitException.MAX_COLLECTION_ALLOCATION_PROPERTY, "1000"); + org.apache.avro.TestSystemLimitException.resetLimits(); + try { + Schema schema = Schema.createArray(Schema.create(Schema.Type.NULL)); + GenericDatumReader reader = new GenericDatumReader<>(schema); + + // A single block declaring 200,000 null elements: only ~4 payload bytes, + // but would allocate a 200,000-slot backing array. Exceeds the limit of + // 1000, so it must be rejected before allocating. + byte[] data = encodeVarints(200_000L, 0L); + BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(data, null); + assertThrows(SystemLimitException.class, () -> reader.read(null, decoder)); + + // Cumulative growth across multiple blocks is also rejected: two blocks of + // 600 nulls each (1200 > 1000) must throw on the second block. + byte[] cumulative = encodeVarints(600L, 600L, 0L); + BinaryDecoder decoder2 = DecoderFactory.get().binaryDecoder(cumulative, null); + assertThrows(SystemLimitException.class, () -> reader.read(null, decoder2)); + } finally { + System.clearProperty(SystemLimitException.MAX_COLLECTION_ALLOCATION_PROPERTY); + org.apache.avro.TestSystemLimitException.resetLimits(); + } + } + + /** + * A legitimate array of {@code null} elements within the allocation limit still + * decodes correctly, so the guard does not reject valid data. + */ + @Test + void arrayOfNullsWithinAllocationLimitStillDecodes() throws Exception { + System.setProperty(SystemLimitException.MAX_COLLECTION_ALLOCATION_PROPERTY, "1000"); + org.apache.avro.TestSystemLimitException.resetLimits(); + try { + Schema schema = Schema.createArray(Schema.create(Schema.Type.NULL)); + GenericDatumReader reader = new GenericDatumReader<>(schema); + + byte[] data = encodeVarints(1000L, 0L); + BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(data, null); + GenericData.Array result = (GenericData.Array) reader.read(null, decoder); + assertEquals(1000, result.size()); + } finally { + System.clearProperty(SystemLimitException.MAX_COLLECTION_ALLOCATION_PROPERTY); + org.apache.avro.TestSystemLimitException.resetLimits(); + } + } + + /** + * An array whose element is a record with only {@code null} fields also encodes + * to 0 bytes per element and must be bounded by the allocation limit. + */ + @Test + void arrayOfAllNullRecordsRejectsCountAboveAllocationLimit() throws Exception { + System.setProperty(SystemLimitException.MAX_COLLECTION_ALLOCATION_PROPERTY, "1000"); + org.apache.avro.TestSystemLimitException.resetLimits(); + try { + Schema recWithNull = Schema.createRecord("AllNull", null, "test", false); + recWithNull.setFields(Collections.singletonList(new Schema.Field("n", Schema.create(Schema.Type.NULL)))); + Schema schema = Schema.createArray(recWithNull); + GenericDatumReader reader = new GenericDatumReader<>(schema); + + byte[] data = encodeVarints(200_000L, 0L); + BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(data, null); + assertThrows(SystemLimitException.class, () -> reader.read(null, decoder)); + } finally { + System.clearProperty(SystemLimitException.MAX_COLLECTION_ALLOCATION_PROPERTY); + org.apache.avro.TestSystemLimitException.resetLimits(); + } + } } From 24d6455e5e25dd090960f295864f0b8bd62cc8fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Mej=C3=ADa?= Date: Sun, 12 Jul 2026 13:00:56 +0200 Subject: [PATCH 02/25] AVRO-4300: [java] Apply the available-bytes check on the fast reader path The fast reader (FastReaderBuilder, the default decode path) never received the AVRO-4241 bytes-remaining guard: that change only touched the classic GenericDatumReader. As a result an array of non-zero-byte elements with a huge declared block count and no data (e.g. array/array with a count of 200,000,000) still pre-allocated new GenericData.Array<>((int) count) on the default path and exhausted the heap. Expose GenericDatumReader.ensureAvailableCollectionBytes and apply it, together with the zero-byte allocation cap, before allocating each array block in FastReaderBuilder, so the fast and classic readers enforce identical guards. Maps were already safe on both paths (each entry carries a >=1-byte key). Verified with a matrix of array and map at a huge count under -Xmx256m: every combination is now rejected (SystemLimitException for zero-byte elements, EOFException otherwise) on both reader paths instead of OOM. Assisted-by: GitHub Copilot:claude-opus-4.8 --- .../avro/generic/GenericDatumReader.java | 6 +- .../org/apache/avro/io/FastReaderBuilder.java | 37 ++++++++---- .../avro/generic/TestGenericDatumReader.java | 60 +++++++++++++++++++ 3 files changed, 88 insertions(+), 15 deletions(-) diff --git a/lang/java/avro/src/main/java/org/apache/avro/generic/GenericDatumReader.java b/lang/java/avro/src/main/java/org/apache/avro/generic/GenericDatumReader.java index 8773904460f..d919d3c1e78 100644 --- a/lang/java/avro/src/main/java/org/apache/avro/generic/GenericDatumReader.java +++ b/lang/java/avro/src/main/java/org/apache/avro/generic/GenericDatumReader.java @@ -509,9 +509,11 @@ private static int minBytesPerElement(Schema schema, Set visited) { * reports fewer remaining bytes than required. *

    * This check prevents out-of-memory errors from pre-allocating huge backing - * arrays when the source data is truncated or malicious. + * arrays when the source data is truncated or malicious. It is exposed so the + * fast reader ({@code FastReaderBuilder}) can apply the same guard as this + * classic reader. */ - private static void ensureAvailableCollectionBytes(Decoder decoder, long count, Schema elementSchema) + public static void ensureAvailableCollectionBytes(Decoder decoder, long count, Schema elementSchema) throws EOFException { if (count <= 0) { return; diff --git a/lang/java/avro/src/main/java/org/apache/avro/io/FastReaderBuilder.java b/lang/java/avro/src/main/java/org/apache/avro/io/FastReaderBuilder.java index 7c03d76eec6..9c5660948f5 100644 --- a/lang/java/avro/src/main/java/org/apache/avro/io/FastReaderBuilder.java +++ b/lang/java/avro/src/main/java/org/apache/avro/io/FastReaderBuilder.java @@ -470,22 +470,21 @@ private Optional> findClass(String clazz) { @SuppressWarnings("unchecked") private FieldReader createArrayReader(Schema readerSchema, Container action) throws IOException { FieldReader elementReader = getReaderFor(action.elementAction, null); + Schema elementType = readerSchema.getElementType(); // Elements whose schema encodes to zero bytes (null, or a record with only // zero-byte fields) consume no input, so the block count cannot be bounded by // the bytes remaining in the stream. Cap such collections against a heap-aware // limit so a tiny payload cannot declare a huge block count and drive an // unbounded backing-array allocation (AVRO-4300). - boolean zeroByteElements = GenericDatumReader.isZeroByteSchema(readerSchema.getElementType()); + boolean zeroByteElements = GenericDatumReader.isZeroByteSchema(elementType); return reusingReader((reuse, decoder) -> { if (reuse instanceof GenericArray) { GenericArray reuseArray = (GenericArray) reuse; long l = decoder.readArrayStart(); long total = 0; - if (zeroByteElements && l > 0) { - SystemLimitException.checkMaxCollectionAllocation(total, l); - } + checkArrayBlock(decoder, elementType, zeroByteElements, total, l); reuseArray.clear(); while (l > 0) { @@ -494,17 +493,13 @@ private FieldReader createArrayReader(Schema readerSchema, Container action) thr } total += l; l = decoder.arrayNext(); - if (zeroByteElements && l > 0) { - SystemLimitException.checkMaxCollectionAllocation(total, l); - } + checkArrayBlock(decoder, elementType, zeroByteElements, total, l); } return reuseArray; } else { long l = decoder.readArrayStart(); long total = 0; - if (zeroByteElements && l > 0) { - SystemLimitException.checkMaxCollectionAllocation(total, l); - } + checkArrayBlock(decoder, elementType, zeroByteElements, total, l); List array = (reuse instanceof List) ? (List) reuse : new GenericData.Array<>((int) l, readerSchema); array.clear(); @@ -514,15 +509,31 @@ private FieldReader createArrayReader(Schema readerSchema, Container action) thr } total += l; l = decoder.arrayNext(); - if (zeroByteElements && l > 0) { - SystemLimitException.checkMaxCollectionAllocation(total, l); - } + checkArrayBlock(decoder, elementType, zeroByteElements, total, l); } return array; } }); } + /** + * Validates an array block count before its elements are allocated, applying + * the same guards as the classic {@code GenericDatumReader}: the + * bytes-remaining check for elements with a positive minimum size, and the + * heap-aware allocation cap for zero-byte elements (which the bytes check + * cannot bound). + */ + private static void checkArrayBlock(Decoder decoder, Schema elementType, boolean zeroByteElements, long total, + long count) throws IOException { + if (count <= 0) { + return; + } + GenericDatumReader.ensureAvailableCollectionBytes(decoder, count, elementType); + if (zeroByteElements) { + SystemLimitException.checkMaxCollectionAllocation(total, count); + } + } + private FieldReader createEnumReader(EnumAdjust action) { return reusingReader((reuse, decoder) -> { int index = decoder.readEnum(); diff --git a/lang/java/avro/src/test/java/org/apache/avro/generic/TestGenericDatumReader.java b/lang/java/avro/src/test/java/org/apache/avro/generic/TestGenericDatumReader.java index db91bb08b6e..06420715bc2 100644 --- a/lang/java/avro/src/test/java/org/apache/avro/generic/TestGenericDatumReader.java +++ b/lang/java/avro/src/test/java/org/apache/avro/generic/TestGenericDatumReader.java @@ -25,6 +25,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Random; @@ -383,4 +384,63 @@ void arrayOfAllNullRecordsRejectsCountAboveAllocationLimit() throws Exception { org.apache.avro.TestSystemLimitException.resetLimits(); } } + + private static GenericDatumReader arrayReader(Schema elementType, boolean fastReader) { + Schema schema = Schema.createArray(elementType); + GenericData data = new GenericData(); + data.setFastReaderEnabled(fastReader); + return new GenericDatumReader<>(schema, schema, data); + } + + /** + * The fast reader (the default decode path) must apply the same guards as the + * classic reader. A non-zero-byte element array with a huge count and no data + * must be rejected by the bytes-remaining check rather than pre-allocating a + * huge backing array. + */ + @Test + void fastAndClassicReaderRejectHugeNonZeroByteArray() throws Exception { + for (boolean fast : new boolean[] { true, false }) { + GenericDatumReader reader = arrayReader(Schema.create(Schema.Type.LONG), fast); + byte[] data = encodeVarints(200_000_000L, 0L); + BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(data, null); + assertThrows(EOFException.class, () -> reader.read(null, decoder), "fastReader=" + fast); + } + } + + /** + * The heap-aware zero-byte allocation cap is enforced on both the fast and the + * classic reader path. + */ + @Test + void fastAndClassicReaderRejectHugeNullArray() throws Exception { + System.setProperty(SystemLimitException.MAX_COLLECTION_ALLOCATION_PROPERTY, "1000"); + org.apache.avro.TestSystemLimitException.resetLimits(); + try { + for (boolean fast : new boolean[] { true, false }) { + GenericDatumReader reader = arrayReader(Schema.create(Schema.Type.NULL), fast); + byte[] data = encodeVarints(200_000L, 0L); + BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(data, null); + assertThrows(SystemLimitException.class, () -> reader.read(null, decoder), "fastReader=" + fast); + } + } finally { + System.clearProperty(SystemLimitException.MAX_COLLECTION_ALLOCATION_PROPERTY); + org.apache.avro.TestSystemLimitException.resetLimits(); + } + } + + /** + * A legitimate array of nulls within the limit still decodes on both reader + * paths. + */ + @Test + void fastAndClassicReaderDecodeSmallNullArray() throws Exception { + for (boolean fast : new boolean[] { true, false }) { + GenericDatumReader reader = arrayReader(Schema.create(Schema.Type.NULL), fast); + byte[] data = encodeVarints(1000L, 0L); + BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(data, null); + Collection result = (Collection) reader.read(null, decoder); + assertEquals(1000, result.size(), "fastReader=" + fast); + } + } } From 57a247f8cc631203aa81a3c10ed4badac489496f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Mej=C3=ADa?= Date: Sun, 12 Jul 2026 13:06:31 +0200 Subject: [PATCH 03/25] AVRO-4300: [java] Cover all collection/reader combinations in tests Add a matrix test asserting every collection kind is rejected (never OOM) with a huge block count and no data, on both the fast (default) and classic reader: array via the heap-aware allocation cap (SystemLimitException) and array, array, map, map via the bytes-remaining check (EOFException) -- the full 5x2 set of combinations. Keep a cumulative multi-block null test and a positive within-limit decode test on both readers. Assisted-by: GitHub Copilot:claude-opus-4.8 --- .../avro/generic/TestGenericDatumReader.java | 64 ++++++++++++++----- 1 file changed, 49 insertions(+), 15 deletions(-) diff --git a/lang/java/avro/src/test/java/org/apache/avro/generic/TestGenericDatumReader.java b/lang/java/avro/src/test/java/org/apache/avro/generic/TestGenericDatumReader.java index 06420715bc2..e7f24188a9f 100644 --- a/lang/java/avro/src/test/java/org/apache/avro/generic/TestGenericDatumReader.java +++ b/lang/java/avro/src/test/java/org/apache/avro/generic/TestGenericDatumReader.java @@ -386,40 +386,74 @@ void arrayOfAllNullRecordsRejectsCountAboveAllocationLimit() throws Exception { } private static GenericDatumReader arrayReader(Schema elementType, boolean fastReader) { - Schema schema = Schema.createArray(elementType); + return readerFor(Schema.createArray(elementType), fastReader); + } + + private static GenericDatumReader readerFor(Schema schema, boolean fastReader) { GenericData data = new GenericData(); data.setFastReaderEnabled(fastReader); return new GenericDatumReader<>(schema, schema, data); } /** - * The fast reader (the default decode path) must apply the same guards as the - * classic reader. A non-zero-byte element array with a huge count and no data - * must be rejected by the bytes-remaining check rather than pre-allocating a - * huge backing array. + * Full matrix: every collection kind must be rejected (never OOM) with a huge + * declared block count and no element data, on both the fast (default) and the + * classic reader. Zero-byte-element arrays are bounded by the heap-aware + * allocation cap (SystemLimitException); every other kind is bounded by the + * bytes-remaining check (EOFException). Maps always carry a >=1-byte key so + * they fall in the latter group regardless of the value type. */ @Test - void fastAndClassicReaderRejectHugeNonZeroByteArray() throws Exception { - for (boolean fast : new boolean[] { true, false }) { - GenericDatumReader reader = arrayReader(Schema.create(Schema.Type.LONG), fast); - byte[] data = encodeVarints(200_000_000L, 0L); - BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(data, null); - assertThrows(EOFException.class, () -> reader.read(null, decoder), "fastReader=" + fast); + void hugeCollectionsRejectedOnBothReaderPaths() throws Exception { + // element/value type -> expected exception when the count is huge and no data + Schema nullType = Schema.create(Schema.Type.NULL); + Schema longType = Schema.create(Schema.Type.LONG); + Schema intType = Schema.create(Schema.Type.INT); + + // (schema, expected exception). array is the only zero-byte case here. + Object[][] cases = { { Schema.createArray(nullType), SystemLimitException.class }, + { Schema.createArray(longType), EOFException.class }, { Schema.createArray(intType), EOFException.class }, + { Schema.createMap(nullType), EOFException.class }, { Schema.createMap(longType), EOFException.class }, }; + + // Small allocation limit so the zero-byte array case is rejected + // deterministically regardless of the test JVM heap size. + System.setProperty(SystemLimitException.MAX_COLLECTION_ALLOCATION_PROPERTY, "1000"); + org.apache.avro.TestSystemLimitException.resetLimits(); + try { + for (Object[] c : cases) { + Schema schema = (Schema) c[0]; + @SuppressWarnings("unchecked") + Class expected = (Class) c[1]; + for (boolean fast : new boolean[] { true, false }) { + GenericDatumReader reader = readerFor(schema, fast); + byte[] data = encodeVarints(200_000_000L, 0L); + BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(data, null); + assertThrows(expected, () -> reader.read(null, decoder), () -> schema + " fast=" + fast); + } + } + } finally { + System.clearProperty(SystemLimitException.MAX_COLLECTION_ALLOCATION_PROPERTY); + org.apache.avro.TestSystemLimitException.resetLimits(); } } /** - * The heap-aware zero-byte allocation cap is enforced on both the fast and the - * classic reader path. + * The fast reader (the default decode path) must apply the same guards as the + * classic reader. A non-zero-byte element array with a huge count and no data + * must be rejected by the bytes-remaining check rather than pre-allocating a + * huge backing array. Also verifies the cumulative allocation cap across + * multiple blocks for the zero-byte case, which the single-block matrix does + * not exercise. */ @Test - void fastAndClassicReaderRejectHugeNullArray() throws Exception { + void fastAndClassicReaderRejectCumulativeNullBlocks() throws Exception { System.setProperty(SystemLimitException.MAX_COLLECTION_ALLOCATION_PROPERTY, "1000"); org.apache.avro.TestSystemLimitException.resetLimits(); try { for (boolean fast : new boolean[] { true, false }) { GenericDatumReader reader = arrayReader(Schema.create(Schema.Type.NULL), fast); - byte[] data = encodeVarints(200_000L, 0L); + // Two blocks of 600 nulls each (1200 > 1000) must throw on the second. + byte[] data = encodeVarints(600L, 600L, 0L); BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(data, null); assertThrows(SystemLimitException.class, () -> reader.read(null, decoder), "fastReader=" + fast); } From eee0f0a110719741d70e3d868cf008aee1971baf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Mej=C3=ADa?= Date: Sun, 12 Jul 2026 13:16:38 +0200 Subject: [PATCH 04/25] AVRO-4300: [java] Test negative block count is bounded on both readers The C and Python collection-limit tests cover a negative block count (abs(count) zero-byte elements preceded by a block byte-size); the Java tests did not. The decoder normalizes the negative count to a positive one, which must still be bounded by the heap-aware allocation cap. Add a test asserting this on both the fast and classic reader paths. Assisted-by: GitHub Copilot:claude-opus-4.8 --- .../avro/generic/TestGenericDatumReader.java | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/lang/java/avro/src/test/java/org/apache/avro/generic/TestGenericDatumReader.java b/lang/java/avro/src/test/java/org/apache/avro/generic/TestGenericDatumReader.java index e7f24188a9f..bf02dcd1fd1 100644 --- a/lang/java/avro/src/test/java/org/apache/avro/generic/TestGenericDatumReader.java +++ b/lang/java/avro/src/test/java/org/apache/avro/generic/TestGenericDatumReader.java @@ -463,6 +463,31 @@ void fastAndClassicReaderRejectCumulativeNullBlocks() throws Exception { } } + /** + * A negative block count encodes {@code abs(count)} zero-byte elements preceded + * by a block byte-size; the decoder normalizes it to a positive count, which + * must still be bounded by the allocation cap on both reader paths (matching + * the negative-block-count coverage of the C and Python SDKs). + */ + @Test + void fastAndClassicReaderRejectNegativeNullBlockCount() throws Exception { + System.setProperty(SystemLimitException.MAX_COLLECTION_ALLOCATION_PROPERTY, "1000"); + org.apache.avro.TestSystemLimitException.resetLimits(); + try { + for (boolean fast : new boolean[] { true, false }) { + GenericDatumReader reader = arrayReader(Schema.create(Schema.Type.NULL), fast); + // -200000 items (zigzag negative), followed by a block byte-size of 0, + // then the end-of-array terminator. Normalized to 200000 > 1000. + byte[] data = encodeVarints(-200_000L, 0L, 0L); + BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(data, null); + assertThrows(SystemLimitException.class, () -> reader.read(null, decoder), "fastReader=" + fast); + } + } finally { + System.clearProperty(SystemLimitException.MAX_COLLECTION_ALLOCATION_PROPERTY); + org.apache.avro.TestSystemLimitException.resetLimits(); + } + } + /** * A legitimate array of nulls within the limit still decodes on both reader * paths. From 67ae8148caf0176f5327ef01c49d0932b7422398 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Mej=C3=ADa?= Date: Sun, 12 Jul 2026 13:20:37 +0200 Subject: [PATCH 05/25] AVRO-4300: [java] Test Long.MIN_VALUE block count is handled safely Mirror the C SDK's INT64_MIN edge case. Long.MIN_VALUE as a block count is the pathological overflow: negating it overflows back to a negative value. Java's decoder normalizes this to an empty collection (no allocation) rather than rejecting it as the C SDK does, which is equally non-exploitable. Add a test asserting the safe, allocation-free result on both the fast and classic reader paths. Assisted-by: GitHub Copilot:claude-opus-4.8 --- .../avro/generic/TestGenericDatumReader.java | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/lang/java/avro/src/test/java/org/apache/avro/generic/TestGenericDatumReader.java b/lang/java/avro/src/test/java/org/apache/avro/generic/TestGenericDatumReader.java index bf02dcd1fd1..a51fdf75210 100644 --- a/lang/java/avro/src/test/java/org/apache/avro/generic/TestGenericDatumReader.java +++ b/lang/java/avro/src/test/java/org/apache/avro/generic/TestGenericDatumReader.java @@ -488,6 +488,27 @@ void fastAndClassicReaderRejectNegativeNullBlockCount() throws Exception { } } + /** + * {@code Long.MIN_VALUE} as a block count is the pathological overflow case: + * negating it overflows back to a negative value. It must be handled safely + * without allocating -- the decoder normalizes it to an empty collection rather + * than a huge one -- on both reader paths. (The C SDK rejects it outright; this + * normalization is equally non-exploitable.) + */ + @Test + void fastAndClassicReaderHandleMinValueBlockCountSafely() throws Exception { + for (boolean fast : new boolean[] { true, false }) { + GenericDatumReader reader = arrayReader(Schema.create(Schema.Type.NULL), fast); + // Long.MIN_VALUE items (zigzag), a block byte-size of 0, then the + // end-of-array terminator. Negating Long.MIN_VALUE overflows, so this must + // not drive an allocation. + byte[] data = encodeVarints(Long.MIN_VALUE, 0L, 0L); + BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(data, null); + Collection result = (Collection) reader.read(null, decoder); + assertEquals(0, result.size(), "fastReader=" + fast); + } + } + /** * A legitimate array of nulls within the limit still decodes on both reader * paths. From ac0065027ca3a867b99ccf81d4ab919f709ef26b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Mej=C3=ADa?= Date: Sun, 12 Jul 2026 14:56:28 +0200 Subject: [PATCH 06/25] AVRO-4300: [java] Bound the array/map skip path The skip path was unbounded: BinaryDecoder.skipArray()/skipMap() returned doSkipItems() without calling checkMaxCollectionLength, and GenericDatumReader.skip() looped over the returned count. Skipping a huge block of zero-byte elements (e.g. a writer array field absent from the reader schema, skipped during projection) could therefore loop unboundedly -- a CPU exhaustion even though skipping reads and allocates nothing. Two complementary bounds: - BinaryDecoder.skipArray()/skipMap() now apply the structural collection cap (checkMaxCollectionLength), mirroring readArrayStart()/readMapStart() and covering the resolving-decoder projection skip path on both reader types. - GenericDatumReader.skip() additionally bounds the cumulative count, using the heap-aware allocation cap for zero-byte element arrays and the structural cap otherwise, matching the read path and the other language SDKs. Assisted-by: GitHub Copilot:claude-opus-4.8 --- .../avro/generic/GenericDatumReader.java | 16 ++++ .../org/apache/avro/io/BinaryDecoder.java | 4 +- .../avro/generic/TestGenericDatumReader.java | 75 +++++++++++++++++++ 3 files changed, 93 insertions(+), 2 deletions(-) diff --git a/lang/java/avro/src/main/java/org/apache/avro/generic/GenericDatumReader.java b/lang/java/avro/src/main/java/org/apache/avro/generic/GenericDatumReader.java index d919d3c1e78..c0ac5b26553 100644 --- a/lang/java/avro/src/main/java/org/apache/avro/generic/GenericDatumReader.java +++ b/lang/java/avro/src/main/java/org/apache/avro/generic/GenericDatumReader.java @@ -777,7 +777,19 @@ public static void skip(Schema schema, Decoder in) throws IOException { break; case ARRAY: Schema elementType = schema.getElementType(); + // Bound the cumulative element count: skipping a huge block of zero-byte + // elements (e.g. null) would otherwise loop unboundedly (a CPU + // exhaustion) even though it reads nothing. Zero-byte elements use the + // heap-aware allocation cap; others the structural collection cap. + boolean zeroByteElements = isZeroByteSchema(elementType); + long arrayTotal = 0; for (long l = in.skipArray(); l > 0; l = in.skipArray()) { + if (zeroByteElements) { + SystemLimitException.checkMaxCollectionAllocation(arrayTotal, l); + } else { + SystemLimitException.checkMaxCollectionLength(arrayTotal, l); + } + arrayTotal += l; for (long i = 0; i < l; i++) { skip(elementType, in); } @@ -785,7 +797,11 @@ public static void skip(Schema schema, Decoder in) throws IOException { break; case MAP: Schema value = schema.getValueType(); + // Map entries always carry a >= 1 byte key, so the structural cap applies. + long mapTotal = 0; for (long l = in.skipMap(); l > 0; l = in.skipMap()) { + SystemLimitException.checkMaxCollectionLength(mapTotal, l); + mapTotal += l; for (long i = 0; i < l; i++) { in.skipString(); skip(value, in); diff --git a/lang/java/avro/src/main/java/org/apache/avro/io/BinaryDecoder.java b/lang/java/avro/src/main/java/org/apache/avro/io/BinaryDecoder.java index 77fc8490764..d2f049c55d4 100644 --- a/lang/java/avro/src/main/java/org/apache/avro/io/BinaryDecoder.java +++ b/lang/java/avro/src/main/java/org/apache/avro/io/BinaryDecoder.java @@ -458,7 +458,7 @@ public long arrayNext() throws IOException { @Override public long skipArray() throws IOException { - return doSkipItems(); + return SystemLimitException.checkMaxCollectionLength(doSkipItems()); } @Override @@ -476,7 +476,7 @@ public long mapNext() throws IOException { @Override public long skipMap() throws IOException { - return doSkipItems(); + return SystemLimitException.checkMaxCollectionLength(doSkipItems()); } @Override diff --git a/lang/java/avro/src/test/java/org/apache/avro/generic/TestGenericDatumReader.java b/lang/java/avro/src/test/java/org/apache/avro/generic/TestGenericDatumReader.java index a51fdf75210..c6f072b1262 100644 --- a/lang/java/avro/src/test/java/org/apache/avro/generic/TestGenericDatumReader.java +++ b/lang/java/avro/src/test/java/org/apache/avro/generic/TestGenericDatumReader.java @@ -523,4 +523,79 @@ void fastAndClassicReaderDecodeSmallNullArray() throws Exception { assertEquals(1000, result.size(), "fastReader=" + fast); } } + + /** + * Skipping a huge zero-byte array (e.g. during schema projection) is bounded by + * the heap-aware allocation cap, so it cannot loop unboundedly even though it + * reads nothing. + */ + @Test + void skipArrayOfNullRejectsHugeCount() throws Exception { + System.setProperty(SystemLimitException.MAX_COLLECTION_ALLOCATION_PROPERTY, "1000"); + org.apache.avro.TestSystemLimitException.resetLimits(); + try { + Schema schema = Schema.createArray(Schema.create(Schema.Type.NULL)); + byte[] data = encodeVarints(200_000L, 0L); + BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(data, null); + assertThrows(SystemLimitException.class, () -> GenericDatumReader.skip(schema, decoder)); + } finally { + System.clearProperty(SystemLimitException.MAX_COLLECTION_ALLOCATION_PROPERTY); + org.apache.avro.TestSystemLimitException.resetLimits(); + } + } + + /** + * A legitimate small zero-byte array is skipped without error. + */ + @Test + void skipSmallNullArraySucceeds() throws Exception { + Schema schema = Schema.createArray(Schema.create(Schema.Type.NULL)); + // 1000 nulls then the end marker, followed by a trailing long we can read to + // confirm the skip advanced correctly. + byte[] data = encodeVarints(1000L, 0L, 42L); + BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(data, null); + GenericDatumReader.skip(schema, decoder); + assertEquals(42L, decoder.readLong()); + } + + /** + * Skipping a huge map is bounded by the structural collection cap. + */ + @Test + void skipMapRejectsHugeCount() throws Exception { + Schema schema = Schema.createMap(Schema.create(Schema.Type.NULL)); + // A single block declaring more than Integer.MAX_VALUE - 8 entries. + byte[] data = encodeVarints((long) Integer.MAX_VALUE, 0L); + BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(data, null); + assertThrows(RuntimeException.class, () -> GenericDatumReader.skip(schema, decoder)); + } + + /** + * A writer array field that the reader schema omits is skipped during + * resolution through the decoder's skipArray; a huge count must be bounded on + * both the fast and classic reader paths. + */ + @Test + void resolvingSkipOfHugeNullArrayFieldIsBounded() throws Exception { + System.setProperty(SystemLimitException.MAX_COLLECTION_LENGTH_PROPERTY, "1000"); + org.apache.avro.TestSystemLimitException.resetLimits(); + try { + Schema writer = new Schema.Parser().parse("{\"type\":\"record\",\"name\":\"R\",\"fields\":[" + + "{\"name\":\"arr\",\"type\":{\"type\":\"array\",\"items\":\"null\"}}," + + "{\"name\":\"a\",\"type\":\"long\"}]}"); + Schema reader = new Schema.Parser() + .parse("{\"type\":\"record\",\"name\":\"R\",\"fields\":[{\"name\":\"a\",\"type\":\"long\"}]}"); + byte[] data = encodeVarints(2000L, 0L, 42L); + for (boolean fast : new boolean[] { true, false }) { + GenericData data2 = new GenericData(); + data2.setFastReaderEnabled(fast); + GenericDatumReader r = new GenericDatumReader<>(writer, reader, data2); + BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(data, null); + assertThrows(RuntimeException.class, () -> r.read(null, decoder), "fastReader=" + fast); + } + } finally { + System.clearProperty(SystemLimitException.MAX_COLLECTION_LENGTH_PROPERTY); + org.apache.avro.TestSystemLimitException.resetLimits(); + } + } } From f2c686f745f037b2dd0800dfe840d59aa4a6982d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Mej=C3=ADa?= Date: Sun, 12 Jul 2026 18:59:13 +0200 Subject: [PATCH 07/25] AVRO-4300: [java] Fix limit Javadoc markup; clamp allocation cap to VM limit Addresses review feedback: - Fix the malformed
  • markup in the limit-properties list (each item closed
    • prematurely after the tag) and correct the bytes property name (org.apache.avro.limits.bytes.maxLength) so the Javadoc renders correctly. - Clamp maxCollectionAllocation to MAX_ARRAY_VM_LIMIT when refreshing limits, so a configured (or large-heap-derived) zero-byte allocation cap stays consistent with the other collection caps and cannot exceed the VM array ceiling. Assisted-by: GitHub Copilot:claude-opus-4.8 --- .../org/apache/avro/SystemLimitException.java | 26 +++++++++++-------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/lang/java/avro/src/main/java/org/apache/avro/SystemLimitException.java b/lang/java/avro/src/main/java/org/apache/avro/SystemLimitException.java index 33fd97531bf..cb30a52ea74 100644 --- a/lang/java/avro/src/main/java/org/apache/avro/SystemLimitException.java +++ b/lang/java/avro/src/main/java/org/apache/avro/SystemLimitException.java @@ -28,19 +28,19 @@ * The following system properties can be set to limit the size of bytes, * strings and collection types to be allocated: *
      - *
    • org.apache.avro.limits.byte.maxLength
      • limits the maximum - * size of byte types. - *
    • org.apache.avro.limits.collectionItems.maxLength
      • limits the + *
    • org.apache.avro.limits.bytes.maxLength limits the maximum size + * of bytes types.
      • + *
    • org.apache.avro.limits.collectionItems.maxLength limits the * maximum number of map and list items that can be read at * once single sequence.
      • - *
    • org.apache.avro.limits.string.maxLength
      • limits the maximum - * size of string types. - *
    • org.apache.avro.limits.collectionItems.maxAllocation
      • limits - * the number of array elements whose schema encodes to zero bytes - * (such as null or a self-referencing record) that may be allocated at - * once. Unlike other element types, these cannot be bounded by the number of - * bytes remaining in the stream, so the limit defaults to a fraction of the - * maximum heap. + *
    • org.apache.avro.limits.string.maxLength limits the maximum size + * of string types.
      • + *
    • org.apache.avro.limits.collectionItems.maxAllocation limits the + * number of array elements whose schema encodes to zero bytes (such as + * null or a self-referencing record) that may be allocated at once. + * Unlike other element types, these cannot be bounded by the number of bytes + * remaining in the stream, so the limit defaults to a fraction of the maximum + * heap.
      • *
    * * The default is to permit sizes up to {@link #MAX_ARRAY_VM_LIMIT}. @@ -375,5 +375,9 @@ static void resetLimits() { maxStringLength = getLimitFromProperty(MAX_STRING_LENGTH_PROPERTY, MAX_ARRAY_VM_LIMIT); maxCollectionAllocation = getLongLimitFromProperty(MAX_COLLECTION_ALLOCATION_PROPERTY, defaultMaxCollectionAllocation()); + // A collection cannot hold more than MAX_ARRAY_VM_LIMIT elements, so keep the + // zero-byte allocation cap consistent with the other collection limits even + // when it is configured (or derived from a very large heap) above that. + maxCollectionAllocation = Math.min(maxCollectionAllocation, MAX_ARRAY_VM_LIMIT); } } From e47f206f4dee330feac736e0ff81c775d5fff080 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Mej=C3=ADa?= Date: Sun, 12 Jul 2026 20:25:29 +0200 Subject: [PATCH 08/25] AVRO-4300: [java] Fix limit Javadoc grammar; make heap-cap test deterministic Addresses review feedback: - Javadoc: "read at once single sequence" -> "read in a single sequence". - testCheckMaxCollectionAllocation asserts with MAX_ARRAY_VM_LIMIT + 1 instead of Integer.MAX_VALUE - 8. Since the default allocation cap is derived from the heap and then clamped to MAX_ARRAY_VM_LIMIT, a value equal to that limit may not exceed the computed default on a very large heap; +1 guarantees it does, keeping the test deterministic across environments. Assisted-by: GitHub Copilot:claude-opus-4.8 --- .../src/main/java/org/apache/avro/SystemLimitException.java | 4 ++-- .../test/java/org/apache/avro/TestSystemLimitException.java | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/lang/java/avro/src/main/java/org/apache/avro/SystemLimitException.java b/lang/java/avro/src/main/java/org/apache/avro/SystemLimitException.java index cb30a52ea74..32d978ea4e8 100644 --- a/lang/java/avro/src/main/java/org/apache/avro/SystemLimitException.java +++ b/lang/java/avro/src/main/java/org/apache/avro/SystemLimitException.java @@ -31,8 +31,8 @@ *
  • org.apache.avro.limits.bytes.maxLength limits the maximum size * of bytes types.
    • *
  • org.apache.avro.limits.collectionItems.maxLength limits the - * maximum number of map and list items that can be read at - * once single sequence.
    • + * maximum number of map and list items that can be read in a + * single sequence. *
  • org.apache.avro.limits.string.maxLength limits the maximum size * of string types.
    • *
  • org.apache.avro.limits.collectionItems.maxAllocation limits the diff --git a/lang/java/avro/src/test/java/org/apache/avro/TestSystemLimitException.java b/lang/java/avro/src/test/java/org/apache/avro/TestSystemLimitException.java index 7477ce58dc4..a0b59f55e27 100644 --- a/lang/java/avro/src/test/java/org/apache/avro/TestSystemLimitException.java +++ b/lang/java/avro/src/test/java/org/apache/avro/TestSystemLimitException.java @@ -148,7 +148,10 @@ void testCheckMaxCollectionAllocationDefaultsToHeapFraction() { resetLimits(); assertEquals(1024L, checkMaxCollectionAllocation(0L, 1024L)); // A pathologically large zero-byte collection is rejected without allocating. - assertThrows(SystemLimitException.class, () -> checkMaxCollectionAllocation(0L, (long) Integer.MAX_VALUE - 8)); + // Use MAX_ARRAY_VM_LIMIT + 1 so it exceeds the cap regardless of heap size + // (the default is derived from the heap and then clamped to MAX_ARRAY_VM_LIMIT, + // so Integer.MAX_VALUE - 8 alone would not exceed it on a very large heap). + assertThrows(SystemLimitException.class, () -> checkMaxCollectionAllocation(0L, (long) MAX_ARRAY_VM_LIMIT + 1L)); } @Test From 6bf53e35d622ff51b3c2d46a7daf739da3cab828 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Mej=C3=ADa?= Date: Sun, 12 Jul 2026 20:56:03 +0200 Subject: [PATCH 09/25] AVRO-4300: [java] Reword zero-byte Javadoc; tighten skip test assertions Addresses review feedback: - Javadoc: "a self-referencing record" is not inherently zero-byte (the code only computes a 0-byte minimum for some recursive schemas to break recursion). Reword the three occurrences to describe actual zero-byte encodings: null, a zero-length fixed, or a record whose fields all encode to zero bytes. - skipMapRejectsHugeCount now asserts UnsupportedOperationException (a count of Integer.MAX_VALUE deterministically hits the VM structural-limit path), and resolvingSkipOfHugeNullArrayFieldIsBounded asserts SystemLimitException, so the tests pin the intended exception rather than a generic RuntimeException. Assisted-by: GitHub Copilot:claude-opus-4.8 --- .../java/org/apache/avro/SystemLimitException.java | 11 +++++++---- .../apache/avro/generic/TestGenericDatumReader.java | 6 ++++-- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/lang/java/avro/src/main/java/org/apache/avro/SystemLimitException.java b/lang/java/avro/src/main/java/org/apache/avro/SystemLimitException.java index 32d978ea4e8..368736c056a 100644 --- a/lang/java/avro/src/main/java/org/apache/avro/SystemLimitException.java +++ b/lang/java/avro/src/main/java/org/apache/avro/SystemLimitException.java @@ -37,7 +37,8 @@ * of string types.
    • *
  • org.apache.avro.limits.collectionItems.maxAllocation limits the * number of array elements whose schema encodes to zero bytes (such as - * null or a self-referencing record) that may be allocated at once. + * null, a zero-length fixed, or a record whose fields all + * encode to zero bytes) that may be allocated at once. * Unlike other element types, these cannot be bounded by the number of bytes * remaining in the stream, so the limit defaults to a fraction of the maximum * heap.
    • @@ -91,7 +92,8 @@ public class SystemLimitException extends AvroRuntimeException { /** * System property declaring the maximum number of zero-byte-encoded array - * elements (e.g. {@code null} or a self-referencing record) to allocate at + * elements (e.g. {@code null}, a zero-length fixed, or a record whose fields + * all encode to zero bytes) to allocate at * once: {@value}. */ public static final String MAX_COLLECTION_ALLOCATION_PROPERTY = "org.apache.avro.limits.collectionItems.maxAllocation"; @@ -295,8 +297,9 @@ public static int checkMaxCollectionLength(long items) { * Check to ensure that allocating storage for the specified number of * zero-byte-encoded array elements remains within the heap-aware limit. *

    - * Elements whose schema encodes to zero bytes (e.g. {@code null} or a - * self-referencing record) consume no input bytes, so the number that may be + * Elements whose schema encodes to zero bytes (e.g. {@code null}, a + * zero-length fixed, or a record whose fields all encode to zero bytes) + * consume no input bytes, so the number that may be * declared is not bounded by the bytes remaining in the stream. Without a cap, * a tiny payload can declare an enormous block count and drive an unbounded * backing-array allocation. This limit is derived from the maximum heap (see diff --git a/lang/java/avro/src/test/java/org/apache/avro/generic/TestGenericDatumReader.java b/lang/java/avro/src/test/java/org/apache/avro/generic/TestGenericDatumReader.java index c6f072b1262..d58a532308b 100644 --- a/lang/java/avro/src/test/java/org/apache/avro/generic/TestGenericDatumReader.java +++ b/lang/java/avro/src/test/java/org/apache/avro/generic/TestGenericDatumReader.java @@ -567,7 +567,9 @@ void skipMapRejectsHugeCount() throws Exception { // A single block declaring more than Integer.MAX_VALUE - 8 entries. byte[] data = encodeVarints((long) Integer.MAX_VALUE, 0L); BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(data, null); - assertThrows(RuntimeException.class, () -> GenericDatumReader.skip(schema, decoder)); + // Integer.MAX_VALUE exceeds MAX_ARRAY_VM_LIMIT, so this hits the VM + // structural-limit path (an UnsupportedOperationException). + assertThrows(UnsupportedOperationException.class, () -> GenericDatumReader.skip(schema, decoder)); } /** @@ -591,7 +593,7 @@ void resolvingSkipOfHugeNullArrayFieldIsBounded() throws Exception { data2.setFastReaderEnabled(fast); GenericDatumReader r = new GenericDatumReader<>(writer, reader, data2); BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(data, null); - assertThrows(RuntimeException.class, () -> r.read(null, decoder), "fastReader=" + fast); + assertThrows(SystemLimitException.class, () -> r.read(null, decoder), "fastReader=" + fast); } } finally { System.clearProperty(SystemLimitException.MAX_COLLECTION_LENGTH_PROPERTY); From d6fc66ce7e84fd1fefc2fec8c5283290897c2ef0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Mej=C3=ADa?= Date: Sun, 12 Jul 2026 21:19:04 +0200 Subject: [PATCH 10/25] AVRO-4300: [java] Apply spotless formatting to reworded Javadoc The zero-byte Javadoc rewording did not match the Eclipse formatter's line-wrapping, failing the spotless check. Reflowed via `mvn spotless:apply`. Assisted-by: GitHub Copilot:claude-opus-4.8 --- .../org/apache/avro/SystemLimitException.java | 22 +++++++++---------- 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/lang/java/avro/src/main/java/org/apache/avro/SystemLimitException.java b/lang/java/avro/src/main/java/org/apache/avro/SystemLimitException.java index 368736c056a..19b39364f57 100644 --- a/lang/java/avro/src/main/java/org/apache/avro/SystemLimitException.java +++ b/lang/java/avro/src/main/java/org/apache/avro/SystemLimitException.java @@ -38,10 +38,9 @@ *
  • org.apache.avro.limits.collectionItems.maxAllocation limits the * number of array elements whose schema encodes to zero bytes (such as * null, a zero-length fixed, or a record whose fields all - * encode to zero bytes) that may be allocated at once. - * Unlike other element types, these cannot be bounded by the number of bytes - * remaining in the stream, so the limit defaults to a fraction of the maximum - * heap.
    • + * encode to zero bytes) that may be allocated at once. Unlike other element + * types, these cannot be bounded by the number of bytes remaining in the + * stream, so the limit defaults to a fraction of the maximum heap. * * * The default is to permit sizes up to {@link #MAX_ARRAY_VM_LIMIT}. @@ -93,8 +92,7 @@ public class SystemLimitException extends AvroRuntimeException { /** * System property declaring the maximum number of zero-byte-encoded array * elements (e.g. {@code null}, a zero-length fixed, or a record whose fields - * all encode to zero bytes) to allocate at - * once: {@value}. + * all encode to zero bytes) to allocate at once: {@value}. */ public static final String MAX_COLLECTION_ALLOCATION_PROPERTY = "org.apache.avro.limits.collectionItems.maxAllocation"; @@ -297,12 +295,12 @@ public static int checkMaxCollectionLength(long items) { * Check to ensure that allocating storage for the specified number of * zero-byte-encoded array elements remains within the heap-aware limit. *

    - * Elements whose schema encodes to zero bytes (e.g. {@code null}, a - * zero-length fixed, or a record whose fields all encode to zero bytes) - * consume no input bytes, so the number that may be - * declared is not bounded by the bytes remaining in the stream. Without a cap, - * a tiny payload can declare an enormous block count and drive an unbounded - * backing-array allocation. This limit is derived from the maximum heap (see + * Elements whose schema encodes to zero bytes (e.g. {@code null}, a zero-length + * fixed, or a record whose fields all encode to zero bytes) consume no input + * bytes, so the number that may be declared is not bounded by the bytes + * remaining in the stream. Without a cap, a tiny payload can declare an + * enormous block count and drive an unbounded backing-array allocation. This + * limit is derived from the maximum heap (see * {@link #MAX_COLLECTION_ALLOCATION_PROPERTY}). * * @param existing The number of elements already allocated for the collection. From dc2250c0ce12f4e27badb1c0c94e79e2061b23ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Mej=C3=ADa?= Date: Sun, 12 Jul 2026 21:29:14 +0200 Subject: [PATCH 11/25] AVRO-4300: [java] Clarify zero-byte comments to match the actual predicate Addresses review feedback: the "encodes to zero bytes" wording in GenericDatumReader (the array cap comment and isZeroByteSchema Javadoc) and FastReaderBuilder is imprecise. The guard is minBytesPerElement(schema) == 0, which is also true for recursive schemas whose cycle is broken by returning a 0 minimum. Reword to describe the "minimum encoded size is zero" predicate so the docs match the actual condition under which the heap-aware cap applies. Assisted-by: GitHub Copilot:claude-opus-4.8 --- .../avro/generic/GenericDatumReader.java | 27 +++++++++++-------- .../org/apache/avro/io/FastReaderBuilder.java | 11 ++++---- 2 files changed, 22 insertions(+), 16 deletions(-) diff --git a/lang/java/avro/src/main/java/org/apache/avro/generic/GenericDatumReader.java b/lang/java/avro/src/main/java/org/apache/avro/generic/GenericDatumReader.java index c0ac5b26553..21bc81c1de6 100644 --- a/lang/java/avro/src/main/java/org/apache/avro/generic/GenericDatumReader.java +++ b/lang/java/avro/src/main/java/org/apache/avro/generic/GenericDatumReader.java @@ -297,11 +297,13 @@ protected Object readArray(Object old, Schema expected, ResolvingDecoder in) thr long base = 0; if (l > 0) { ensureAvailableCollectionBytes(in, l, expectedType); - // Elements whose schema encodes to zero bytes (null, or a self-referencing - // record) consume no input, so ensureAvailableCollectionBytes cannot bound - // their count from the bytes remaining. Cap such collections against a - // heap-aware limit so a tiny payload cannot declare a huge block count and - // drive an unbounded backing-array allocation. + // Elements whose minimum encoded size is zero (null, a zero-length fixed, a + // record whose fields are all zero-byte, or a recursive schema where the + // cycle is broken with a 0 minimum) consume no guaranteed input, so + // ensureAvailableCollectionBytes cannot bound their count from the bytes + // remaining. Cap such collections against a heap-aware limit so a tiny + // payload cannot declare a huge block count and drive an unbounded + // backing-array allocation. boolean zeroByteElements = minBytesPerElement(expectedType) == 0; if (zeroByteElements) { SystemLimitException.checkMaxCollectionAllocation(base, l); @@ -456,14 +458,17 @@ static int minBytesPerElement(Schema schema) { } /** - * Whether values of the given schema encode to zero bytes (e.g. {@code null}, - * zero-length {@code fixed}, or a record whose fields are all zero-byte). Such - * elements cannot be bounded by the number of bytes remaining in the stream, so - * a collection of them must be bounded by a heap-aware allocation limit - * instead. + * Whether the minimum encoded size of the given schema is zero, i.e. + * {@link #minBytesPerElement(Schema)} is {@code 0}. This is true for values + * that always encode to zero bytes (e.g. {@code null}, a zero-length + * {@code fixed}, or a record whose fields are all zero-byte), and + * conservatively for recursive schemas, where the cycle is broken by returning + * a 0 minimum. Such elements cannot be bounded by the number of bytes remaining + * in the stream, so a collection of them must be bounded by a heap-aware + * allocation limit instead. * * @param schema the element (or map value) schema - * @return {@code true} if the schema encodes to zero bytes + * @return {@code true} if the schema's minimum encoded size is zero */ public static boolean isZeroByteSchema(Schema schema) { return minBytesPerElement(schema) == 0; diff --git a/lang/java/avro/src/main/java/org/apache/avro/io/FastReaderBuilder.java b/lang/java/avro/src/main/java/org/apache/avro/io/FastReaderBuilder.java index 9c5660948f5..e99280f5f9e 100644 --- a/lang/java/avro/src/main/java/org/apache/avro/io/FastReaderBuilder.java +++ b/lang/java/avro/src/main/java/org/apache/avro/io/FastReaderBuilder.java @@ -472,11 +472,12 @@ private FieldReader createArrayReader(Schema readerSchema, Container action) thr FieldReader elementReader = getReaderFor(action.elementAction, null); Schema elementType = readerSchema.getElementType(); - // Elements whose schema encodes to zero bytes (null, or a record with only - // zero-byte fields) consume no input, so the block count cannot be bounded by - // the bytes remaining in the stream. Cap such collections against a heap-aware - // limit so a tiny payload cannot declare a huge block count and drive an - // unbounded backing-array allocation (AVRO-4300). + // Elements whose minimum encoded size is zero (null, a record with only + // zero-byte fields, or a recursive schema whose cycle is broken with a 0 + // minimum) consume no guaranteed input, so the block count cannot be bounded + // by the bytes remaining in the stream. Cap such collections against a + // heap-aware limit so a tiny payload cannot declare a huge block count and + // drive an unbounded backing-array allocation (AVRO-4300). boolean zeroByteElements = GenericDatumReader.isZeroByteSchema(elementType); return reusingReader((reuse, decoder) -> { From c2ab33b123ffe082e04035d14fa825e189986846 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Mej=C3=ADa?= Date: Sun, 12 Jul 2026 21:50:22 +0200 Subject: [PATCH 12/25] AVRO-4300: [java] Reject Long.MIN_VALUE block count as malformed Addresses review feedback: doReadItemCount() negates a negative block count, but Long.MIN_VALUE negates back to Long.MIN_VALUE (still negative). The single-arg checkMaxCollectionLength does not reject negatives, so it was truncated via (int) cast to 0, silently ending the collection without consuming the end-of-array/map marker and desynchronizing decoding of subsequent fields. Reject Long.MIN_VALUE outright as malformed, matching the C SDK, instead of normalizing to an empty collection. Update the test to assert the rejection. Assisted-by: GitHub Copilot:claude-opus-4.8 --- .../java/org/apache/avro/io/BinaryDecoder.java | 10 +++++++++- .../avro/generic/TestGenericDatumReader.java | 18 +++++++++--------- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/lang/java/avro/src/main/java/org/apache/avro/io/BinaryDecoder.java b/lang/java/avro/src/main/java/org/apache/avro/io/BinaryDecoder.java index d2f049c55d4..e8bfed93d4d 100644 --- a/lang/java/avro/src/main/java/org/apache/avro/io/BinaryDecoder.java +++ b/lang/java/avro/src/main/java/org/apache/avro/io/BinaryDecoder.java @@ -408,8 +408,16 @@ protected void doReadBytes(byte[] bytes, int start, int length) throws IOExcepti protected long doReadItemCount() throws IOException { long result = readLong(); if (result < 0L) { - // Consume byte-count if present + // A negative block count is followed by a block byte-size; consume it. readLong(); + if (result == Long.MIN_VALUE) { + // Long.MIN_VALUE cannot be negated (-Long.MIN_VALUE overflows back to + // Long.MIN_VALUE), so it is not a valid block count. Reject it rather + // than letting it fall through as a negative "count" that would later be + // truncated to 0 and silently terminate the collection without consuming + // the end marker, desynchronizing decoding of subsequent fields. + throw new AvroRuntimeException("Malformed data. Block count is invalid: " + result); + } result = -result; } return result; diff --git a/lang/java/avro/src/test/java/org/apache/avro/generic/TestGenericDatumReader.java b/lang/java/avro/src/test/java/org/apache/avro/generic/TestGenericDatumReader.java index d58a532308b..2b9b02cf405 100644 --- a/lang/java/avro/src/test/java/org/apache/avro/generic/TestGenericDatumReader.java +++ b/lang/java/avro/src/test/java/org/apache/avro/generic/TestGenericDatumReader.java @@ -32,6 +32,7 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; +import org.apache.avro.AvroRuntimeException; import org.apache.avro.Schema; import org.apache.avro.SystemLimitException; import org.apache.avro.io.BinaryDecoder; @@ -490,22 +491,21 @@ void fastAndClassicReaderRejectNegativeNullBlockCount() throws Exception { /** * {@code Long.MIN_VALUE} as a block count is the pathological overflow case: - * negating it overflows back to a negative value. It must be handled safely - * without allocating -- the decoder normalizes it to an empty collection rather - * than a huge one -- on both reader paths. (The C SDK rejects it outright; this - * normalization is equally non-exploitable.) + * negating it overflows back to a negative value. Rather than letting it be + * truncated to {@code 0} (which would silently end the collection without + * consuming the end marker and desynchronize decoding of following fields), the + * decoder rejects it outright as malformed, matching the C SDK. */ @Test - void fastAndClassicReaderHandleMinValueBlockCountSafely() throws Exception { + void fastAndClassicReaderRejectMinValueBlockCount() throws Exception { for (boolean fast : new boolean[] { true, false }) { GenericDatumReader reader = arrayReader(Schema.create(Schema.Type.NULL), fast); // Long.MIN_VALUE items (zigzag), a block byte-size of 0, then the - // end-of-array terminator. Negating Long.MIN_VALUE overflows, so this must - // not drive an allocation. + // end-of-array terminator. Negating Long.MIN_VALUE overflows, so it must be + // rejected as malformed instead of decoded as an empty array. byte[] data = encodeVarints(Long.MIN_VALUE, 0L, 0L); BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(data, null); - Collection result = (Collection) reader.read(null, decoder); - assertEquals(0, result.size(), "fastReader=" + fast); + assertThrows(AvroRuntimeException.class, () -> reader.read(null, decoder), "fastReader=" + fast); } } From 178027d643ede536382d193aa3078a1841eebb56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Mej=C3=ADa?= Date: Sun, 12 Jul 2026 22:20:15 +0200 Subject: [PATCH 13/25] AVRO-4300: [java] Clamp collection preallocation from declared block count The array/map readers passed the declared block count straight to newArray/ newMap as the initial capacity. On a stream source the bytes-available guard is skipped (BinaryDecoder.remainingBytes() returns -1 for sources other than ByteArrayInputStream/ByteBufferInputStream), so a large declared count reaches the allocation path directly and drives a huge up-front allocation (e.g. new Object[count]) before a single element is read. Clamp the initial capacity to a modest bound (MAX_COLLECTION_PREALLOC) via initialCollectionCapacity(); the backing array/map still grows on demand as elements are decoded, so a truncated or hostile stream now fails with EOFException after a bounded allocation instead of attempting to preallocate hundreds of millions of slots. Applies to both the classic and fast readers. Assisted-by: GitHub Copilot:claude-opus-4.8 --- .../avro/generic/GenericDatumReader.java | 24 +++++++++++++++-- .../org/apache/avro/io/FastReaderBuilder.java | 2 +- .../avro/generic/TestGenericDatumReader.java | 27 +++++++++++++++++++ 3 files changed, 50 insertions(+), 3 deletions(-) diff --git a/lang/java/avro/src/main/java/org/apache/avro/generic/GenericDatumReader.java b/lang/java/avro/src/main/java/org/apache/avro/generic/GenericDatumReader.java index 21bc81c1de6..d959aa0f263 100644 --- a/lang/java/avro/src/main/java/org/apache/avro/generic/GenericDatumReader.java +++ b/lang/java/avro/src/main/java/org/apache/avro/generic/GenericDatumReader.java @@ -115,6 +115,26 @@ public void setExpected(Schema reader) { private static final ThreadLocal>> RESOLVER_CACHE = ThreadLocalWithInitial .of(WeakIdentityHashMap::new); + /** + * Upper bound on the initial capacity eagerly allocated for a collection from + * its declared block count. The backing array/map grows on demand as elements + * are read, so this is only a starting hint: it prevents a large declared count + * from driving a huge up-front allocation before any element is decoded. This + * matters most for stream sources, where the decoder cannot know how many bytes + * remain and so cannot otherwise bound the declared count against the input. + */ + private static final int MAX_COLLECTION_PREALLOC = 1024; + + /** + * Clamp a declared collection block count to a safe initial allocation size. + * + * @param count the declared (already limit-checked) block count + * @return {@code count} capped at {@link #MAX_COLLECTION_PREALLOC} + */ + public static int initialCollectionCapacity(long count) { + return (int) Math.min(count, MAX_COLLECTION_PREALLOC); + } + /** * Gets a resolving decoder for use by this GenericDatumReader. Unstable API. * Currently uses a thread local cache to prevent constructing the resolvers too @@ -310,7 +330,7 @@ protected Object readArray(Object old, Schema expected, ResolvingDecoder in) thr } LogicalType logicalType = expectedType.getLogicalType(); Conversion conversion = getData().getConversionFor(logicalType); - Object array = newArray(old, (int) l, expected); + Object array = newArray(old, initialCollectionCapacity(l), expected); do { if (logicalType != null && conversion != null) { for (long i = 0; i < l; i++) { @@ -380,7 +400,7 @@ protected Object readMap(Object old, Schema expected, ResolvingDecoder in) throw LogicalType logicalType = eValue.getLogicalType(); Conversion conversion = getData().getConversionFor(logicalType); ensureAvailableMapBytes(in, l, eValue); - Object map = newMap(old, (int) l); + Object map = newMap(old, initialCollectionCapacity(l)); if (l > 0) { do { if (logicalType != null && conversion != null) { diff --git a/lang/java/avro/src/main/java/org/apache/avro/io/FastReaderBuilder.java b/lang/java/avro/src/main/java/org/apache/avro/io/FastReaderBuilder.java index e99280f5f9e..1951439ffe4 100644 --- a/lang/java/avro/src/main/java/org/apache/avro/io/FastReaderBuilder.java +++ b/lang/java/avro/src/main/java/org/apache/avro/io/FastReaderBuilder.java @@ -502,7 +502,7 @@ private FieldReader createArrayReader(Schema readerSchema, Container action) thr long total = 0; checkArrayBlock(decoder, elementType, zeroByteElements, total, l); List array = (reuse instanceof List) ? (List) reuse - : new GenericData.Array<>((int) l, readerSchema); + : new GenericData.Array<>(GenericDatumReader.initialCollectionCapacity(l), readerSchema); array.clear(); while (l > 0) { for (long i = 0; i < l; i++) { diff --git a/lang/java/avro/src/test/java/org/apache/avro/generic/TestGenericDatumReader.java b/lang/java/avro/src/test/java/org/apache/avro/generic/TestGenericDatumReader.java index 2b9b02cf405..37449b80b7f 100644 --- a/lang/java/avro/src/test/java/org/apache/avro/generic/TestGenericDatumReader.java +++ b/lang/java/avro/src/test/java/org/apache/avro/generic/TestGenericDatumReader.java @@ -20,9 +20,12 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; +import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; +import java.io.BufferedInputStream; import java.io.EOFException; import java.io.IOException; +import java.io.InputStream; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -219,6 +222,30 @@ void arrayOfIntsRejectsHugeCount() throws Exception { assertThrows(EOFException.class, () -> reader.read(null, decoder)); } + /** + * On a stream source the decoder cannot know how many bytes remain, so the + * bytes-available guard is skipped and a large declared array count reaches the + * allocation path directly. The initial backing-array capacity must be clamped + * so a hostile count cannot drive a huge up-front allocation; a truncated + * stream therefore fails with {@link EOFException} (once the declared elements + * cannot be read) rather than attempting to preallocate hundreds of millions of + * slots. + */ + @Test + void arrayHugeCountOnStreamClampsPreallocation() throws Exception { + Schema schema = Schema.createArray(Schema.create(Schema.Type.LONG)); + GenericDatumReader reader = new GenericDatumReader<>(schema); + + // A huge (non-zero-byte) block count followed by no element data. Wrapping in + // a BufferedInputStream keeps the source from being a ByteArrayInputStream, so + // it cannot report its remaining byte count (remainingBytes() == -1) and the + // bytes-available guard is disabled -- exercising the preallocation clamp. + byte[] data = encodeVarints(200_000_000L); + InputStream stream = new BufferedInputStream(new ByteArrayInputStream(data)); + BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(stream, null); + assertThrows(EOFException.class, () -> reader.read(null, decoder)); + } + /** * Verify that reading an array of nulls with a large count SUCCEEDS because * null elements are 0 bytes each, so the byte check is correctly skipped. From a9198754838b001b2e09de57f940687d45d34170 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Mej=C3=ADa?= Date: Sun, 12 Jul 2026 23:28:23 +0200 Subject: [PATCH 14/25] AVRO-4300: [java] Reject out-of-range union and enum indices with AvroTypeException The union branch index and enum symbol index were used to index the branch/ symbol tables without an explicit range check on several paths, surfacing a malformed index as a generic IndexOutOfBoundsException instead of a clear Avro-specific error: - Union: Symbol.Alternative.getSymbol (used by both the validating and resolving decoders, and therefore by GenericDatumReader and the fast reader) indexed symbols[] directly. Add a [0, size) check throwing AvroTypeException. - Enum: ResolvingDecoder.readEnum returned the raw index in the no-adjustments case and indexed the adjustment table otherwise, both without validation (ValidatingDecoder.readEnum already checked). Add the range checks. - The fast reader reads from a plain decoder (resolution is pre-baked), so its union and enum readers need their own [0, length) checks. Adds tests covering negative and too-large union and enum indices on both the classic and fast reader paths. Assisted-by: GitHub Copilot:claude-opus-4.8 --- .../org/apache/avro/io/FastReaderBuilder.java | 8 ++++ .../org/apache/avro/io/ResolvingDecoder.java | 10 +++++ .../org/apache/avro/io/parsing/Symbol.java | 5 +++ .../avro/generic/TestGenericDatumReader.java | 37 +++++++++++++++++++ 4 files changed, 60 insertions(+) diff --git a/lang/java/avro/src/main/java/org/apache/avro/io/FastReaderBuilder.java b/lang/java/avro/src/main/java/org/apache/avro/io/FastReaderBuilder.java index 1951439ffe4..e71b9d78d66 100644 --- a/lang/java/avro/src/main/java/org/apache/avro/io/FastReaderBuilder.java +++ b/lang/java/avro/src/main/java/org/apache/avro/io/FastReaderBuilder.java @@ -420,6 +420,10 @@ private FieldReader createUnionReader(WriterUnion action) throws IOException { private FieldReader createUnionReader(FieldReader[] unionReaders) { return reusingReader((reuse, decoder) -> { final int selection = decoder.readIndex(); + if (selection < 0 || selection >= unionReaders.length) { + throw new AvroTypeException( + "Union branch index out of range: must be in [0, " + unionReaders.length + "), but received " + selection); + } return unionReaders[selection].read(null, decoder); }); @@ -538,6 +542,10 @@ private static void checkArrayBlock(Decoder decoder, Schema elementType, boolean private FieldReader createEnumReader(EnumAdjust action) { return reusingReader((reuse, decoder) -> { int index = decoder.readEnum(); + if (index < 0 || index >= action.values.length) { + throw new AvroTypeException( + "Enumeration out of range: max is " + action.values.length + " but received " + index); + } Object resultObject = action.values[index]; if (resultObject == null) { throw new AvroTypeException("No match for " + action.writer.getEnumSymbols().get(index)); diff --git a/lang/java/avro/src/main/java/org/apache/avro/io/ResolvingDecoder.java b/lang/java/avro/src/main/java/org/apache/avro/io/ResolvingDecoder.java index 6bdb16a332c..9c846c3c8fa 100644 --- a/lang/java/avro/src/main/java/org/apache/avro/io/ResolvingDecoder.java +++ b/lang/java/avro/src/main/java/org/apache/avro/io/ResolvingDecoder.java @@ -260,8 +260,18 @@ public int readEnum() throws IOException { Symbol.EnumAdjustAction top = (Symbol.EnumAdjustAction) parser.popSymbol(); int n = in.readEnum(); if (top.noAdjustments) { + // n is used directly as an index into the reader enum's symbols, so it + // must fall within the reader symbol count. + if (n < 0 || n >= top.size) { + throw new AvroTypeException("Enumeration out of range: max is " + top.size + " but received " + n); + } return n; } + // Otherwise n indexes the writer-to-reader adjustment table; reject an index + // outside it rather than letting the array access throw. + if (n < 0 || n >= top.adjustments.length) { + throw new AvroTypeException("Enumeration out of range: max is " + top.adjustments.length + " but received " + n); + } Object o = top.adjustments[n]; if (o instanceof Integer) { return (Integer) o; diff --git a/lang/java/avro/src/main/java/org/apache/avro/io/parsing/Symbol.java b/lang/java/avro/src/main/java/org/apache/avro/io/parsing/Symbol.java index b5dcbeb68f0..039b1517fd6 100644 --- a/lang/java/avro/src/main/java/org/apache/avro/io/parsing/Symbol.java +++ b/lang/java/avro/src/main/java/org/apache/avro/io/parsing/Symbol.java @@ -26,6 +26,7 @@ import java.util.NoSuchElementException; import java.util.Set; +import org.apache.avro.AvroTypeException; import org.apache.avro.Schema; /** @@ -457,6 +458,10 @@ private Alternative(Symbol[] symbols, String[] labels) { } public Symbol getSymbol(int index) { + if (index < 0 || index >= symbols.length) { + throw new AvroTypeException( + "Union branch index out of range: must be in [0, " + symbols.length + "), but received " + index); + } return symbols[index]; } diff --git a/lang/java/avro/src/test/java/org/apache/avro/generic/TestGenericDatumReader.java b/lang/java/avro/src/test/java/org/apache/avro/generic/TestGenericDatumReader.java index 37449b80b7f..4787407879d 100644 --- a/lang/java/avro/src/test/java/org/apache/avro/generic/TestGenericDatumReader.java +++ b/lang/java/avro/src/test/java/org/apache/avro/generic/TestGenericDatumReader.java @@ -536,6 +536,43 @@ void fastAndClassicReaderRejectMinValueBlockCount() throws Exception { } } + /** + * A union branch index outside {@code [0, branch count)} is malformed and must + * be rejected on both reader paths rather than throwing a generic + * {@code IndexOutOfBoundsException}. + */ + @Test + void fastAndClassicReaderRejectOutOfRangeUnionIndex() throws Exception { + Schema schema = Schema.createUnion(Schema.create(Schema.Type.NULL), Schema.create(Schema.Type.LONG)); + for (boolean fast : new boolean[] { true, false }) { + for (long index : new long[] { 5L, -1L }) { + GenericDatumReader reader = readerFor(schema, fast); + byte[] data = encodeVarints(index); + BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(data, null); + assertThrows(org.apache.avro.AvroTypeException.class, () -> reader.read(null, decoder), + "fastReader=" + fast + " index=" + index); + } + } + } + + /** + * An enum symbol index outside {@code [0, symbol count)} is malformed and must + * be rejected on both reader paths. + */ + @Test + void fastAndClassicReaderRejectOutOfRangeEnumIndex() throws Exception { + Schema schema = Schema.createEnum("E", null, null, Arrays.asList("A", "B")); + for (boolean fast : new boolean[] { true, false }) { + for (int index : new int[] { 9, -1 }) { + GenericDatumReader reader = readerFor(schema, fast); + byte[] data = encodeVarints(index); + BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(data, null); + assertThrows(org.apache.avro.AvroTypeException.class, () -> reader.read(null, decoder), + "fastReader=" + fast + " index=" + index); + } + } + } + /** * A legitimate array of nulls within the limit still decodes on both reader * paths. From d6ddf5367bc612702bb297ce3d9ec3254d2d326a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Mej=C3=ADa?= Date: Mon, 13 Jul 2026 01:15:01 +0200 Subject: [PATCH 15/25] AVRO-4300: [java] Reword remaining "zero-byte" docs to the actual predicate The maxAllocation limit is applied when a schema's minimum encoded size is zero (GenericDatumReader.isZeroByteSchema, i.e. minBytesPerElement == 0), which also covers recursive schemas whose cycle is conservatively broken with a 0 minimum. Reword the remaining "encodes to zero bytes" wording -- the class Javadoc, the MAX_COLLECTION_ALLOCATION_PROPERTY Javadoc, the SystemLimitException message, and the skipArray comment -- to say "minimum encoded size is zero" so the docs match the condition under which the cap actually applies. Assisted-by: GitHub Copilot:claude-opus-4.8 --- .../org/apache/avro/SystemLimitException.java | 23 +++++++++++-------- .../avro/generic/GenericDatumReader.java | 9 ++++---- 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/lang/java/avro/src/main/java/org/apache/avro/SystemLimitException.java b/lang/java/avro/src/main/java/org/apache/avro/SystemLimitException.java index 19b39364f57..b2725f31db5 100644 --- a/lang/java/avro/src/main/java/org/apache/avro/SystemLimitException.java +++ b/lang/java/avro/src/main/java/org/apache/avro/SystemLimitException.java @@ -36,11 +36,12 @@ *
  • org.apache.avro.limits.string.maxLength limits the maximum size * of string types.
    • *
  • org.apache.avro.limits.collectionItems.maxAllocation limits the - * number of array elements whose schema encodes to zero bytes (such as - * null, a zero-length fixed, or a record whose fields all - * encode to zero bytes) that may be allocated at once. Unlike other element - * types, these cannot be bounded by the number of bytes remaining in the - * stream, so the limit defaults to a fraction of the maximum heap.
    • + * number of array elements whose minimum encoded size is zero (such as + * null, a zero-length fixed, a record whose fields are all + * zero-byte, or a recursive schema whose cycle is conservatively broken with a + * 0 minimum) that may be allocated at once. Unlike other element types, these + * cannot be bounded by the number of bytes remaining in the stream, so the + * limit defaults to a fraction of the maximum heap. * * * The default is to permit sizes up to {@link #MAX_ARRAY_VM_LIMIT}. @@ -90,9 +91,10 @@ public class SystemLimitException extends AvroRuntimeException { defaultMaxDecompressLength()); /** - * System property declaring the maximum number of zero-byte-encoded array - * elements (e.g. {@code null}, a zero-length fixed, or a record whose fields - * all encode to zero bytes) to allocate at once: {@value}. + * System property declaring the maximum number of array elements whose minimum + * encoded size is zero (e.g. {@code null}, a zero-length fixed, a record whose + * fields are all zero-byte, or a recursive schema conservatively treated as a 0 + * minimum) to allocate at once: {@value}. */ public static final String MAX_COLLECTION_ALLOCATION_PROPERTY = "org.apache.avro.limits.collectionItems.maxAllocation"; @@ -320,8 +322,9 @@ public static long checkMaxCollectionAllocation(long existing, long items) { long total = existing + items; if (total < existing || total > maxCollectionAllocation) { throw new SystemLimitException("Cannot allocate " + (total < existing ? "more than Long.MAX_VALUE" : total) - + " zero-byte collection elements: exceeds the maximum allowed of " + maxCollectionAllocation - + " (configure with the system property " + MAX_COLLECTION_ALLOCATION_PROPERTY + ")"); + + " collection elements whose minimum encoded size is zero: exceeds the maximum allowed of " + + maxCollectionAllocation + " (configure with the system property " + MAX_COLLECTION_ALLOCATION_PROPERTY + + ")"); } return total; } diff --git a/lang/java/avro/src/main/java/org/apache/avro/generic/GenericDatumReader.java b/lang/java/avro/src/main/java/org/apache/avro/generic/GenericDatumReader.java index d959aa0f263..d236483f216 100644 --- a/lang/java/avro/src/main/java/org/apache/avro/generic/GenericDatumReader.java +++ b/lang/java/avro/src/main/java/org/apache/avro/generic/GenericDatumReader.java @@ -802,10 +802,11 @@ public static void skip(Schema schema, Decoder in) throws IOException { break; case ARRAY: Schema elementType = schema.getElementType(); - // Bound the cumulative element count: skipping a huge block of zero-byte - // elements (e.g. null) would otherwise loop unboundedly (a CPU - // exhaustion) even though it reads nothing. Zero-byte elements use the - // heap-aware allocation cap; others the structural collection cap. + // Bound the cumulative element count: skipping a huge block of elements + // whose minimum encoded size is zero (e.g. null) would otherwise loop + // unboundedly (a CPU exhaustion) even though it reads nothing. Such + // elements use the heap-aware allocation cap; others the structural + // collection cap. boolean zeroByteElements = isZeroByteSchema(elementType); long arrayTotal = 0; for (long l = in.skipArray(); l > 0; l = in.skipArray()) { From f8c3337434b2d22171a486340e841cfd5a44e97d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Mej=C3=ADa?= Date: Mon, 13 Jul 2026 01:46:54 +0200 Subject: [PATCH 16/25] AVRO-4300: [java] Half-open range in enum messages; finish zero-byte reword Address review feedback: - The enum out-of-range messages said "max is ", implying is a valid index. Reword to the half-open "must be in [0, )" to match the actual check and the union messages, in ResolvingDecoder (both the no-adjustments and adjustments branches), FastReaderBuilder, and ValidatingDecoder. - Reword the remaining "encodes to zero bytes" wording in the checkMaxCollectionAllocation Javadoc to "minimum encoded size is zero" (including the recursive-schema case), matching isZeroByteSchema. Assisted-by: GitHub Copilot:claude-opus-4.8 --- .../src/main/java/org/apache/avro/SystemLimitException.java | 5 +++-- .../src/main/java/org/apache/avro/io/FastReaderBuilder.java | 2 +- .../src/main/java/org/apache/avro/io/ResolvingDecoder.java | 5 +++-- .../src/main/java/org/apache/avro/io/ValidatingDecoder.java | 2 +- 4 files changed, 8 insertions(+), 6 deletions(-) diff --git a/lang/java/avro/src/main/java/org/apache/avro/SystemLimitException.java b/lang/java/avro/src/main/java/org/apache/avro/SystemLimitException.java index b2725f31db5..88f03dbbced 100644 --- a/lang/java/avro/src/main/java/org/apache/avro/SystemLimitException.java +++ b/lang/java/avro/src/main/java/org/apache/avro/SystemLimitException.java @@ -297,8 +297,9 @@ public static int checkMaxCollectionLength(long items) { * Check to ensure that allocating storage for the specified number of * zero-byte-encoded array elements remains within the heap-aware limit. *

    - * Elements whose schema encodes to zero bytes (e.g. {@code null}, a zero-length - * fixed, or a record whose fields all encode to zero bytes) consume no input + * Elements whose minimum encoded size is zero (e.g. {@code null}, a zero-length + * fixed, a record whose fields are all zero-byte, or a recursive schema whose + * cycle is conservatively broken with a 0 minimum) consume no guaranteed input * bytes, so the number that may be declared is not bounded by the bytes * remaining in the stream. Without a cap, a tiny payload can declare an * enormous block count and drive an unbounded backing-array allocation. This diff --git a/lang/java/avro/src/main/java/org/apache/avro/io/FastReaderBuilder.java b/lang/java/avro/src/main/java/org/apache/avro/io/FastReaderBuilder.java index e71b9d78d66..f5158940183 100644 --- a/lang/java/avro/src/main/java/org/apache/avro/io/FastReaderBuilder.java +++ b/lang/java/avro/src/main/java/org/apache/avro/io/FastReaderBuilder.java @@ -544,7 +544,7 @@ private FieldReader createEnumReader(EnumAdjust action) { int index = decoder.readEnum(); if (index < 0 || index >= action.values.length) { throw new AvroTypeException( - "Enumeration out of range: max is " + action.values.length + " but received " + index); + "Enumeration out of range: must be in [0, " + action.values.length + "), but received " + index); } Object resultObject = action.values[index]; if (resultObject == null) { diff --git a/lang/java/avro/src/main/java/org/apache/avro/io/ResolvingDecoder.java b/lang/java/avro/src/main/java/org/apache/avro/io/ResolvingDecoder.java index 9c846c3c8fa..0b124579f71 100644 --- a/lang/java/avro/src/main/java/org/apache/avro/io/ResolvingDecoder.java +++ b/lang/java/avro/src/main/java/org/apache/avro/io/ResolvingDecoder.java @@ -263,14 +263,15 @@ public int readEnum() throws IOException { // n is used directly as an index into the reader enum's symbols, so it // must fall within the reader symbol count. if (n < 0 || n >= top.size) { - throw new AvroTypeException("Enumeration out of range: max is " + top.size + " but received " + n); + throw new AvroTypeException("Enumeration out of range: must be in [0, " + top.size + "), but received " + n); } return n; } // Otherwise n indexes the writer-to-reader adjustment table; reject an index // outside it rather than letting the array access throw. if (n < 0 || n >= top.adjustments.length) { - throw new AvroTypeException("Enumeration out of range: max is " + top.adjustments.length + " but received " + n); + throw new AvroTypeException( + "Enumeration out of range: must be in [0, " + top.adjustments.length + "), but received " + n); } Object o = top.adjustments[n]; if (o instanceof Integer) { diff --git a/lang/java/avro/src/main/java/org/apache/avro/io/ValidatingDecoder.java b/lang/java/avro/src/main/java/org/apache/avro/io/ValidatingDecoder.java index 26f79a16ff2..3c5e0842189 100644 --- a/lang/java/avro/src/main/java/org/apache/avro/io/ValidatingDecoder.java +++ b/lang/java/avro/src/main/java/org/apache/avro/io/ValidatingDecoder.java @@ -164,7 +164,7 @@ public int readEnum() throws IOException { Symbol.IntCheckAction top = (Symbol.IntCheckAction) parser.popSymbol(); int result = in.readEnum(); if (result < 0 || result >= top.size) { - throw new AvroTypeException("Enumeration out of range: max is " + top.size + " but received " + result); + throw new AvroTypeException("Enumeration out of range: must be in [0, " + top.size + "), but received " + result); } return result; } From d5daffc328e8125b4f4ee28bd9382e5e87ede7f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Mej=C3=ADa?= Date: Mon, 13 Jul 2026 02:10:25 +0200 Subject: [PATCH 17/25] AVRO-4300: [java] Enforce structural cap when skipping zero-byte arrays; tidy Address review feedback: - skip(ARRAY) bounded zero-byte-element arrays only by the heap-aware allocation cap, so the configurable structural collection limit (MAX_COLLECTION_LENGTH_PROPERTY) was not enforced cumulatively and a large count split into many blocks could drive a long skip loop. Always enforce checkMaxCollectionLength cumulatively, and additionally checkMaxCollectionAllocation for zero-byte schemas. - FastReaderBuilder.checkArrayBlock: skip the ensureAvailableCollectionBytes call for zero-byte elements (it recomputes minBytesPerElement and no-ops), applying the allocation cap directly instead. - Reword the checkMaxCollectionAllocation summary line from "zero-byte-encoded" to "minimum encoded size is zero". Assisted-by: GitHub Copilot:claude-opus-4.8 --- .../main/java/org/apache/avro/SystemLimitException.java | 5 +++-- .../java/org/apache/avro/generic/GenericDatumReader.java | 7 +++++-- .../main/java/org/apache/avro/io/FastReaderBuilder.java | 6 +++++- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/lang/java/avro/src/main/java/org/apache/avro/SystemLimitException.java b/lang/java/avro/src/main/java/org/apache/avro/SystemLimitException.java index 88f03dbbced..db0f95bcc79 100644 --- a/lang/java/avro/src/main/java/org/apache/avro/SystemLimitException.java +++ b/lang/java/avro/src/main/java/org/apache/avro/SystemLimitException.java @@ -294,8 +294,9 @@ public static int checkMaxCollectionLength(long items) { } /** - * Check to ensure that allocating storage for the specified number of - * zero-byte-encoded array elements remains within the heap-aware limit. + * Check to ensure that allocating storage for the specified number of array + * elements whose minimum encoded size is zero remains within the heap-aware + * limit. *

    * Elements whose minimum encoded size is zero (e.g. {@code null}, a zero-length * fixed, a record whose fields are all zero-byte, or a recursive schema whose diff --git a/lang/java/avro/src/main/java/org/apache/avro/generic/GenericDatumReader.java b/lang/java/avro/src/main/java/org/apache/avro/generic/GenericDatumReader.java index d236483f216..cc72297a2eb 100644 --- a/lang/java/avro/src/main/java/org/apache/avro/generic/GenericDatumReader.java +++ b/lang/java/avro/src/main/java/org/apache/avro/generic/GenericDatumReader.java @@ -810,10 +810,13 @@ public static void skip(Schema schema, Decoder in) throws IOException { boolean zeroByteElements = isZeroByteSchema(elementType); long arrayTotal = 0; for (long l = in.skipArray(); l > 0; l = in.skipArray()) { + // Always enforce the cumulative structural cap, then additionally the + // heap-aware allocation cap for zero-byte elements (which the structural + // cap alone does not bound tightly), so a huge count split across blocks + // cannot drive an unbounded skip loop. + SystemLimitException.checkMaxCollectionLength(arrayTotal, l); if (zeroByteElements) { SystemLimitException.checkMaxCollectionAllocation(arrayTotal, l); - } else { - SystemLimitException.checkMaxCollectionLength(arrayTotal, l); } arrayTotal += l; for (long i = 0; i < l; i++) { diff --git a/lang/java/avro/src/main/java/org/apache/avro/io/FastReaderBuilder.java b/lang/java/avro/src/main/java/org/apache/avro/io/FastReaderBuilder.java index f5158940183..96767ad9934 100644 --- a/lang/java/avro/src/main/java/org/apache/avro/io/FastReaderBuilder.java +++ b/lang/java/avro/src/main/java/org/apache/avro/io/FastReaderBuilder.java @@ -533,9 +533,13 @@ private static void checkArrayBlock(Decoder decoder, Schema elementType, boolean if (count <= 0) { return; } - GenericDatumReader.ensureAvailableCollectionBytes(decoder, count, elementType); if (zeroByteElements) { + // The bytes-remaining check cannot bound zero-byte elements (minBytes is + // 0, so ensureAvailableCollectionBytes would no-op after recomputing it); + // apply the heap-aware allocation cap instead. SystemLimitException.checkMaxCollectionAllocation(total, count); + } else { + GenericDatumReader.ensureAvailableCollectionBytes(decoder, count, elementType); } } From d5e57a310d3aa051ff3a136ebf282333f31d081b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Mej=C3=ADa?= Date: Mon, 13 Jul 2026 02:38:00 +0200 Subject: [PATCH 18/25] AVRO-4300: [java] Reword defaultMaxCollectionAllocation Javadoc Update the last "zero-byte-encoded" wording (the private defaultMaxCollectionAllocation helper) to "minimum encoded size is zero", consistent with the public property/Javadoc and the isZeroByteSchema predicate. Assisted-by: GitHub Copilot:claude-opus-4.8 --- .../java/org/apache/avro/SystemLimitException.java | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/lang/java/avro/src/main/java/org/apache/avro/SystemLimitException.java b/lang/java/avro/src/main/java/org/apache/avro/SystemLimitException.java index db0f95bcc79..6734013d350 100644 --- a/lang/java/avro/src/main/java/org/apache/avro/SystemLimitException.java +++ b/lang/java/avro/src/main/java/org/apache/avro/SystemLimitException.java @@ -188,12 +188,13 @@ private static long defaultMaxDecompressLength() { } /** - * Calculate the default maximum number of zero-byte-encoded array elements to - * allocate at once, as a fraction of the maximum heap. Such elements consume no - * input bytes, so the usual "bytes remaining" bound does not apply and the - * allocation must instead be capped relative to the available memory. + * Calculate the default maximum number of array elements whose minimum encoded + * size is zero to allocate at once, as a fraction of the maximum heap. Such + * elements consume no guaranteed input bytes, so the usual "bytes remaining" + * bound does not apply and the allocation must instead be capped relative to + * the available memory. * - * @return the calculated default max zero-byte element count. + * @return the calculated default max element count. */ private static long defaultMaxCollectionAllocation() { long heapBudget = Math.max(1L, Runtime.getRuntime().maxMemory() / DEFAULT_MAX_COLLECTION_ALLOCATION_HEAP_FRACTION); From 112a3fa922e79b9a513e4593b867069915d5dca8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Mej=C3=ADa?= Date: Mon, 13 Jul 2026 02:48:45 +0200 Subject: [PATCH 19/25] AVRO-4300: [java] Reword last zero-byte field doc; sort test imports - Reword the maxCollectionAllocation field Javadoc from "zero-byte-encoded" to "minimum encoded size is zero". - Sort the java.io imports in TestGenericDatumReader (BufferedInputStream before the ByteArray* imports) to satisfy Spotless. Assisted-by: GitHub Copilot:claude-opus-4.8 --- .../src/main/java/org/apache/avro/SystemLimitException.java | 5 +++-- .../java/org/apache/avro/generic/TestGenericDatumReader.java | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/lang/java/avro/src/main/java/org/apache/avro/SystemLimitException.java b/lang/java/avro/src/main/java/org/apache/avro/SystemLimitException.java index 6734013d350..7f60068e175 100644 --- a/lang/java/avro/src/main/java/org/apache/avro/SystemLimitException.java +++ b/lang/java/avro/src/main/java/org/apache/avro/SystemLimitException.java @@ -112,8 +112,9 @@ public class SystemLimitException extends AvroRuntimeException { private static final long BYTES_PER_COLLECTION_SLOT = 8; /** - * Maximum number of zero-byte-encoded array elements to allocate at once. - * Recomputed from the system property (or the heap) by {@link #resetLimits()}. + * Maximum number of array elements whose minimum encoded size is zero to + * allocate at once. Recomputed from the system property (or the heap) by + * {@link #resetLimits()}. */ private static long maxCollectionAllocation = defaultMaxCollectionAllocation(); diff --git a/lang/java/avro/src/test/java/org/apache/avro/generic/TestGenericDatumReader.java b/lang/java/avro/src/test/java/org/apache/avro/generic/TestGenericDatumReader.java index 4787407879d..1c9f64d39b1 100644 --- a/lang/java/avro/src/test/java/org/apache/avro/generic/TestGenericDatumReader.java +++ b/lang/java/avro/src/test/java/org/apache/avro/generic/TestGenericDatumReader.java @@ -20,9 +20,9 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; +import java.io.BufferedInputStream; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; -import java.io.BufferedInputStream; import java.io.EOFException; import java.io.IOException; import java.io.InputStream; From 2cdb92020d96a207f04a9231f1517b925c3b112f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Mej=C3=ADa?= Date: Mon, 13 Jul 2026 03:01:36 +0200 Subject: [PATCH 20/25] AVRO-4300: [java] Reject Long.MIN_VALUE and negative byte-size in doSkipItems doSkipItems accepted a Long.MIN_VALUE block count (treating it as a byte-sized block and continuing to skip), inconsistent with doReadItemCount which rejects it. Reject Long.MIN_VALUE, and also reject a negative block byte-size, so the skip path fails fast on malformed input like the read path. Assisted-by: GitHub Copilot:claude-opus-4.8 --- .../main/java/org/apache/avro/io/BinaryDecoder.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/lang/java/avro/src/main/java/org/apache/avro/io/BinaryDecoder.java b/lang/java/avro/src/main/java/org/apache/avro/io/BinaryDecoder.java index e8bfed93d4d..12e710d83db 100644 --- a/lang/java/avro/src/main/java/org/apache/avro/io/BinaryDecoder.java +++ b/lang/java/avro/src/main/java/org/apache/avro/io/BinaryDecoder.java @@ -444,7 +444,17 @@ protected long doReadItemCount() throws IOException { private long doSkipItems() throws IOException { long result = readLong(); while (result < 0L) { + if (result == Long.MIN_VALUE) { + // Consistent with doReadItemCount: Long.MIN_VALUE is not a valid block + // count (it cannot be negated), so reject it rather than treating it as + // a byte-sized block and continuing to skip. + readLong(); + throw new AvroRuntimeException("Malformed data. Block count is invalid: " + result); + } final long bytecount = readLong(); + if (bytecount < 0L) { + throw new AvroRuntimeException("Malformed data. Block byte-size is negative: " + bytecount); + } doSkipBytes(bytecount); result = readLong(); } From 59aa3c74087653de0c67b2ddcd944eae08fffe9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Mej=C3=ADa?= Date: Mon, 13 Jul 2026 03:23:14 +0200 Subject: [PATCH 21/25] AVRO-4300: [java] Reject negative block byte-size in doReadItemCount doReadItemCount consumed the block byte-size for a negative-count block without validating it. doSkipItems now rejects a negative byte-size, so the read path does the same for consistency and to reject malformed encodings. Assisted-by: GitHub Copilot:claude-opus-4.8 --- .../src/main/java/org/apache/avro/io/BinaryDecoder.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lang/java/avro/src/main/java/org/apache/avro/io/BinaryDecoder.java b/lang/java/avro/src/main/java/org/apache/avro/io/BinaryDecoder.java index 12e710d83db..c2e2ab8351c 100644 --- a/lang/java/avro/src/main/java/org/apache/avro/io/BinaryDecoder.java +++ b/lang/java/avro/src/main/java/org/apache/avro/io/BinaryDecoder.java @@ -409,7 +409,7 @@ protected long doReadItemCount() throws IOException { long result = readLong(); if (result < 0L) { // A negative block count is followed by a block byte-size; consume it. - readLong(); + final long bytecount = readLong(); if (result == Long.MIN_VALUE) { // Long.MIN_VALUE cannot be negated (-Long.MIN_VALUE overflows back to // Long.MIN_VALUE), so it is not a valid block count. Reject it rather @@ -418,6 +418,11 @@ protected long doReadItemCount() throws IOException { // the end marker, desynchronizing decoding of subsequent fields. throw new AvroRuntimeException("Malformed data. Block count is invalid: " + result); } + if (bytecount < 0L) { + // The block byte-size is a byte count and must be non-negative, matching + // doSkipItems(). + throw new AvroRuntimeException("Malformed data. Block byte-size is negative: " + bytecount); + } result = -result; } return result; From b3c547ffa2b793460fc823f215ca26ef52673add Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Mej=C3=ADa?= Date: Mon, 13 Jul 2026 03:45:53 +0200 Subject: [PATCH 22/25] AVRO-4300: [java] Clarify test Javadoc: trailing 0L varint, not "no data" The huge-collection matrix payload includes a trailing 0L varint (an element value for arrays, a 0-length key for maps); reword "no element data" to reflect that. Assisted-by: GitHub Copilot:claude-opus-4.8 --- .../apache/avro/generic/TestGenericDatumReader.java | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/lang/java/avro/src/test/java/org/apache/avro/generic/TestGenericDatumReader.java b/lang/java/avro/src/test/java/org/apache/avro/generic/TestGenericDatumReader.java index 1c9f64d39b1..190547858ac 100644 --- a/lang/java/avro/src/test/java/org/apache/avro/generic/TestGenericDatumReader.java +++ b/lang/java/avro/src/test/java/org/apache/avro/generic/TestGenericDatumReader.java @@ -425,11 +425,13 @@ private static GenericDatumReader readerFor(Schema schema, boolean fastR /** * Full matrix: every collection kind must be rejected (never OOM) with a huge - * declared block count and no element data, on both the fast (default) and the - * classic reader. Zero-byte-element arrays are bounded by the heap-aware - * allocation cap (SystemLimitException); every other kind is bounded by the - * bytes-remaining check (EOFException). Maps always carry a >=1-byte key so - * they fall in the latter group regardless of the value type. + * declared block count and effectively no element data (only a trailing {@code + * 0L} varint, used as a single element value or a 0-length map key), on both + * the fast (default) and the classic reader. Zero-byte-element arrays are + * bounded by the heap-aware allocation cap (SystemLimitException); every other + * kind is bounded by the bytes-remaining check (EOFException). Maps always + * carry a >=1-byte key so they fall in the latter group regardless of the value + * type. */ @Test void hugeCollectionsRejectedOnBothReaderPaths() throws Exception { From 9d27ea3528d474d1e7ede6460283d0d03bf8d351 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Mej=C3=ADa?= Date: Mon, 13 Jul 2026 06:26:56 +0200 Subject: [PATCH 23/25] AVRO-4300: [java] Reword heap-fraction Javadoc to the actual predicate Reword the DEFAULT_MAX_COLLECTION_ALLOCATION_HEAP_FRACTION Javadoc from "zero-byte elements" to "elements whose minimum encoded size is zero". Assisted-by: GitHub Copilot:claude-opus-4.8 --- .../main/java/org/apache/avro/SystemLimitException.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lang/java/avro/src/main/java/org/apache/avro/SystemLimitException.java b/lang/java/avro/src/main/java/org/apache/avro/SystemLimitException.java index 7f60068e175..5ce0b25ba6f 100644 --- a/lang/java/avro/src/main/java/org/apache/avro/SystemLimitException.java +++ b/lang/java/avro/src/main/java/org/apache/avro/SystemLimitException.java @@ -99,9 +99,10 @@ public class SystemLimitException extends AvroRuntimeException { public static final String MAX_COLLECTION_ALLOCATION_PROPERTY = "org.apache.avro.limits.collectionItems.maxAllocation"; /** - * Fraction of the maximum heap a single decoded collection of zero-byte - * elements may occupy by default. Keeps the backing allocation below the heap - * so a small payload declaring a huge block count cannot exhaust the JVM. + * Fraction of the maximum heap a single decoded collection of elements whose + * minimum encoded size is zero may occupy by default. Keeps the backing + * allocation below the heap so a small payload declaring a huge block count + * cannot exhaust the JVM. */ private static final long DEFAULT_MAX_COLLECTION_ALLOCATION_HEAP_FRACTION = 4; From bb0de738f14ed09b5479aeff8925106404ffb2b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Mej=C3=ADa?= Date: Mon, 13 Jul 2026 07:11:12 +0200 Subject: [PATCH 24/25] AVRO-4300: [java] Make array preallocation-clamp test regression-proof arrayHugeCountOnStreamClampsPreallocation only asserted an EOFException, which a reintroduced new Object[(int) count] preallocation could still satisfy on a large-heap JVM (allocating ~200M slots then hitting EOF). Override newArray to record the largest requested capacity and assert it stays clamped to initialCollectionCapacity, and assert remainingBytes() == -1 to confirm the unknown-remaining-bytes precondition. Assisted-by: GitHub Copilot:claude-opus-4.8 --- .../avro/generic/TestGenericDatumReader.java | 22 ++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/lang/java/avro/src/test/java/org/apache/avro/generic/TestGenericDatumReader.java b/lang/java/avro/src/test/java/org/apache/avro/generic/TestGenericDatumReader.java index 190547858ac..211692c4d7b 100644 --- a/lang/java/avro/src/test/java/org/apache/avro/generic/TestGenericDatumReader.java +++ b/lang/java/avro/src/test/java/org/apache/avro/generic/TestGenericDatumReader.java @@ -19,6 +19,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.BufferedInputStream; import java.io.ByteArrayInputStream; @@ -234,7 +235,19 @@ void arrayOfIntsRejectsHugeCount() throws Exception { @Test void arrayHugeCountOnStreamClampsPreallocation() throws Exception { Schema schema = Schema.createArray(Schema.create(Schema.Type.LONG)); - GenericDatumReader reader = new GenericDatumReader<>(schema); + // Record the largest capacity ever requested from newArray so we can assert + // the preallocation is clamped rather than sized to the declared block count. + // Without this, a regression that reintroduces new Object[(int) count] could + // allocate ~200M slots and still pass (with an EOFException) on a large-heap + // JVM, hiding the regression. + final int[] maxRequestedCapacity = { 0 }; + GenericDatumReader reader = new GenericDatumReader(schema) { + @Override + protected Object newArray(Object old, int size, Schema schema) { + maxRequestedCapacity[0] = Math.max(maxRequestedCapacity[0], size); + return super.newArray(old, size, schema); + } + }; // A huge (non-zero-byte) block count followed by no element data. Wrapping in // a BufferedInputStream keeps the source from being a ByteArrayInputStream, so @@ -243,7 +256,14 @@ void arrayHugeCountOnStreamClampsPreallocation() throws Exception { byte[] data = encodeVarints(200_000_000L); InputStream stream = new BufferedInputStream(new ByteArrayInputStream(data)); BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(stream, null); + // Confirm the guard-disabling precondition: the decoder cannot report how + // many bytes remain, so this really is the unknown-remaining-bytes path. + assertEquals(-1, decoder.remainingBytes()); assertThrows(EOFException.class, () -> reader.read(null, decoder)); + // The declared count is 200,000,000 but the initial allocation must stay + // clamped to GenericDatumReader.initialCollectionCapacity (<= 1024). + assertTrue(maxRequestedCapacity[0] <= GenericDatumReader.initialCollectionCapacity(200_000_000L), + "preallocation was not clamped: requested capacity " + maxRequestedCapacity[0]); } /** From d01583e7841384071775e655d4fe01f993091337 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Mej=C3=ADa?= Date: Sun, 19 Jul 2026 14:54:32 +0200 Subject: [PATCH 25/25] AVRO-4300: [java] Address review nits: use isZeroByteSchema, tidy block-count guards Apply RyanSkraba's review feedback: - GenericDatumReader.readArray uses the isZeroByteSchema helper instead of inlining minBytesPerElement == 0, matching skip(). - BinaryDecoder.doReadItemCount rejects Long.MIN_VALUE before consuming the block byte-size, so no readLong runs once the count is invalid. - BinaryDecoder.doSkipItems drops the readLong before throwing on an invalid Long.MIN_VALUE count, consistent with doReadItemCount. - FastReaderBuilder.createArrayReader drops the redundant comment; the rationale is documented on checkArrayBlock. --- .../java/org/apache/avro/generic/GenericDatumReader.java | 2 +- .../src/main/java/org/apache/avro/io/BinaryDecoder.java | 5 ++--- .../src/main/java/org/apache/avro/io/FastReaderBuilder.java | 6 ------ 3 files changed, 3 insertions(+), 10 deletions(-) diff --git a/lang/java/avro/src/main/java/org/apache/avro/generic/GenericDatumReader.java b/lang/java/avro/src/main/java/org/apache/avro/generic/GenericDatumReader.java index cc72297a2eb..8a80d3ab641 100644 --- a/lang/java/avro/src/main/java/org/apache/avro/generic/GenericDatumReader.java +++ b/lang/java/avro/src/main/java/org/apache/avro/generic/GenericDatumReader.java @@ -324,7 +324,7 @@ protected Object readArray(Object old, Schema expected, ResolvingDecoder in) thr // remaining. Cap such collections against a heap-aware limit so a tiny // payload cannot declare a huge block count and drive an unbounded // backing-array allocation. - boolean zeroByteElements = minBytesPerElement(expectedType) == 0; + boolean zeroByteElements = isZeroByteSchema(expectedType); if (zeroByteElements) { SystemLimitException.checkMaxCollectionAllocation(base, l); } diff --git a/lang/java/avro/src/main/java/org/apache/avro/io/BinaryDecoder.java b/lang/java/avro/src/main/java/org/apache/avro/io/BinaryDecoder.java index c2e2ab8351c..3974eb5c621 100644 --- a/lang/java/avro/src/main/java/org/apache/avro/io/BinaryDecoder.java +++ b/lang/java/avro/src/main/java/org/apache/avro/io/BinaryDecoder.java @@ -408,8 +408,6 @@ protected void doReadBytes(byte[] bytes, int start, int length) throws IOExcepti protected long doReadItemCount() throws IOException { long result = readLong(); if (result < 0L) { - // A negative block count is followed by a block byte-size; consume it. - final long bytecount = readLong(); if (result == Long.MIN_VALUE) { // Long.MIN_VALUE cannot be negated (-Long.MIN_VALUE overflows back to // Long.MIN_VALUE), so it is not a valid block count. Reject it rather @@ -418,6 +416,8 @@ protected long doReadItemCount() throws IOException { // the end marker, desynchronizing decoding of subsequent fields. throw new AvroRuntimeException("Malformed data. Block count is invalid: " + result); } + // A negative block count is followed by a block byte-size; consume it. + final long bytecount = readLong(); if (bytecount < 0L) { // The block byte-size is a byte count and must be non-negative, matching // doSkipItems(). @@ -453,7 +453,6 @@ private long doSkipItems() throws IOException { // Consistent with doReadItemCount: Long.MIN_VALUE is not a valid block // count (it cannot be negated), so reject it rather than treating it as // a byte-sized block and continuing to skip. - readLong(); throw new AvroRuntimeException("Malformed data. Block count is invalid: " + result); } final long bytecount = readLong(); diff --git a/lang/java/avro/src/main/java/org/apache/avro/io/FastReaderBuilder.java b/lang/java/avro/src/main/java/org/apache/avro/io/FastReaderBuilder.java index 96767ad9934..8c61acf2018 100644 --- a/lang/java/avro/src/main/java/org/apache/avro/io/FastReaderBuilder.java +++ b/lang/java/avro/src/main/java/org/apache/avro/io/FastReaderBuilder.java @@ -476,12 +476,6 @@ private FieldReader createArrayReader(Schema readerSchema, Container action) thr FieldReader elementReader = getReaderFor(action.elementAction, null); Schema elementType = readerSchema.getElementType(); - // Elements whose minimum encoded size is zero (null, a record with only - // zero-byte fields, or a recursive schema whose cycle is broken with a 0 - // minimum) consume no guaranteed input, so the block count cannot be bounded - // by the bytes remaining in the stream. Cap such collections against a - // heap-aware limit so a tiny payload cannot declare a huge block count and - // drive an unbounded backing-array allocation (AVRO-4300). boolean zeroByteElements = GenericDatumReader.isZeroByteSchema(elementType); return reusingReader((reuse, decoder) -> {