From b803a42235578d4764c2614e1bcfc6f9087313a7 Mon Sep 17 00:00:00 2001 From: Siew Kam Onn Date: Tue, 16 Jun 2026 19:21:35 +0800 Subject: [PATCH 01/14] feat: enhance runtime casting for FixedSizeList and improve planner/runtime parity - Updated runtime casting to recognize FixedSizeList when source and target list sizes match, allowing recursive adaptation of the child struct via a new helper. - Extended planner/runtime parity by updating `validate_data_type_compatibility` and `requires_nested_struct_cast` to handle equal-size FixedSizeList children. test: add regression coverage for nested field changes - Added tests for: - Additive nullable nested-field evolution - Planner acceptance for compatible cases - Non-nullable added-field rejection - All-null column handling - Planner/runtime parity on incompatible nested type changes --- datafusion/common/src/nested_struct.rs | 244 ++++++++++++++++++++++++- 1 file changed, 240 insertions(+), 4 deletions(-) diff --git a/datafusion/common/src/nested_struct.rs b/datafusion/common/src/nested_struct.rs index cdd6215d08e2f..fc76aa95302d2 100644 --- a/datafusion/common/src/nested_struct.rs +++ b/datafusion/common/src/nested_struct.rs @@ -18,8 +18,8 @@ use crate::error::{_plan_err, Result}; use arrow::{ array::{ - Array, ArrayRef, DictionaryArray, GenericListArray, GenericListViewArray, - StructArray, downcast_integer, new_null_array, + Array, ArrayRef, DictionaryArray, FixedSizeListArray, GenericListArray, + GenericListViewArray, StructArray, downcast_integer, new_null_array, }, compute::{CastOptions, can_cast_types, cast_with_options}, datatypes::{DataType, DataType::Struct, Field, FieldRef}, @@ -183,6 +183,15 @@ pub fn cast_column( (DataType::LargeList(_), DataType::LargeList(target_inner)) => { cast_list_column::(source_col, target_inner, cast_options) } + ( + DataType::FixedSizeList(_, source_list_size), + DataType::FixedSizeList(target_inner, target_list_size), + ) if source_list_size == target_list_size => cast_fixed_size_list_column( + source_col, + target_inner, + *target_list_size, + cast_options, + ), (DataType::ListView(_), DataType::ListView(target_inner)) => { cast_list_view_column::(source_col, target_inner, cast_options) } @@ -264,6 +273,36 @@ fn cast_list_view_column( Ok(Arc::new(result)) } +fn cast_fixed_size_list_column( + source_col: &ArrayRef, + target_inner_field: &FieldRef, + target_list_size: i32, + cast_options: &CastOptions, +) -> Result { + let source_list = source_col + .as_any() + .downcast_ref::() + .ok_or_else(|| { + crate::error::DataFusionError::Plan(format!( + "Expected fixed-size list array but got {}", + source_col.data_type() + )) + })?; + + let cast_values = cast_column( + source_list.values(), + target_inner_field.data_type(), + cast_options, + )?; + + Ok(Arc::new(FixedSizeListArray::new( + Arc::clone(target_inner_field), + target_list_size, + cast_values, + source_list.nulls().cloned(), + ))) +} + fn cast_dictionary_column( source_col: &ArrayRef, source_key_type: &DataType, @@ -425,12 +464,22 @@ pub fn validate_data_type_compatibility( (Struct(source_nested), Struct(target_nested)) => { validate_struct_compatibility(source_nested, target_nested)?; } + ( + DataType::FixedSizeList(s, source_list_size), + DataType::FixedSizeList(t, target_list_size), + ) if source_list_size == target_list_size => { + validate_field_compatibility(s, t)?; + } (DataType::List(s), DataType::List(t)) | (DataType::LargeList(s), DataType::LargeList(t)) | (DataType::ListView(s), DataType::ListView(t)) | (DataType::LargeListView(s), DataType::LargeListView(t)) => { validate_field_compatibility(s, t)?; } + ( + DataType::FixedSizeList(_, source_list_size), + DataType::FixedSizeList(_, target_list_size), + ) if source_list_size != target_list_size => {} (DataType::Dictionary(s_key, s_val), DataType::Dictionary(t_key, t_val)) => { if !can_cast_types(s_key, t_key) { return _plan_err!( @@ -472,12 +521,22 @@ pub fn requires_nested_struct_cast( ) -> bool { match (source_type, target_type) { (Struct(_), Struct(_)) => true, + ( + DataType::FixedSizeList(s, source_list_size), + DataType::FixedSizeList(t, target_list_size), + ) if source_list_size == target_list_size => { + requires_nested_struct_cast(s.data_type(), t.data_type()) + } (DataType::List(s), DataType::List(t)) | (DataType::LargeList(s), DataType::LargeList(t)) | (DataType::ListView(s), DataType::ListView(t)) | (DataType::LargeListView(s), DataType::LargeListView(t)) => { requires_nested_struct_cast(s.data_type(), t.data_type()) } + ( + DataType::FixedSizeList(_, source_list_size), + DataType::FixedSizeList(_, target_list_size), + ) if source_list_size != target_list_size => false, (DataType::Dictionary(_, s_val), DataType::Dictionary(_, t_val)) => { requires_nested_struct_cast(s_val, t_val) } @@ -508,8 +567,9 @@ mod tests { use crate::{assert_contains, format::DEFAULT_CAST_OPTIONS}; use arrow::{ array::{ - BinaryArray, Int32Array, Int32Builder, Int64Array, ListArray, ListViewArray, - MapArray, MapBuilder, NullArray, StringArray, StringBuilder, + BinaryArray, FixedSizeListArray, Int32Array, Int32Builder, Int64Array, + ListArray, ListViewArray, MapArray, MapBuilder, NullArray, StringArray, + StringBuilder, }, buffer::{NullBuffer, ScalarBuffer}, datatypes::{DataType, Field, FieldRef, Int32Type}, @@ -1307,6 +1367,168 @@ mod tests { assert!(b_col.iter().all(|v| v.is_none())); } + #[test] + fn test_cast_fixed_size_list_struct() { + let struct_arr = StructArray::from(vec![( + arc_field("a", DataType::Int32), + Arc::new(Int32Array::from(vec![1, 2, 3, 4])) as ArrayRef, + )]); + + let source_field = + arc_field("item", struct_type(vec![field("a", DataType::Int32)])); + let target_field = arc_field( + "item", + struct_type(vec![ + field("a", DataType::Int64), + field("b", DataType::Utf8), + ]), + ); + let source_col: ArrayRef = Arc::new(FixedSizeListArray::new( + source_field, + 2, + Arc::new(struct_arr), + Some(NullBuffer::from(vec![true, false])), + )); + let target_type = DataType::FixedSizeList(target_field, 2); + + let result = + cast_column(&source_col, &target_type, &DEFAULT_CAST_OPTIONS).unwrap(); + let result_list = result + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(result_list.len(), 2); + assert!(result_list.is_valid(0)); + assert!(result_list.is_null(1)); + + let struct_values = result_list + .values() + .as_any() + .downcast_ref::() + .unwrap(); + let a_col = get_column_as!(&struct_values, "a", Int64Array); + assert_eq!(a_col.values(), &[1, 2, 3, 4]); + let b_col = get_column_as!(&struct_values, "b", StringArray); + assert!(b_col.iter().all(|v| v.is_none())); + } + + #[test] + fn test_validate_fixed_size_list_struct_compatibility() { + let source = DataType::FixedSizeList( + arc_field("item", struct_type(vec![field("a", DataType::Int32)])), + 2, + ); + let target = DataType::FixedSizeList( + arc_field( + "item", + struct_type(vec![ + field("a", DataType::Int64), + field("b", DataType::Utf8), + ]), + ), + 2, + ); + + assert!(requires_nested_struct_cast(&source, &target)); + assert!(validate_data_type_compatibility("col", &source, &target).is_ok()); + } + + #[test] + fn test_validate_fixed_size_list_struct_missing_non_nullable_field_rejected() { + let source = DataType::FixedSizeList( + arc_field("item", struct_type(vec![field("a", DataType::Int32)])), + 2, + ); + let target = DataType::FixedSizeList( + arc_field( + "item", + struct_type(vec![ + field("a", DataType::Int32), + non_null_field("b", DataType::Utf8), + ]), + ), + 2, + ); + + let error = validate_data_type_compatibility("col", &source, &target) + .unwrap_err() + .to_string(); + assert_contains!( + error, + "target field 'b' is non-nullable but missing from source" + ); + } + + #[test] + fn test_cast_fixed_size_list_struct_all_null() { + let source_field = + arc_field("item", struct_type(vec![field("a", DataType::Int32)])); + let target_field = arc_field( + "item", + struct_type(vec![ + field("a", DataType::Int64), + field("b", DataType::Utf8), + ]), + ); + let source_col: ArrayRef = + Arc::new(FixedSizeListArray::new_null(source_field, 2, 2)); + let target_type = DataType::FixedSizeList(target_field, 2); + + let result = + cast_column(&source_col, &target_type, &DEFAULT_CAST_OPTIONS).unwrap(); + let result_list = result + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(result_list.null_count(), 2); + + let struct_values = result_list + .values() + .as_any() + .downcast_ref::() + .unwrap(); + let a_col = get_column_as!(&struct_values, "a", Int64Array); + let b_col = get_column_as!(&struct_values, "b", StringArray); + assert!(a_col.iter().all(|v| v.is_none())); + assert!(b_col.iter().all(|v| v.is_none())); + } + + #[test] + fn test_fixed_size_list_struct_planner_runtime_parity_on_incompatible_type() { + let source_type = DataType::FixedSizeList( + arc_field("item", struct_type(vec![field("a", DataType::Binary)])), + 2, + ); + let target_type = DataType::FixedSizeList( + arc_field("item", struct_type(vec![field("a", DataType::Int32)])), + 2, + ); + let validation_error = + validate_data_type_compatibility("col", &source_type, &target_type) + .unwrap_err() + .to_string(); + assert_contains!(validation_error, "Cannot cast struct field 'a'"); + + let struct_arr = StructArray::from(vec![( + arc_field("a", DataType::Binary), + Arc::new(BinaryArray::from(vec![ + Some(b"x".as_ref()), + Some(b"y".as_ref()), + ])) as ArrayRef, + )]); + let source_col: ArrayRef = Arc::new(FixedSizeListArray::new( + arc_field("item", struct_type(vec![field("a", DataType::Binary)])), + 2, + Arc::new(struct_arr), + None, + )); + + let runtime_error = cast_column(&source_col, &target_type, &DEFAULT_CAST_OPTIONS) + .unwrap_err() + .to_string(); + assert_contains!(runtime_error, "Cannot cast struct field 'a'"); + } + #[test] fn test_requires_nested_struct_cast() { let s1 = struct_type(vec![field("a", DataType::Int32)]); @@ -1325,6 +1547,16 @@ mod tests { &DataType::ListView(arc_field("item", s1)), &DataType::ListView(arc_field("item", s2)), )); + assert!(requires_nested_struct_cast( + &DataType::FixedSizeList( + arc_field("item", struct_type(vec![field("a", DataType::Int32)])), + 2, + ), + &DataType::FixedSizeList( + arc_field("item", struct_type(vec![field("a", DataType::Int64)])), + 2, + ), + )); // Non-struct types should return false. assert!(!requires_nested_struct_cast( @@ -1335,5 +1567,9 @@ mod tests { &DataType::List(arc_field("item", DataType::Int32)), &DataType::List(arc_field("item", DataType::Int64)), )); + assert!(!requires_nested_struct_cast( + &DataType::FixedSizeList(arc_field("item", DataType::Int32), 2), + &DataType::FixedSizeList(arc_field("item", DataType::Int64), 2), + )); } } From 0e82dcb560ff8346aca058ad842060721814cd91 Mon Sep 17 00:00:00 2001 From: Siew Kam Onn Date: Tue, 16 Jun 2026 19:33:01 +0800 Subject: [PATCH 02/14] feat: add downcast_array macro for improved type safety in casting - Introduced `downcast_array!` macro to simplify downcasting of array types and improve error handling with descriptive messages. - Refactored existing functions `cast_list_column`, `cast_list_view_column`, and `cast_fixed_size_list_column` to utilize the new macro for improved readability and maintainability. - Added helper function `create_fsl_test_fields` to streamline test field creation for fixed-size list struct tests. - Updated related tests to use the new helper function for improved clarity and reduced redundancy. --- datafusion/common/src/nested_struct.rs | 117 ++++++++++++------------- 1 file changed, 58 insertions(+), 59 deletions(-) diff --git a/datafusion/common/src/nested_struct.rs b/datafusion/common/src/nested_struct.rs index fc76aa95302d2..9a39bed1526b7 100644 --- a/datafusion/common/src/nested_struct.rs +++ b/datafusion/common/src/nested_struct.rs @@ -26,6 +26,19 @@ use arrow::{ }; use std::{collections::HashSet, sync::Arc}; +/// Helper macro to downcast an array and convert to Result with appropriate error. +macro_rules! downcast_array { + ($col:expr, $ty:ty, $type_name:expr) => { + $col.as_any().downcast_ref::<$ty>().ok_or_else(|| { + crate::error::DataFusionError::Plan(format!( + "Expected {} but got {}", + $type_name, + $col.data_type() + )) + }) + }; +} + /// Cast a struct column to match target struct fields, handling nested structs recursively. /// /// This function implements struct-to-struct casting with the assumption that **structs should @@ -217,15 +230,7 @@ fn cast_list_column( target_inner_field: &FieldRef, cast_options: &CastOptions, ) -> Result { - let source_list = source_col - .as_any() - .downcast_ref::>() - .ok_or_else(|| { - crate::error::DataFusionError::Plan(format!( - "Expected list array but got {}", - source_col.data_type() - )) - })?; + let source_list = downcast_array!(source_col, GenericListArray, "list array")?; let cast_values = cast_column( source_list.values(), @@ -247,15 +252,8 @@ fn cast_list_view_column( target_inner_field: &FieldRef, cast_options: &CastOptions, ) -> Result { - let source_list = source_col - .as_any() - .downcast_ref::>() - .ok_or_else(|| { - crate::error::DataFusionError::Plan(format!( - "Expected list view array but got {}", - source_col.data_type() - )) - })?; + let source_list = + downcast_array!(source_col, GenericListViewArray, "list view array")?; let cast_values = cast_column( source_list.values(), @@ -279,15 +277,8 @@ fn cast_fixed_size_list_column( target_list_size: i32, cast_options: &CastOptions, ) -> Result { - let source_list = source_col - .as_any() - .downcast_ref::() - .ok_or_else(|| { - crate::error::DataFusionError::Plan(format!( - "Expected fixed-size list array but got {}", - source_col.data_type() - )) - })?; + let source_list = + downcast_array!(source_col, FixedSizeListArray, "fixed-size list array")?; let cast_values = cast_column( source_list.values(), @@ -1367,6 +1358,31 @@ mod tests { assert!(b_col.iter().all(|v| v.is_none())); } + fn create_fsl_test_fields( + source_struct_fields: Vec<(&str, DataType)>, + target_struct_fields: Vec<(&str, DataType)>, + ) -> (FieldRef, FieldRef) { + let source_field = arc_field( + "item", + struct_type( + source_struct_fields + .into_iter() + .map(|(name, dt)| field(name, dt)) + .collect(), + ), + ); + let target_field = arc_field( + "item", + struct_type( + target_struct_fields + .into_iter() + .map(|(name, dt)| field(name, dt)) + .collect(), + ), + ); + (source_field, target_field) + } + #[test] fn test_cast_fixed_size_list_struct() { let struct_arr = StructArray::from(vec![( @@ -1374,14 +1390,9 @@ mod tests { Arc::new(Int32Array::from(vec![1, 2, 3, 4])) as ArrayRef, )]); - let source_field = - arc_field("item", struct_type(vec![field("a", DataType::Int32)])); - let target_field = arc_field( - "item", - struct_type(vec![ - field("a", DataType::Int64), - field("b", DataType::Utf8), - ]), + let (source_field, target_field) = create_fsl_test_fields( + vec![("a", DataType::Int32)], + vec![("a", DataType::Int64), ("b", DataType::Utf8)], ); let source_col: ArrayRef = Arc::new(FixedSizeListArray::new( source_field, @@ -1414,20 +1425,12 @@ mod tests { #[test] fn test_validate_fixed_size_list_struct_compatibility() { - let source = DataType::FixedSizeList( - arc_field("item", struct_type(vec![field("a", DataType::Int32)])), - 2, - ); - let target = DataType::FixedSizeList( - arc_field( - "item", - struct_type(vec![ - field("a", DataType::Int64), - field("b", DataType::Utf8), - ]), - ), - 2, + let (source_field, target_field) = create_fsl_test_fields( + vec![("a", DataType::Int32)], + vec![("a", DataType::Int64), ("b", DataType::Utf8)], ); + let source = DataType::FixedSizeList(source_field, 2); + let target = DataType::FixedSizeList(target_field, 2); assert!(requires_nested_struct_cast(&source, &target)); assert!(validate_data_type_compatibility("col", &source, &target).is_ok()); @@ -1435,10 +1438,11 @@ mod tests { #[test] fn test_validate_fixed_size_list_struct_missing_non_nullable_field_rejected() { - let source = DataType::FixedSizeList( - arc_field("item", struct_type(vec![field("a", DataType::Int32)])), - 2, + let (source_field, _) = create_fsl_test_fields( + vec![("a", DataType::Int32)], + vec![("a", DataType::Int64), ("b", DataType::Utf8)], ); + let source = DataType::FixedSizeList(source_field, 2); let target = DataType::FixedSizeList( arc_field( "item", @@ -1461,14 +1465,9 @@ mod tests { #[test] fn test_cast_fixed_size_list_struct_all_null() { - let source_field = - arc_field("item", struct_type(vec![field("a", DataType::Int32)])); - let target_field = arc_field( - "item", - struct_type(vec![ - field("a", DataType::Int64), - field("b", DataType::Utf8), - ]), + let (source_field, target_field) = create_fsl_test_fields( + vec![("a", DataType::Int32)], + vec![("a", DataType::Int64), ("b", DataType::Utf8)], ); let source_col: ArrayRef = Arc::new(FixedSizeListArray::new_null(source_field, 2, 2)); From 62c5adffe8c9320067502cea13c211093c43f591 Mon Sep 17 00:00:00 2001 From: Siew Kam Onn Date: Tue, 16 Jun 2026 19:40:59 +0800 Subject: [PATCH 03/14] fix: close FixedSizeList size-mismatch loophole in nested_struct.rs - Prevent validation from accepting unsupported length changes - Ensure planner behavior aligns with runtime casting - Add regression test for rejected mismatch case --- datafusion/common/src/nested_struct.rs | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/datafusion/common/src/nested_struct.rs b/datafusion/common/src/nested_struct.rs index 9a39bed1526b7..3c01a2910c2aa 100644 --- a/datafusion/common/src/nested_struct.rs +++ b/datafusion/common/src/nested_struct.rs @@ -467,10 +467,6 @@ pub fn validate_data_type_compatibility( | (DataType::LargeListView(s), DataType::LargeListView(t)) => { validate_field_compatibility(s, t)?; } - ( - DataType::FixedSizeList(_, source_list_size), - DataType::FixedSizeList(_, target_list_size), - ) if source_list_size != target_list_size => {} (DataType::Dictionary(s_key, s_val), DataType::Dictionary(t_key, t_val)) => { if !can_cast_types(s_key, t_key) { return _plan_err!( @@ -1463,6 +1459,21 @@ mod tests { ); } + #[test] + fn test_validate_fixed_size_list_struct_size_mismatch_rejected() { + let (source_field, target_field) = create_fsl_test_fields( + vec![("a", DataType::Int32)], + vec![("a", DataType::Int64), ("b", DataType::Utf8)], + ); + let source = DataType::FixedSizeList(source_field, 2); + let target = DataType::FixedSizeList(target_field, 3); + + let error = validate_data_type_compatibility("col", &source, &target) + .unwrap_err() + .to_string(); + assert_contains!(error, "Cannot cast struct field 'col'"); + } + #[test] fn test_cast_fixed_size_list_struct_all_null() { let (source_field, target_field) = create_fsl_test_fields( From b561ef97562b0f290d3f7b5fd74cd5c3c612781a Mon Sep 17 00:00:00 2001 From: Siew Kam Onn Date: Tue, 16 Jun 2026 19:57:21 +0800 Subject: [PATCH 04/14] refactor: rename create_fsl_test_fields to create_fixed_size_list_test_fields for clarity --- datafusion/common/src/nested_struct.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/datafusion/common/src/nested_struct.rs b/datafusion/common/src/nested_struct.rs index 3c01a2910c2aa..ebedbaa7ed5b6 100644 --- a/datafusion/common/src/nested_struct.rs +++ b/datafusion/common/src/nested_struct.rs @@ -1354,7 +1354,7 @@ mod tests { assert!(b_col.iter().all(|v| v.is_none())); } - fn create_fsl_test_fields( + fn create_fixed_size_list_test_fields( source_struct_fields: Vec<(&str, DataType)>, target_struct_fields: Vec<(&str, DataType)>, ) -> (FieldRef, FieldRef) { @@ -1386,7 +1386,7 @@ mod tests { Arc::new(Int32Array::from(vec![1, 2, 3, 4])) as ArrayRef, )]); - let (source_field, target_field) = create_fsl_test_fields( + let (source_field, target_field) = create_fixed_size_list_test_fields( vec![("a", DataType::Int32)], vec![("a", DataType::Int64), ("b", DataType::Utf8)], ); @@ -1421,7 +1421,7 @@ mod tests { #[test] fn test_validate_fixed_size_list_struct_compatibility() { - let (source_field, target_field) = create_fsl_test_fields( + let (source_field, target_field) = create_fixed_size_list_test_fields( vec![("a", DataType::Int32)], vec![("a", DataType::Int64), ("b", DataType::Utf8)], ); @@ -1434,7 +1434,7 @@ mod tests { #[test] fn test_validate_fixed_size_list_struct_missing_non_nullable_field_rejected() { - let (source_field, _) = create_fsl_test_fields( + let (source_field, _) = create_fixed_size_list_test_fields( vec![("a", DataType::Int32)], vec![("a", DataType::Int64), ("b", DataType::Utf8)], ); @@ -1461,7 +1461,7 @@ mod tests { #[test] fn test_validate_fixed_size_list_struct_size_mismatch_rejected() { - let (source_field, target_field) = create_fsl_test_fields( + let (source_field, target_field) = create_fixed_size_list_test_fields( vec![("a", DataType::Int32)], vec![("a", DataType::Int64), ("b", DataType::Utf8)], ); @@ -1476,7 +1476,7 @@ mod tests { #[test] fn test_cast_fixed_size_list_struct_all_null() { - let (source_field, target_field) = create_fsl_test_fields( + let (source_field, target_field) = create_fixed_size_list_test_fields( vec![("a", DataType::Int32)], vec![("a", DataType::Int64), ("b", DataType::Utf8)], ); From 3286a0c1e31d046b4293429d3b891eeb8607ca49 Mon Sep 17 00:00:00 2001 From: Siew Kam Onn Date: Tue, 16 Jun 2026 21:58:18 +0800 Subject: [PATCH 05/14] feat(tests): add regression and parity tests for FixedSizeList behavior - Added regression test for null FixedSizeList parent hiding invalid child values. - Implemented runtime retry mechanism for child slots masked as null when cast fails due to hidden null-parent values. - Added runtime parity test to ensure missing non-nullable nested fields are still rejected. --- datafusion/common/src/nested_struct.rs | 126 ++++++++++++++++++++++++- 1 file changed, 123 insertions(+), 3 deletions(-) diff --git a/datafusion/common/src/nested_struct.rs b/datafusion/common/src/nested_struct.rs index ebedbaa7ed5b6..524c927907e7d 100644 --- a/datafusion/common/src/nested_struct.rs +++ b/datafusion/common/src/nested_struct.rs @@ -19,8 +19,9 @@ use crate::error::{_plan_err, Result}; use arrow::{ array::{ Array, ArrayRef, DictionaryArray, FixedSizeListArray, GenericListArray, - GenericListViewArray, StructArray, downcast_integer, new_null_array, + GenericListViewArray, StructArray, downcast_integer, make_array, new_null_array, }, + buffer::NullBuffer, compute::{CastOptions, can_cast_types, cast_with_options}, datatypes::{DataType, DataType::Struct, Field, FieldRef}, }; @@ -280,11 +281,30 @@ fn cast_fixed_size_list_column( let source_list = downcast_array!(source_col, FixedSizeListArray, "fixed-size list array")?; - let cast_values = cast_column( + validate_data_type_compatibility( + target_inner_field.name(), + source_list.values().data_type(), + target_inner_field.data_type(), + )?; + + let cast_values = match cast_column( source_list.values(), target_inner_field.data_type(), cast_options, - )?; + ) { + Ok(cast_values) => cast_values, + Err(error) => match source_list.nulls() { + Some(parent_nulls) if parent_nulls.null_count() > 0 => { + let values = mask_fixed_size_list_hidden_values( + source_list.values(), + parent_nulls, + target_list_size, + )?; + cast_column(&values, target_inner_field.data_type(), cast_options)? + } + _ => return Err(error), + }, + }; Ok(Arc::new(FixedSizeListArray::new( Arc::clone(target_inner_field), @@ -294,6 +314,42 @@ fn cast_fixed_size_list_column( ))) } +fn mask_fixed_size_list_hidden_values( + values: &ArrayRef, + parent_nulls: &NullBuffer, + list_size: i32, +) -> Result { + let hidden_child_nulls = parent_nulls.expand(list_size as usize); + mask_array_values(values, &hidden_child_nulls) +} + +fn mask_array_values( + values: &ArrayRef, + additional_nulls: &NullBuffer, +) -> Result { + let nulls = NullBuffer::union(values.nulls(), Some(additional_nulls)); + + if let Some(struct_array) = values.as_any().downcast_ref::() { + let struct_nulls = nulls + .as_ref() + .expect("additional nulls always produce nulls"); + let arrays = struct_array + .columns() + .iter() + .map(|child| mask_array_values(child, struct_nulls)) + .collect::>>()?; + return Ok(Arc::new(StructArray::new( + struct_array.fields().clone(), + arrays, + nulls, + ))); + } + + Ok(make_array( + values.to_data().into_builder().nulls(nulls).build()?, + )) +} + fn cast_dictionary_column( source_col: &ArrayRef, source_key_type: &DataType, @@ -1539,6 +1595,70 @@ mod tests { assert_contains!(runtime_error, "Cannot cast struct field 'a'"); } + #[test] + fn test_cast_fixed_size_list_struct_missing_non_nullable_field_runtime_rejected() { + let source_field = + arc_field("item", struct_type(vec![field("a", DataType::Int32)])); + let target_field = arc_field( + "item", + struct_type(vec![ + field("a", DataType::Int32), + non_null_field("b", DataType::Utf8), + ]), + ); + let source_col: ArrayRef = + Arc::new(FixedSizeListArray::new_null(source_field, 2, 1)); + let target_type = DataType::FixedSizeList(target_field, 2); + + let error = cast_column(&source_col, &target_type, &DEFAULT_CAST_OPTIONS) + .unwrap_err() + .to_string(); + assert_contains!( + error, + "target field 'b' is non-nullable but missing from source" + ); + } + + #[test] + fn test_cast_fixed_size_list_struct_ignores_hidden_child_values_for_null_parent() { + let source_field = + arc_field("item", struct_type(vec![field("a", DataType::Utf8)])); + let target_field = + arc_field("item", struct_type(vec![field("a", DataType::Int32)])); + let struct_arr = StructArray::from(vec![( + arc_field("a", DataType::Utf8), + Arc::new(StringArray::from(vec!["not_int", "also_bad", "1", "2"])) + as ArrayRef, + )]); + let source_col: ArrayRef = Arc::new(FixedSizeListArray::new( + source_field, + 2, + Arc::new(struct_arr), + Some(NullBuffer::from(vec![false, true])), + )); + let target_type = DataType::FixedSizeList(target_field, 2); + + let result = + cast_column(&source_col, &target_type, &DEFAULT_CAST_OPTIONS).unwrap(); + let result_list = result + .as_any() + .downcast_ref::() + .unwrap(); + assert!(result_list.is_null(0)); + assert!(result_list.is_valid(1)); + + let struct_values = result_list + .values() + .as_any() + .downcast_ref::() + .unwrap(); + let a_col = get_column_as!(&struct_values, "a", Int32Array); + assert!(a_col.is_null(0)); + assert!(a_col.is_null(1)); + assert_eq!(a_col.value(2), 1); + assert_eq!(a_col.value(3), 2); + } + #[test] fn test_requires_nested_struct_cast() { let s1 = struct_type(vec![field("a", DataType::Int32)]); From 38bdcf7910a744cd1bfb2f76e21d6083909b4cc5 Mon Sep 17 00:00:00 2001 From: Siew Kam Onn Date: Tue, 16 Jun 2026 22:05:39 +0800 Subject: [PATCH 06/14] chore: refactor code for improved clarity and reduced redundancy - Removed redundant FixedSizeList size-mismatch match arm - Bound repeated source_values and target_type - Renamed masked fallback local for better readability - Inlined single-use mask wrapper - Added test helpers to reduce duplication in tests - Reused fields in parity test for efficiency - Reused s1 and s2 in requires_nested_struct_cast test to minimize duplication --- datafusion/common/src/nested_struct.rs | 144 +++++++++---------------- 1 file changed, 51 insertions(+), 93 deletions(-) diff --git a/datafusion/common/src/nested_struct.rs b/datafusion/common/src/nested_struct.rs index 524c927907e7d..effe2b891cd0d 100644 --- a/datafusion/common/src/nested_struct.rs +++ b/datafusion/common/src/nested_struct.rs @@ -281,26 +281,23 @@ fn cast_fixed_size_list_column( let source_list = downcast_array!(source_col, FixedSizeListArray, "fixed-size list array")?; + let source_values = source_list.values(); + let target_type = target_inner_field.data_type(); + validate_data_type_compatibility( target_inner_field.name(), - source_list.values().data_type(), - target_inner_field.data_type(), + source_values.data_type(), + target_type, )?; - let cast_values = match cast_column( - source_list.values(), - target_inner_field.data_type(), - cast_options, - ) { + let cast_values = match cast_column(source_values, target_type, cast_options) { Ok(cast_values) => cast_values, Err(error) => match source_list.nulls() { Some(parent_nulls) if parent_nulls.null_count() > 0 => { - let values = mask_fixed_size_list_hidden_values( - source_list.values(), - parent_nulls, - target_list_size, - )?; - cast_column(&values, target_inner_field.data_type(), cast_options)? + let hidden_child_nulls = parent_nulls.expand(target_list_size as usize); + let masked_values = + mask_array_values(source_values, &hidden_child_nulls)?; + cast_column(&masked_values, target_type, cast_options)? } _ => return Err(error), }, @@ -314,15 +311,6 @@ fn cast_fixed_size_list_column( ))) } -fn mask_fixed_size_list_hidden_values( - values: &ArrayRef, - parent_nulls: &NullBuffer, - list_size: i32, -) -> Result { - let hidden_child_nulls = parent_nulls.expand(list_size as usize); - mask_array_values(values, &hidden_child_nulls) -} - fn mask_array_values( values: &ArrayRef, additional_nulls: &NullBuffer, @@ -576,10 +564,6 @@ pub fn requires_nested_struct_cast( | (DataType::LargeListView(s), DataType::LargeListView(t)) => { requires_nested_struct_cast(s.data_type(), t.data_type()) } - ( - DataType::FixedSizeList(_, source_list_size), - DataType::FixedSizeList(_, target_list_size), - ) if source_list_size != target_list_size => false, (DataType::Dictionary(_, s_val), DataType::Dictionary(_, t_val)) => { requires_nested_struct_cast(s_val, t_val) } @@ -1410,29 +1394,38 @@ mod tests { assert!(b_col.iter().all(|v| v.is_none())); } - fn create_fixed_size_list_test_fields( - source_struct_fields: Vec<(&str, DataType)>, - target_struct_fields: Vec<(&str, DataType)>, - ) -> (FieldRef, FieldRef) { - let source_field = arc_field( + fn fixed_size_list_struct_field(fields: Vec<(&str, DataType)>) -> FieldRef { + arc_field( "item", struct_type( - source_struct_fields + fields .into_iter() - .map(|(name, dt)| field(name, dt)) + .map(|(name, data_type)| field(name, data_type)) .collect(), ), - ); - let target_field = arc_field( - "item", - struct_type( - target_struct_fields - .into_iter() - .map(|(name, dt)| field(name, dt)) - .collect(), - ), - ); - (source_field, target_field) + ) + } + + fn create_fixed_size_list_test_fields( + source_struct_fields: Vec<(&str, DataType)>, + target_struct_fields: Vec<(&str, DataType)>, + ) -> (FieldRef, FieldRef) { + ( + fixed_size_list_struct_field(source_struct_fields), + fixed_size_list_struct_field(target_struct_fields), + ) + } + + fn fixed_size_list_struct_values( + array: &ArrayRef, + ) -> (&FixedSizeListArray, &StructArray) { + let list = array.as_any().downcast_ref::().unwrap(); + let values = list + .values() + .as_any() + .downcast_ref::() + .unwrap(); + (list, values) } #[test] @@ -1456,19 +1449,10 @@ mod tests { let result = cast_column(&source_col, &target_type, &DEFAULT_CAST_OPTIONS).unwrap(); - let result_list = result - .as_any() - .downcast_ref::() - .unwrap(); + let (result_list, struct_values) = fixed_size_list_struct_values(&result); assert_eq!(result_list.len(), 2); assert!(result_list.is_valid(0)); assert!(result_list.is_null(1)); - - let struct_values = result_list - .values() - .as_any() - .downcast_ref::() - .unwrap(); let a_col = get_column_as!(&struct_values, "a", Int64Array); assert_eq!(a_col.values(), &[1, 2, 3, 4]); let b_col = get_column_as!(&struct_values, "b", StringArray); @@ -1542,17 +1526,8 @@ mod tests { let result = cast_column(&source_col, &target_type, &DEFAULT_CAST_OPTIONS).unwrap(); - let result_list = result - .as_any() - .downcast_ref::() - .unwrap(); + let (result_list, struct_values) = fixed_size_list_struct_values(&result); assert_eq!(result_list.null_count(), 2); - - let struct_values = result_list - .values() - .as_any() - .downcast_ref::() - .unwrap(); let a_col = get_column_as!(&struct_values, "a", Int64Array); let b_col = get_column_as!(&struct_values, "b", StringArray); assert!(a_col.iter().all(|v| v.is_none())); @@ -1561,14 +1536,12 @@ mod tests { #[test] fn test_fixed_size_list_struct_planner_runtime_parity_on_incompatible_type() { - let source_type = DataType::FixedSizeList( - arc_field("item", struct_type(vec![field("a", DataType::Binary)])), - 2, - ); - let target_type = DataType::FixedSizeList( - arc_field("item", struct_type(vec![field("a", DataType::Int32)])), - 2, - ); + let source_field = + arc_field("item", struct_type(vec![field("a", DataType::Binary)])); + let target_field = + arc_field("item", struct_type(vec![field("a", DataType::Int32)])); + let source_type = DataType::FixedSizeList(Arc::clone(&source_field), 2); + let target_type = DataType::FixedSizeList(target_field, 2); let validation_error = validate_data_type_compatibility("col", &source_type, &target_type) .unwrap_err() @@ -1583,7 +1556,7 @@ mod tests { ])) as ArrayRef, )]); let source_col: ArrayRef = Arc::new(FixedSizeListArray::new( - arc_field("item", struct_type(vec![field("a", DataType::Binary)])), + source_field, 2, Arc::new(struct_arr), None, @@ -1640,18 +1613,9 @@ mod tests { let result = cast_column(&source_col, &target_type, &DEFAULT_CAST_OPTIONS).unwrap(); - let result_list = result - .as_any() - .downcast_ref::() - .unwrap(); + let (result_list, struct_values) = fixed_size_list_struct_values(&result); assert!(result_list.is_null(0)); assert!(result_list.is_valid(1)); - - let struct_values = result_list - .values() - .as_any() - .downcast_ref::() - .unwrap(); let a_col = get_column_as!(&struct_values, "a", Int32Array); assert!(a_col.is_null(0)); assert!(a_col.is_null(1)); @@ -1674,18 +1638,12 @@ mod tests { &DataType::Dictionary(Box::new(DataType::Int32), Box::new(s2.clone())), )); assert!(requires_nested_struct_cast( - &DataType::ListView(arc_field("item", s1)), - &DataType::ListView(arc_field("item", s2)), + &DataType::ListView(arc_field("item", s1.clone())), + &DataType::ListView(arc_field("item", s2.clone())), )); assert!(requires_nested_struct_cast( - &DataType::FixedSizeList( - arc_field("item", struct_type(vec![field("a", DataType::Int32)])), - 2, - ), - &DataType::FixedSizeList( - arc_field("item", struct_type(vec![field("a", DataType::Int64)])), - 2, - ), + &DataType::FixedSizeList(arc_field("item", s1), 2), + &DataType::FixedSizeList(arc_field("item", s2), 2), )); // Non-struct types should return false. From d8c1b17fddff6adfe8ba9585d3d33e72079b2673 Mon Sep 17 00:00:00 2001 From: Siew Kam Onn Date: Thu, 18 Jun 2026 12:58:31 +0800 Subject: [PATCH 07/14] feat: refactor nested_struct to use Arrow methods and remove downcast_array! - Removed local `downcast_array!` implementation - Used Arrow methods: - `source_col.as_list::()` - `source_col.as_list_view::()` - `source_col.as_fixed_size_list()` - Added `AsArray` import --- datafusion/common/src/nested_struct.rs | 23 ++++------------------- 1 file changed, 4 insertions(+), 19 deletions(-) diff --git a/datafusion/common/src/nested_struct.rs b/datafusion/common/src/nested_struct.rs index effe2b891cd0d..6b7340d2bdb59 100644 --- a/datafusion/common/src/nested_struct.rs +++ b/datafusion/common/src/nested_struct.rs @@ -18,7 +18,7 @@ use crate::error::{_plan_err, Result}; use arrow::{ array::{ - Array, ArrayRef, DictionaryArray, FixedSizeListArray, GenericListArray, + Array, ArrayRef, AsArray, DictionaryArray, FixedSizeListArray, GenericListArray, GenericListViewArray, StructArray, downcast_integer, make_array, new_null_array, }, buffer::NullBuffer, @@ -27,19 +27,6 @@ use arrow::{ }; use std::{collections::HashSet, sync::Arc}; -/// Helper macro to downcast an array and convert to Result with appropriate error. -macro_rules! downcast_array { - ($col:expr, $ty:ty, $type_name:expr) => { - $col.as_any().downcast_ref::<$ty>().ok_or_else(|| { - crate::error::DataFusionError::Plan(format!( - "Expected {} but got {}", - $type_name, - $col.data_type() - )) - }) - }; -} - /// Cast a struct column to match target struct fields, handling nested structs recursively. /// /// This function implements struct-to-struct casting with the assumption that **structs should @@ -231,7 +218,7 @@ fn cast_list_column( target_inner_field: &FieldRef, cast_options: &CastOptions, ) -> Result { - let source_list = downcast_array!(source_col, GenericListArray, "list array")?; + let source_list = source_col.as_list::(); let cast_values = cast_column( source_list.values(), @@ -253,8 +240,7 @@ fn cast_list_view_column( target_inner_field: &FieldRef, cast_options: &CastOptions, ) -> Result { - let source_list = - downcast_array!(source_col, GenericListViewArray, "list view array")?; + let source_list = source_col.as_list_view::(); let cast_values = cast_column( source_list.values(), @@ -278,8 +264,7 @@ fn cast_fixed_size_list_column( target_list_size: i32, cast_options: &CastOptions, ) -> Result { - let source_list = - downcast_array!(source_col, FixedSizeListArray, "fixed-size list array")?; + let source_list = source_col.as_fixed_size_list(); let source_values = source_list.values(); let target_type = target_inner_field.data_type(); From 70237b39a1f03cf62e1ed4e75e728d9d4bb73d71 Mon Sep 17 00:00:00 2001 From: Siew Kam Onn Date: Thu, 18 Jun 2026 14:04:01 +0800 Subject: [PATCH 08/14] feat: enhance data type validation with improved fallback structure - Added comment before `validate_data_type_compatibility` to explain planner/runtime guard. - Restructured fallback into an explicit helper: - Only attempts masked retry when parent nulls are present. - Returns original cast error in all other cases. - Clarified the split between guard and fallback logic. --- datafusion/common/src/nested_struct.rs | 35 ++++++++++++++++++++------ 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/datafusion/common/src/nested_struct.rs b/datafusion/common/src/nested_struct.rs index 6b7340d2bdb59..0fb5b58921e41 100644 --- a/datafusion/common/src/nested_struct.rs +++ b/datafusion/common/src/nested_struct.rs @@ -269,6 +269,10 @@ fn cast_fixed_size_list_column( let source_values = source_list.values(); let target_type = target_inner_field.data_type(); + // Guard schema compatibility before masking null-parent child values below. + // The retry path is only for value-level cast failures in child slots hidden + // by a null parent list, and must not make runtime accept schemas that + // planning rejects. validate_data_type_compatibility( target_inner_field.name(), source_values.data_type(), @@ -277,14 +281,15 @@ fn cast_fixed_size_list_column( let cast_values = match cast_column(source_values, target_type, cast_options) { Ok(cast_values) => cast_values, - Err(error) => match source_list.nulls() { - Some(parent_nulls) if parent_nulls.null_count() > 0 => { - let hidden_child_nulls = parent_nulls.expand(target_list_size as usize); - let masked_values = - mask_array_values(source_values, &hidden_child_nulls)?; - cast_column(&masked_values, target_type, cast_options)? - } - _ => return Err(error), + Err(error) => match cast_fixed_size_list_values_with_parent_nulls( + source_values, + target_type, + cast_options, + source_list.nulls(), + target_list_size, + ) { + Some(masked_cast) => masked_cast?, + None => return Err(error), }, }; @@ -296,6 +301,20 @@ fn cast_fixed_size_list_column( ))) } +fn cast_fixed_size_list_values_with_parent_nulls( + source_values: &ArrayRef, + target_type: &DataType, + cast_options: &CastOptions, + parent_nulls: Option<&NullBuffer>, + list_size: i32, +) -> Option> { + let parent_nulls = parent_nulls.filter(|nulls| nulls.null_count() > 0)?; + + let hidden_child_nulls = parent_nulls.expand(list_size as usize); + let masked_values = mask_array_values(source_values, &hidden_child_nulls); + Some(masked_values.and_then(|values| cast_column(&values, target_type, cast_options))) +} + fn mask_array_values( values: &ArrayRef, additional_nulls: &NullBuffer, From 040691bacf94645fe6e426e293d9f335a0ab1b66 Mon Sep 17 00:00:00 2001 From: Siew Kam Onn Date: Thu, 18 Jun 2026 14:07:49 +0800 Subject: [PATCH 09/14] feat(nested_struct): add targeted fallback comments for FixedSizeList behavior - Explain hidden child slots handling for null parents - Clarify that retries occur only for parent-null masking - State that behavior remains unchanged for scenarios without parent nulls, retaining original cast error --- datafusion/common/src/nested_struct.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/datafusion/common/src/nested_struct.rs b/datafusion/common/src/nested_struct.rs index 0fb5b58921e41..7d2128b57c817 100644 --- a/datafusion/common/src/nested_struct.rs +++ b/datafusion/common/src/nested_struct.rs @@ -310,6 +310,10 @@ fn cast_fixed_size_list_values_with_parent_nulls( ) -> Option> { let parent_nulls = parent_nulls.filter(|nulls| nulls.null_count() > 0)?; + // FixedSizeList stores child slots for null parent lists. Those child + // values are semantically hidden, but recursive casts still inspect them. + // Only in that case, mask hidden child slots to null and retry; if there + // are no parent nulls, propagate the original cast error unchanged. let hidden_child_nulls = parent_nulls.expand(list_size as usize); let masked_values = mask_array_values(source_values, &hidden_child_nulls); Some(masked_values.and_then(|values| cast_column(&values, target_type, cast_options))) From ced5b4143423ac532cfbefddf9a3eff7d6821c7a Mon Sep 17 00:00:00 2001 From: Siew Kam Onn Date: Mon, 29 Jun 2026 11:19:49 +0800 Subject: [PATCH 10/14] feat(nested_struct): refine compatibility checks and restructure casting logic - Removed fixed-size-list compatibility check from the normal cast path. - Moved compatibility guard into the null-parent retry path only. - Adjusted cast_struct_column all-null fast path to follow struct compatibility validation, ensuring all-null structs reject missing non-nullable fields. --- datafusion/common/src/nested_struct.rs | 37 +++++++++++++++----------- 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/datafusion/common/src/nested_struct.rs b/datafusion/common/src/nested_struct.rs index 7d2128b57c817..dbc20ed1b23f4 100644 --- a/datafusion/common/src/nested_struct.rs +++ b/datafusion/common/src/nested_struct.rs @@ -59,9 +59,7 @@ fn cast_struct_column( target_fields: &[Arc], cast_options: &CastOptions, ) -> Result { - if source_col.data_type() == &DataType::Null - || (!source_col.is_empty() && source_col.null_count() == source_col.len()) - { + if source_col.data_type() == &DataType::Null { return Ok(new_null_array( &Struct(target_fields.to_vec().into()), source_col.len(), @@ -71,6 +69,14 @@ fn cast_struct_column( if let Some(source_struct) = source_col.as_any().downcast_ref::() { let source_fields = source_struct.fields(); validate_struct_compatibility(source_fields, target_fields)?; + + if !source_col.is_empty() && source_col.null_count() == source_col.len() { + return Ok(new_null_array( + &Struct(target_fields.to_vec().into()), + source_col.len(), + )); + } + let mut fields: Vec> = Vec::with_capacity(target_fields.len()); let mut arrays: Vec = Vec::with_capacity(target_fields.len()); let num_rows = source_col.len(); @@ -269,16 +275,6 @@ fn cast_fixed_size_list_column( let source_values = source_list.values(); let target_type = target_inner_field.data_type(); - // Guard schema compatibility before masking null-parent child values below. - // The retry path is only for value-level cast failures in child slots hidden - // by a null parent list, and must not make runtime accept schemas that - // planning rejects. - validate_data_type_compatibility( - target_inner_field.name(), - source_values.data_type(), - target_type, - )?; - let cast_values = match cast_column(source_values, target_type, cast_options) { Ok(cast_values) => cast_values, Err(error) => match cast_fixed_size_list_values_with_parent_nulls( @@ -287,6 +283,7 @@ fn cast_fixed_size_list_column( cast_options, source_list.nulls(), target_list_size, + target_inner_field.name(), ) { Some(masked_cast) => masked_cast?, None => return Err(error), @@ -307,13 +304,23 @@ fn cast_fixed_size_list_values_with_parent_nulls( cast_options: &CastOptions, parent_nulls: Option<&NullBuffer>, list_size: i32, + field_name: &str, ) -> Option> { let parent_nulls = parent_nulls.filter(|nulls| nulls.null_count() > 0)?; // FixedSizeList stores child slots for null parent lists. Those child // values are semantically hidden, but recursive casts still inspect them. - // Only in that case, mask hidden child slots to null and retry; if there - // are no parent nulls, propagate the original cast error unchanged. + // Before masking and retrying, guard schema compatibility so the retry only + // handles value-level failures in hidden child slots and cannot make runtime + // accept schemas that planning rejects. + if let Err(error) = validate_data_type_compatibility( + field_name, + source_values.data_type(), + target_type, + ) { + return Some(Err(error)); + } + let hidden_child_nulls = parent_nulls.expand(list_size as usize); let masked_values = mask_array_values(source_values, &hidden_child_nulls); Some(masked_values.and_then(|values| cast_column(&values, target_type, cast_options))) From 96ab3496de18fa4daeabd6de0b4e3c332db2d9a1 Mon Sep 17 00:00:00 2001 From: Siew Kam Onn Date: Fri, 24 Jul 2026 15:48:39 +0800 Subject: [PATCH 11/14] feat(datafusion): simplify nested struct handling by removing compatibility validation and field_name plumbing - Removed retry-path compatibility validation. - Eliminated field_name plumbing. - Retained recursive cast enforcement. --- datafusion/common/src/nested_struct.rs | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/datafusion/common/src/nested_struct.rs b/datafusion/common/src/nested_struct.rs index dbc20ed1b23f4..74c6ce2f771a8 100644 --- a/datafusion/common/src/nested_struct.rs +++ b/datafusion/common/src/nested_struct.rs @@ -283,7 +283,6 @@ fn cast_fixed_size_list_column( cast_options, source_list.nulls(), target_list_size, - target_inner_field.name(), ) { Some(masked_cast) => masked_cast?, None => return Err(error), @@ -304,23 +303,11 @@ fn cast_fixed_size_list_values_with_parent_nulls( cast_options: &CastOptions, parent_nulls: Option<&NullBuffer>, list_size: i32, - field_name: &str, ) -> Option> { let parent_nulls = parent_nulls.filter(|nulls| nulls.null_count() > 0)?; // FixedSizeList stores child slots for null parent lists. Those child // values are semantically hidden, but recursive casts still inspect them. - // Before masking and retrying, guard schema compatibility so the retry only - // handles value-level failures in hidden child slots and cannot make runtime - // accept schemas that planning rejects. - if let Err(error) = validate_data_type_compatibility( - field_name, - source_values.data_type(), - target_type, - ) { - return Some(Err(error)); - } - let hidden_child_nulls = parent_nulls.expand(list_size as usize); let masked_values = mask_array_values(source_values, &hidden_child_nulls); Some(masked_values.and_then(|values| cast_column(&values, target_type, cast_options))) From a59ede741a46dc6b8c54d9d07a05bd70beeeace4 Mon Sep 17 00:00:00 2001 From: Siew Kam Onn Date: Fri, 24 Jul 2026 15:56:04 +0800 Subject: [PATCH 12/14] feat: add FixedSizeList Parquet E2E coverage - Implemented old/new schema evolution via ListingTable - Added support for SELECT * - Enabled nested-field projection - Implemented rejection for missing non-nullable fields - Added rejection for incompatible fields - Corrected existing negative fixture to truly omit chain --- datafusion/core/tests/parquet/expr_adapter.rs | 124 ++++++++++++++---- 1 file changed, 101 insertions(+), 23 deletions(-) diff --git a/datafusion/core/tests/parquet/expr_adapter.rs b/datafusion/core/tests/parquet/expr_adapter.rs index fd70d74a9140c..535828fa29c2f 100644 --- a/datafusion/core/tests/parquet/expr_adapter.rs +++ b/datafusion/core/tests/parquet/expr_adapter.rs @@ -18,8 +18,8 @@ use std::sync::Arc; use arrow::array::{ - Array, ArrayRef, BooleanArray, Int32Array, Int64Array, LargeListArray, ListArray, - RecordBatch, StringArray, StructArray, record_batch, + Array, ArrayRef, BooleanArray, FixedSizeListArray, Int32Array, Int64Array, + LargeListArray, ListArray, RecordBatch, StringArray, StructArray, record_batch, }; use arrow::buffer::OffsetBuffer; use arrow::compute::concat_batches; @@ -60,13 +60,19 @@ async fn write_parquet(batch: RecordBatch, store: Arc, path: &s enum NestedListKind { List, LargeList, + FixedSizeList, } +const FIXED_SIZE_LIST_LEN: usize = 2; + impl NestedListKind { fn field_data_type(self, item_field: Arc) -> DataType { match self { Self::List => DataType::List(item_field), Self::LargeList => DataType::LargeList(item_field), + Self::FixedSizeList => { + DataType::FixedSizeList(item_field, FIXED_SIZE_LIST_LEN as i32) + } } } @@ -89,6 +95,19 @@ impl NestedListKind { values, None, )), + Self::FixedSizeList => { + assert_eq!( + lengths.as_slice(), + &[FIXED_SIZE_LIST_LEN], + "FixedSizeList fixtures must contain exactly {FIXED_SIZE_LIST_LEN} elements per row" + ); + Arc::new(FixedSizeListArray::new( + item_field, + FIXED_SIZE_LIST_LEN as i32, + values, + None, + )) + } } } @@ -96,6 +115,7 @@ impl NestedListKind { match self { Self::List => "list", Self::LargeList => "large_list", + Self::FixedSizeList => "fixed_size_list", } } } @@ -277,7 +297,8 @@ fn nested_list_table_schema( } // Helper to extract message values from a nested list column. -// Returns the values at indices 0 and 1 from either a ListArray or LargeListArray. +// Returns the values at indices 0 and 1 from either a ListArray, LargeListArray, +// or FixedSizeListArray. fn extract_nested_list_values( kind: NestedListKind, column: &ArrayRef, @@ -297,7 +318,50 @@ fn extract_nested_list_values( .expect("messages should be a LargeListArray"); (list.value(0), list.value(1)) } + NestedListKind::FixedSizeList => { + let list = column + .as_any() + .downcast_ref::() + .expect("messages should be a FixedSizeListArray"); + (list.value(0), list.value(1)) + } + } +} + +fn evolved_messages(kind: NestedListKind) -> Vec> { + let mut messages = vec![NestedMessageRow { + id: 30, + name: "gamma", + chain: Some("eth"), + ignored: Some(99), + }]; + if matches!(kind, NestedListKind::FixedSizeList) { + messages.push(NestedMessageRow { + id: 40, + name: "delta", + chain: Some("doge"), + ignored: Some(100), + }); + } + messages +} + +fn error_messages(kind: NestedListKind) -> Vec> { + let mut messages = vec![NestedMessageRow { + id: 10, + name: "alpha", + chain: Some("eth"), + ignored: None, + }]; + if matches!(kind, NestedListKind::FixedSizeList) { + messages.push(NestedMessageRow { + id: 20, + name: "beta", + chain: Some("doge"), + ignored: None, + }); } + messages } // Helper to set up a nested list test fixture. @@ -352,15 +416,11 @@ async fn assert_nested_list_struct_schema_evolution(kind: NestedListKind) -> Res ); // new.parquet shape: messages item struct adds nullable `chain` and extra `ignored`. + let new_messages = evolved_messages(kind); let new_batch = nested_messages_batch( kind, 2, - &[NestedMessageRow { - id: 30, - name: "gamma", - chain: Some("eth"), - ignored: Some(99), - }], + &new_messages, &message_fields(DataType::Utf8, true, true, true), ); @@ -429,7 +489,12 @@ async fn assert_nested_list_struct_schema_evolution(kind: NestedListKind) -> Res .as_any() .downcast_ref::() .unwrap(); - assert_eq!(new_chain.iter().collect::>(), vec![Some("eth")]); + let expected_new_chain = if matches!(kind, NestedListKind::FixedSizeList) { + vec![Some("eth"), Some("doge")] + } else { + vec![Some("eth")] + }; + assert_eq!(new_chain.iter().collect::>(), expected_new_chain); let projected = ctx .sql( @@ -863,12 +928,12 @@ async fn test_struct_schema_evolution_projection_and_filter() -> Result<()> { Ok(()) } -/// Macro to generate paired test functions for List and LargeList variants. -/// Expands to two `#[tokio::test]` functions with the specified names. -macro_rules! test_struct_schema_evolution_pair { +/// Macro to generate schema evolution tests for list-like variants. +macro_rules! test_struct_schema_evolution_variants { ( list: $list_test:ident, large_list: $large_list_test:ident, + fixed_size_list: $fixed_size_list_test:ident, fn: $assertion_fn:path $(, args: $($arg:expr),+)? ) => { #[tokio::test] @@ -880,10 +945,16 @@ macro_rules! test_struct_schema_evolution_pair { async fn $large_list_test() { $assertion_fn(NestedListKind::LargeList $(, $($arg),+)?).await; } + + #[tokio::test] + async fn $fixed_size_list_test() { + $assertion_fn(NestedListKind::FixedSizeList $(, $($arg),+)?).await; + } }; ( list: $list_test:ident, large_list: $large_list_test:ident, + fixed_size_list: $fixed_size_list_test:ident, fn_result: $assertion_fn:path ) => { #[tokio::test] @@ -895,31 +966,34 @@ macro_rules! test_struct_schema_evolution_pair { async fn $large_list_test() -> Result<()> { $assertion_fn(NestedListKind::LargeList).await } + + #[tokio::test] + async fn $fixed_size_list_test() -> Result<()> { + $assertion_fn(NestedListKind::FixedSizeList).await + } }; } -test_struct_schema_evolution_pair!( +test_struct_schema_evolution_variants!( list: test_list_struct_schema_evolution_end_to_end, large_list: test_large_list_struct_schema_evolution_end_to_end, + fixed_size_list: test_fixed_size_list_struct_schema_evolution_end_to_end, fn_result: assert_nested_list_struct_schema_evolution ); async fn assert_nested_list_struct_schema_evolution_errors( kind: NestedListKind, + source_includes_chain: bool, chain_type: DataType, chain_nullable: bool, expected_error: &str, ) { + let messages = error_messages(kind); let batch = nested_messages_batch( kind, 1, - &[NestedMessageRow { - id: 10, - name: "alpha", - chain: Some("eth"), - ignored: None, - }], - &message_fields(DataType::Utf8, true, true, false), + &messages, + &message_fields(DataType::Utf8, true, source_includes_chain, false), ); let table_schema = @@ -949,6 +1023,7 @@ async fn assert_nested_list_struct_schema_evolution_errors( async fn assert_non_nullable_missing_chain_field_fails(kind: NestedListKind) { assert_nested_list_struct_schema_evolution_errors( kind, + false, DataType::Utf8, false, "non-nullable", @@ -959,6 +1034,7 @@ async fn assert_non_nullable_missing_chain_field_fails(kind: NestedListKind) { async fn assert_incompatible_chain_field_fails(kind: NestedListKind) { assert_nested_list_struct_schema_evolution_errors( kind, + true, incompatible_chain_type(), true, "Cannot cast struct field 'chain'", @@ -970,15 +1046,17 @@ fn incompatible_chain_type() -> DataType { DataType::Struct(vec![Arc::new(Field::new("value", DataType::Utf8, true))].into()) } -test_struct_schema_evolution_pair!( +test_struct_schema_evolution_variants!( list: test_list_struct_schema_evolution_non_nullable_missing_field_fails, large_list: test_large_list_struct_schema_evolution_non_nullable_missing_field_fails, + fixed_size_list: test_fixed_size_list_struct_schema_evolution_non_nullable_missing_field_fails, fn: assert_non_nullable_missing_chain_field_fails ); -test_struct_schema_evolution_pair!( +test_struct_schema_evolution_variants!( list: test_list_struct_schema_evolution_incompatible_field_fails, large_list: test_large_list_struct_schema_evolution_incompatible_field_fails, + fixed_size_list: test_fixed_size_list_struct_schema_evolution_incompatible_field_fails, fn: assert_incompatible_chain_field_fails ); From 6600725e1dc794271d758f04dff5b2f4feb5403c Mon Sep 17 00:00:00 2001 From: Siew Kam Onn Date: Fri, 24 Jul 2026 16:19:35 +0800 Subject: [PATCH 13/14] feat: replace panicking FixedSizeListArray::new with fallible try_new - Added constructor-error regression test. - Extended hidden-child regression to utilize sliced FixedSizeList input. --- datafusion/common/src/nested_struct.rs | 44 +++++++++++++++++++------- 1 file changed, 33 insertions(+), 11 deletions(-) diff --git a/datafusion/common/src/nested_struct.rs b/datafusion/common/src/nested_struct.rs index 74c6ce2f771a8..8dd75779addea 100644 --- a/datafusion/common/src/nested_struct.rs +++ b/datafusion/common/src/nested_struct.rs @@ -289,12 +289,12 @@ fn cast_fixed_size_list_column( }, }; - Ok(Arc::new(FixedSizeListArray::new( + Ok(Arc::new(FixedSizeListArray::try_new( Arc::clone(target_inner_field), target_list_size, cast_values, source_list.nulls().cloned(), - ))) + )?)) } fn cast_fixed_size_list_values_with_parent_nulls( @@ -1595,22 +1595,44 @@ mod tests { } #[test] - fn test_cast_fixed_size_list_struct_ignores_hidden_child_values_for_null_parent() { + fn test_cast_fixed_size_list_returns_error_for_non_nullable_child() { + let source_field = Arc::new(Field::new("item", DataType::Int32, true)); + let target_field = Arc::new(Field::new("item", DataType::Int32, false)); + let source_col: ArrayRef = Arc::new(FixedSizeListArray::new( + source_field, + 2, + Arc::new(Int32Array::from(vec![None, Some(1)])), + None, + )); + let target_type = DataType::FixedSizeList(target_field, 2); + + let error = cast_column(&source_col, &target_type, &DEFAULT_CAST_OPTIONS) + .unwrap_err() + .to_string(); + assert_contains!(error, "Found unmasked nulls for non-nullable"); + } + + #[test] + fn test_cast_sliced_fixed_size_list_struct_ignores_hidden_child_values() { let source_field = arc_field("item", struct_type(vec![field("a", DataType::Utf8)])); let target_field = arc_field("item", struct_type(vec![field("a", DataType::Int32)])); let struct_arr = StructArray::from(vec![( arc_field("a", DataType::Utf8), - Arc::new(StringArray::from(vec!["not_int", "also_bad", "1", "2"])) - as ArrayRef, + Arc::new(StringArray::from(vec![ + "0", "0", "not_int", "also_bad", "1", "2", + ])) as ArrayRef, )]); - let source_col: ArrayRef = Arc::new(FixedSizeListArray::new( - source_field, - 2, - Arc::new(struct_arr), - Some(NullBuffer::from(vec![false, true])), - )); + let source_col: ArrayRef = Arc::new( + FixedSizeListArray::new( + source_field, + 2, + Arc::new(struct_arr), + Some(NullBuffer::from(vec![true, false, true])), + ) + .slice(1, 2), + ); let target_type = DataType::FixedSizeList(target_field, 2); let result = From 8b0c47829b8bb982554bdc3cbaa1cea75bc2bde7 Mon Sep 17 00:00:00 2001 From: Siew Kam Onn Date: Fri, 24 Jul 2026 16:36:29 +0800 Subject: [PATCH 14/14] docs: add mention of equal-width FixedSizeList test: enhance size-mismatch test to verify compatibility validation rejection, runtime cast_column rejection, and width-specific Arrow diagnostic --- datafusion/common/src/nested_struct.rs | 40 +++++++++++++++++++------- 1 file changed, 29 insertions(+), 11 deletions(-) diff --git a/datafusion/common/src/nested_struct.rs b/datafusion/common/src/nested_struct.rs index 8dd75779addea..e915b91b911cc 100644 --- a/datafusion/common/src/nested_struct.rs +++ b/datafusion/common/src/nested_struct.rs @@ -542,8 +542,8 @@ pub fn validate_data_type_compatibility( /// name-based nested struct casting logic, rather than Arrow's standard cast. /// /// This is the case when both types are struct types, or both are the same -/// container type (List, LargeList, ListView, LargeListView, Dictionary) wrapping -/// types that recursively contain structs. +/// container type (List, LargeList, equal-width FixedSizeList, ListView, +/// LargeListView, Dictionary) wrapping types that recursively contain structs. /// /// Use this predicate at both planning time (to decide whether to apply struct /// compatibility validation) and execution time (to decide whether to route @@ -1502,18 +1502,36 @@ mod tests { } #[test] - fn test_validate_fixed_size_list_struct_size_mismatch_rejected() { - let (source_field, target_field) = create_fixed_size_list_test_fields( - vec![("a", DataType::Int32)], - vec![("a", DataType::Int64), ("b", DataType::Utf8)], - ); - let source = DataType::FixedSizeList(source_field, 2); - let target = DataType::FixedSizeList(target_field, 3); + fn test_fixed_size_list_struct_size_mismatch_rejected() { + let source_field = fixed_size_list_struct_field(vec![("a", DataType::Int32)]); + let target_field = Arc::clone(&source_field); + let source_type = DataType::FixedSizeList(Arc::clone(&source_field), 2); + let target_type = DataType::FixedSizeList(target_field, 3); - let error = validate_data_type_compatibility("col", &source, &target) + let validation_error = + validate_data_type_compatibility("col", &source_type, &target_type) + .unwrap_err() + .to_string(); + assert_contains!(validation_error, "Cannot cast struct field 'col'"); + + let struct_arr = StructArray::from(vec![( + arc_field("a", DataType::Int32), + Arc::new(Int32Array::from(vec![1, 2])) as ArrayRef, + )]); + let source_col: ArrayRef = Arc::new(FixedSizeListArray::new( + source_field, + 2, + Arc::new(struct_arr), + None, + )); + + let runtime_error = cast_column(&source_col, &target_type, &DEFAULT_CAST_OPTIONS) .unwrap_err() .to_string(); - assert_contains!(error, "Cannot cast struct field 'col'"); + assert_contains!( + runtime_error, + "cannot cast fixed-size-list to fixed-size-list with different size" + ); } #[test]