diff --git a/datafusion/common/src/nested_struct.rs b/datafusion/common/src/nested_struct.rs index cdd6215d08e2f..e915b91b911cc 100644 --- a/datafusion/common/src/nested_struct.rs +++ b/datafusion/common/src/nested_struct.rs @@ -18,9 +18,10 @@ use crate::error::{_plan_err, Result}; use arrow::{ array::{ - Array, ArrayRef, DictionaryArray, GenericListArray, GenericListViewArray, - StructArray, downcast_integer, new_null_array, + Array, ArrayRef, AsArray, DictionaryArray, FixedSizeListArray, GenericListArray, + GenericListViewArray, StructArray, downcast_integer, make_array, new_null_array, }, + buffer::NullBuffer, compute::{CastOptions, can_cast_types, cast_with_options}, datatypes::{DataType, DataType::Struct, Field, FieldRef}, }; @@ -58,9 +59,7 @@ fn cast_struct_column( target_fields: &[Arc], cast_options: &CastOptions, ) -> Result { - if source_col.data_type() == &DataType::Null - || (!source_col.is_empty() && source_col.null_count() == source_col.len()) - { + if source_col.data_type() == &DataType::Null { return Ok(new_null_array( &Struct(target_fields.to_vec().into()), source_col.len(), @@ -70,6 +69,14 @@ fn cast_struct_column( if let Some(source_struct) = source_col.as_any().downcast_ref::() { let source_fields = source_struct.fields(); validate_struct_compatibility(source_fields, target_fields)?; + + if !source_col.is_empty() && source_col.null_count() == source_col.len() { + return Ok(new_null_array( + &Struct(target_fields.to_vec().into()), + source_col.len(), + )); + } + let mut fields: Vec> = Vec::with_capacity(target_fields.len()); let mut arrays: Vec = Vec::with_capacity(target_fields.len()); let num_rows = source_col.len(); @@ -183,6 +190,15 @@ pub fn cast_column( (DataType::LargeList(_), DataType::LargeList(target_inner)) => { cast_list_column::(source_col, target_inner, cast_options) } + ( + DataType::FixedSizeList(_, source_list_size), + DataType::FixedSizeList(target_inner, target_list_size), + ) if source_list_size == target_list_size => cast_fixed_size_list_column( + source_col, + target_inner, + *target_list_size, + cast_options, + ), (DataType::ListView(_), DataType::ListView(target_inner)) => { cast_list_view_column::(source_col, target_inner, cast_options) } @@ -208,15 +224,7 @@ fn cast_list_column( target_inner_field: &FieldRef, cast_options: &CastOptions, ) -> Result { - let source_list = source_col - .as_any() - .downcast_ref::>() - .ok_or_else(|| { - crate::error::DataFusionError::Plan(format!( - "Expected list array but got {}", - source_col.data_type() - )) - })?; + let source_list = source_col.as_list::(); let cast_values = cast_column( source_list.values(), @@ -238,15 +246,7 @@ fn cast_list_view_column( target_inner_field: &FieldRef, cast_options: &CastOptions, ) -> Result { - let source_list = source_col - .as_any() - .downcast_ref::>() - .ok_or_else(|| { - crate::error::DataFusionError::Plan(format!( - "Expected list view array but got {}", - source_col.data_type() - )) - })?; + let source_list = source_col.as_list_view::(); let cast_values = cast_column( source_list.values(), @@ -264,6 +264,82 @@ fn cast_list_view_column( Ok(Arc::new(result)) } +fn cast_fixed_size_list_column( + source_col: &ArrayRef, + target_inner_field: &FieldRef, + target_list_size: i32, + cast_options: &CastOptions, +) -> Result { + let source_list = source_col.as_fixed_size_list(); + + let source_values = source_list.values(); + let target_type = target_inner_field.data_type(); + + let cast_values = match cast_column(source_values, target_type, cast_options) { + Ok(cast_values) => cast_values, + Err(error) => match cast_fixed_size_list_values_with_parent_nulls( + source_values, + target_type, + cast_options, + source_list.nulls(), + target_list_size, + ) { + Some(masked_cast) => masked_cast?, + None => return Err(error), + }, + }; + + Ok(Arc::new(FixedSizeListArray::try_new( + Arc::clone(target_inner_field), + target_list_size, + cast_values, + source_list.nulls().cloned(), + )?)) +} + +fn cast_fixed_size_list_values_with_parent_nulls( + source_values: &ArrayRef, + target_type: &DataType, + cast_options: &CastOptions, + parent_nulls: Option<&NullBuffer>, + list_size: i32, +) -> Option> { + let parent_nulls = parent_nulls.filter(|nulls| nulls.null_count() > 0)?; + + // FixedSizeList stores child slots for null parent lists. Those child + // values are semantically hidden, but recursive casts still inspect them. + let hidden_child_nulls = parent_nulls.expand(list_size as usize); + let masked_values = mask_array_values(source_values, &hidden_child_nulls); + Some(masked_values.and_then(|values| cast_column(&values, target_type, cast_options))) +} + +fn mask_array_values( + values: &ArrayRef, + additional_nulls: &NullBuffer, +) -> Result { + let nulls = NullBuffer::union(values.nulls(), Some(additional_nulls)); + + if let Some(struct_array) = values.as_any().downcast_ref::() { + let struct_nulls = nulls + .as_ref() + .expect("additional nulls always produce nulls"); + let arrays = struct_array + .columns() + .iter() + .map(|child| mask_array_values(child, struct_nulls)) + .collect::>>()?; + return Ok(Arc::new(StructArray::new( + struct_array.fields().clone(), + arrays, + nulls, + ))); + } + + Ok(make_array( + values.to_data().into_builder().nulls(nulls).build()?, + )) +} + fn cast_dictionary_column( source_col: &ArrayRef, source_key_type: &DataType, @@ -425,6 +501,12 @@ pub fn validate_data_type_compatibility( (Struct(source_nested), Struct(target_nested)) => { validate_struct_compatibility(source_nested, target_nested)?; } + ( + DataType::FixedSizeList(s, source_list_size), + DataType::FixedSizeList(t, target_list_size), + ) if source_list_size == target_list_size => { + validate_field_compatibility(s, t)?; + } (DataType::List(s), DataType::List(t)) | (DataType::LargeList(s), DataType::LargeList(t)) | (DataType::ListView(s), DataType::ListView(t)) @@ -460,8 +542,8 @@ pub fn validate_data_type_compatibility( /// name-based nested struct casting logic, rather than Arrow's standard cast. /// /// This is the case when both types are struct types, or both are the same -/// container type (List, LargeList, ListView, LargeListView, Dictionary) wrapping -/// types that recursively contain structs. +/// container type (List, LargeList, equal-width FixedSizeList, ListView, +/// LargeListView, Dictionary) wrapping types that recursively contain structs. /// /// Use this predicate at both planning time (to decide whether to apply struct /// compatibility validation) and execution time (to decide whether to route @@ -472,6 +554,12 @@ pub fn requires_nested_struct_cast( ) -> bool { match (source_type, target_type) { (Struct(_), Struct(_)) => true, + ( + DataType::FixedSizeList(s, source_list_size), + DataType::FixedSizeList(t, target_list_size), + ) if source_list_size == target_list_size => { + requires_nested_struct_cast(s.data_type(), t.data_type()) + } (DataType::List(s), DataType::List(t)) | (DataType::LargeList(s), DataType::LargeList(t)) | (DataType::ListView(s), DataType::ListView(t)) @@ -508,8 +596,9 @@ mod tests { use crate::{assert_contains, format::DEFAULT_CAST_OPTIONS}; use arrow::{ array::{ - BinaryArray, Int32Array, Int32Builder, Int64Array, ListArray, ListViewArray, - MapArray, MapBuilder, NullArray, StringArray, StringBuilder, + BinaryArray, FixedSizeListArray, Int32Array, Int32Builder, Int64Array, + ListArray, ListViewArray, MapArray, MapBuilder, NullArray, StringArray, + StringBuilder, }, buffer::{NullBuffer, ScalarBuffer}, datatypes::{DataType, Field, FieldRef, Int32Type}, @@ -1307,6 +1396,275 @@ mod tests { assert!(b_col.iter().all(|v| v.is_none())); } + fn fixed_size_list_struct_field(fields: Vec<(&str, DataType)>) -> FieldRef { + arc_field( + "item", + struct_type( + fields + .into_iter() + .map(|(name, data_type)| field(name, data_type)) + .collect(), + ), + ) + } + + fn create_fixed_size_list_test_fields( + source_struct_fields: Vec<(&str, DataType)>, + target_struct_fields: Vec<(&str, DataType)>, + ) -> (FieldRef, FieldRef) { + ( + fixed_size_list_struct_field(source_struct_fields), + fixed_size_list_struct_field(target_struct_fields), + ) + } + + fn fixed_size_list_struct_values( + array: &ArrayRef, + ) -> (&FixedSizeListArray, &StructArray) { + let list = array.as_any().downcast_ref::().unwrap(); + let values = list + .values() + .as_any() + .downcast_ref::() + .unwrap(); + (list, values) + } + + #[test] + fn test_cast_fixed_size_list_struct() { + let struct_arr = StructArray::from(vec![( + arc_field("a", DataType::Int32), + Arc::new(Int32Array::from(vec![1, 2, 3, 4])) as ArrayRef, + )]); + + let (source_field, target_field) = create_fixed_size_list_test_fields( + vec![("a", DataType::Int32)], + vec![("a", DataType::Int64), ("b", DataType::Utf8)], + ); + let source_col: ArrayRef = Arc::new(FixedSizeListArray::new( + source_field, + 2, + Arc::new(struct_arr), + Some(NullBuffer::from(vec![true, false])), + )); + let target_type = DataType::FixedSizeList(target_field, 2); + + let result = + cast_column(&source_col, &target_type, &DEFAULT_CAST_OPTIONS).unwrap(); + let (result_list, struct_values) = fixed_size_list_struct_values(&result); + assert_eq!(result_list.len(), 2); + assert!(result_list.is_valid(0)); + assert!(result_list.is_null(1)); + let a_col = get_column_as!(&struct_values, "a", Int64Array); + assert_eq!(a_col.values(), &[1, 2, 3, 4]); + let b_col = get_column_as!(&struct_values, "b", StringArray); + assert!(b_col.iter().all(|v| v.is_none())); + } + + #[test] + fn test_validate_fixed_size_list_struct_compatibility() { + let (source_field, target_field) = create_fixed_size_list_test_fields( + vec![("a", DataType::Int32)], + vec![("a", DataType::Int64), ("b", DataType::Utf8)], + ); + let source = DataType::FixedSizeList(source_field, 2); + let target = DataType::FixedSizeList(target_field, 2); + + assert!(requires_nested_struct_cast(&source, &target)); + assert!(validate_data_type_compatibility("col", &source, &target).is_ok()); + } + + #[test] + fn test_validate_fixed_size_list_struct_missing_non_nullable_field_rejected() { + let (source_field, _) = create_fixed_size_list_test_fields( + vec![("a", DataType::Int32)], + vec![("a", DataType::Int64), ("b", DataType::Utf8)], + ); + let source = DataType::FixedSizeList(source_field, 2); + let target = DataType::FixedSizeList( + arc_field( + "item", + struct_type(vec![ + field("a", DataType::Int32), + non_null_field("b", DataType::Utf8), + ]), + ), + 2, + ); + + let error = validate_data_type_compatibility("col", &source, &target) + .unwrap_err() + .to_string(); + assert_contains!( + error, + "target field 'b' is non-nullable but missing from source" + ); + } + + #[test] + fn test_fixed_size_list_struct_size_mismatch_rejected() { + let source_field = fixed_size_list_struct_field(vec![("a", DataType::Int32)]); + let target_field = Arc::clone(&source_field); + let source_type = DataType::FixedSizeList(Arc::clone(&source_field), 2); + let target_type = DataType::FixedSizeList(target_field, 3); + + let validation_error = + validate_data_type_compatibility("col", &source_type, &target_type) + .unwrap_err() + .to_string(); + assert_contains!(validation_error, "Cannot cast struct field 'col'"); + + let struct_arr = StructArray::from(vec![( + arc_field("a", DataType::Int32), + Arc::new(Int32Array::from(vec![1, 2])) as ArrayRef, + )]); + let source_col: ArrayRef = Arc::new(FixedSizeListArray::new( + source_field, + 2, + Arc::new(struct_arr), + None, + )); + + let runtime_error = cast_column(&source_col, &target_type, &DEFAULT_CAST_OPTIONS) + .unwrap_err() + .to_string(); + assert_contains!( + runtime_error, + "cannot cast fixed-size-list to fixed-size-list with different size" + ); + } + + #[test] + fn test_cast_fixed_size_list_struct_all_null() { + let (source_field, target_field) = create_fixed_size_list_test_fields( + vec![("a", DataType::Int32)], + vec![("a", DataType::Int64), ("b", DataType::Utf8)], + ); + let source_col: ArrayRef = + Arc::new(FixedSizeListArray::new_null(source_field, 2, 2)); + let target_type = DataType::FixedSizeList(target_field, 2); + + let result = + cast_column(&source_col, &target_type, &DEFAULT_CAST_OPTIONS).unwrap(); + let (result_list, struct_values) = fixed_size_list_struct_values(&result); + assert_eq!(result_list.null_count(), 2); + let a_col = get_column_as!(&struct_values, "a", Int64Array); + let b_col = get_column_as!(&struct_values, "b", StringArray); + assert!(a_col.iter().all(|v| v.is_none())); + assert!(b_col.iter().all(|v| v.is_none())); + } + + #[test] + fn test_fixed_size_list_struct_planner_runtime_parity_on_incompatible_type() { + let source_field = + arc_field("item", struct_type(vec![field("a", DataType::Binary)])); + let target_field = + arc_field("item", struct_type(vec![field("a", DataType::Int32)])); + let source_type = DataType::FixedSizeList(Arc::clone(&source_field), 2); + let target_type = DataType::FixedSizeList(target_field, 2); + let validation_error = + validate_data_type_compatibility("col", &source_type, &target_type) + .unwrap_err() + .to_string(); + assert_contains!(validation_error, "Cannot cast struct field 'a'"); + + let struct_arr = StructArray::from(vec![( + arc_field("a", DataType::Binary), + Arc::new(BinaryArray::from(vec![ + Some(b"x".as_ref()), + Some(b"y".as_ref()), + ])) as ArrayRef, + )]); + let source_col: ArrayRef = Arc::new(FixedSizeListArray::new( + source_field, + 2, + Arc::new(struct_arr), + None, + )); + + let runtime_error = cast_column(&source_col, &target_type, &DEFAULT_CAST_OPTIONS) + .unwrap_err() + .to_string(); + assert_contains!(runtime_error, "Cannot cast struct field 'a'"); + } + + #[test] + fn test_cast_fixed_size_list_struct_missing_non_nullable_field_runtime_rejected() { + let source_field = + arc_field("item", struct_type(vec![field("a", DataType::Int32)])); + let target_field = arc_field( + "item", + struct_type(vec![ + field("a", DataType::Int32), + non_null_field("b", DataType::Utf8), + ]), + ); + let source_col: ArrayRef = + Arc::new(FixedSizeListArray::new_null(source_field, 2, 1)); + let target_type = DataType::FixedSizeList(target_field, 2); + + let error = cast_column(&source_col, &target_type, &DEFAULT_CAST_OPTIONS) + .unwrap_err() + .to_string(); + assert_contains!( + error, + "target field 'b' is non-nullable but missing from source" + ); + } + + #[test] + fn test_cast_fixed_size_list_returns_error_for_non_nullable_child() { + let source_field = Arc::new(Field::new("item", DataType::Int32, true)); + let target_field = Arc::new(Field::new("item", DataType::Int32, false)); + let source_col: ArrayRef = Arc::new(FixedSizeListArray::new( + source_field, + 2, + Arc::new(Int32Array::from(vec![None, Some(1)])), + None, + )); + let target_type = DataType::FixedSizeList(target_field, 2); + + let error = cast_column(&source_col, &target_type, &DEFAULT_CAST_OPTIONS) + .unwrap_err() + .to_string(); + assert_contains!(error, "Found unmasked nulls for non-nullable"); + } + + #[test] + fn test_cast_sliced_fixed_size_list_struct_ignores_hidden_child_values() { + let source_field = + arc_field("item", struct_type(vec![field("a", DataType::Utf8)])); + let target_field = + arc_field("item", struct_type(vec![field("a", DataType::Int32)])); + let struct_arr = StructArray::from(vec![( + arc_field("a", DataType::Utf8), + Arc::new(StringArray::from(vec![ + "0", "0", "not_int", "also_bad", "1", "2", + ])) as ArrayRef, + )]); + let source_col: ArrayRef = Arc::new( + FixedSizeListArray::new( + source_field, + 2, + Arc::new(struct_arr), + Some(NullBuffer::from(vec![true, false, true])), + ) + .slice(1, 2), + ); + let target_type = DataType::FixedSizeList(target_field, 2); + + let result = + cast_column(&source_col, &target_type, &DEFAULT_CAST_OPTIONS).unwrap(); + let (result_list, struct_values) = fixed_size_list_struct_values(&result); + assert!(result_list.is_null(0)); + assert!(result_list.is_valid(1)); + let a_col = get_column_as!(&struct_values, "a", Int32Array); + assert!(a_col.is_null(0)); + assert!(a_col.is_null(1)); + assert_eq!(a_col.value(2), 1); + assert_eq!(a_col.value(3), 2); + } + #[test] fn test_requires_nested_struct_cast() { let s1 = struct_type(vec![field("a", DataType::Int32)]); @@ -1322,8 +1680,12 @@ mod tests { &DataType::Dictionary(Box::new(DataType::Int32), Box::new(s2.clone())), )); assert!(requires_nested_struct_cast( - &DataType::ListView(arc_field("item", s1)), - &DataType::ListView(arc_field("item", s2)), + &DataType::ListView(arc_field("item", s1.clone())), + &DataType::ListView(arc_field("item", s2.clone())), + )); + assert!(requires_nested_struct_cast( + &DataType::FixedSizeList(arc_field("item", s1), 2), + &DataType::FixedSizeList(arc_field("item", s2), 2), )); // Non-struct types should return false. @@ -1335,5 +1697,9 @@ mod tests { &DataType::List(arc_field("item", DataType::Int32)), &DataType::List(arc_field("item", DataType::Int64)), )); + assert!(!requires_nested_struct_cast( + &DataType::FixedSizeList(arc_field("item", DataType::Int32), 2), + &DataType::FixedSizeList(arc_field("item", DataType::Int64), 2), + )); } } diff --git a/datafusion/core/tests/parquet/expr_adapter.rs b/datafusion/core/tests/parquet/expr_adapter.rs index fd70d74a9140c..535828fa29c2f 100644 --- a/datafusion/core/tests/parquet/expr_adapter.rs +++ b/datafusion/core/tests/parquet/expr_adapter.rs @@ -18,8 +18,8 @@ use std::sync::Arc; use arrow::array::{ - Array, ArrayRef, BooleanArray, Int32Array, Int64Array, LargeListArray, ListArray, - RecordBatch, StringArray, StructArray, record_batch, + Array, ArrayRef, BooleanArray, FixedSizeListArray, Int32Array, Int64Array, + LargeListArray, ListArray, RecordBatch, StringArray, StructArray, record_batch, }; use arrow::buffer::OffsetBuffer; use arrow::compute::concat_batches; @@ -60,13 +60,19 @@ async fn write_parquet(batch: RecordBatch, store: Arc, path: &s enum NestedListKind { List, LargeList, + FixedSizeList, } +const FIXED_SIZE_LIST_LEN: usize = 2; + impl NestedListKind { fn field_data_type(self, item_field: Arc) -> DataType { match self { Self::List => DataType::List(item_field), Self::LargeList => DataType::LargeList(item_field), + Self::FixedSizeList => { + DataType::FixedSizeList(item_field, FIXED_SIZE_LIST_LEN as i32) + } } } @@ -89,6 +95,19 @@ impl NestedListKind { values, None, )), + Self::FixedSizeList => { + assert_eq!( + lengths.as_slice(), + &[FIXED_SIZE_LIST_LEN], + "FixedSizeList fixtures must contain exactly {FIXED_SIZE_LIST_LEN} elements per row" + ); + Arc::new(FixedSizeListArray::new( + item_field, + FIXED_SIZE_LIST_LEN as i32, + values, + None, + )) + } } } @@ -96,6 +115,7 @@ impl NestedListKind { match self { Self::List => "list", Self::LargeList => "large_list", + Self::FixedSizeList => "fixed_size_list", } } } @@ -277,7 +297,8 @@ fn nested_list_table_schema( } // Helper to extract message values from a nested list column. -// Returns the values at indices 0 and 1 from either a ListArray or LargeListArray. +// Returns the values at indices 0 and 1 from either a ListArray, LargeListArray, +// or FixedSizeListArray. fn extract_nested_list_values( kind: NestedListKind, column: &ArrayRef, @@ -297,7 +318,50 @@ fn extract_nested_list_values( .expect("messages should be a LargeListArray"); (list.value(0), list.value(1)) } + NestedListKind::FixedSizeList => { + let list = column + .as_any() + .downcast_ref::() + .expect("messages should be a FixedSizeListArray"); + (list.value(0), list.value(1)) + } + } +} + +fn evolved_messages(kind: NestedListKind) -> Vec> { + let mut messages = vec![NestedMessageRow { + id: 30, + name: "gamma", + chain: Some("eth"), + ignored: Some(99), + }]; + if matches!(kind, NestedListKind::FixedSizeList) { + messages.push(NestedMessageRow { + id: 40, + name: "delta", + chain: Some("doge"), + ignored: Some(100), + }); + } + messages +} + +fn error_messages(kind: NestedListKind) -> Vec> { + let mut messages = vec![NestedMessageRow { + id: 10, + name: "alpha", + chain: Some("eth"), + ignored: None, + }]; + if matches!(kind, NestedListKind::FixedSizeList) { + messages.push(NestedMessageRow { + id: 20, + name: "beta", + chain: Some("doge"), + ignored: None, + }); } + messages } // Helper to set up a nested list test fixture. @@ -352,15 +416,11 @@ async fn assert_nested_list_struct_schema_evolution(kind: NestedListKind) -> Res ); // new.parquet shape: messages item struct adds nullable `chain` and extra `ignored`. + let new_messages = evolved_messages(kind); let new_batch = nested_messages_batch( kind, 2, - &[NestedMessageRow { - id: 30, - name: "gamma", - chain: Some("eth"), - ignored: Some(99), - }], + &new_messages, &message_fields(DataType::Utf8, true, true, true), ); @@ -429,7 +489,12 @@ async fn assert_nested_list_struct_schema_evolution(kind: NestedListKind) -> Res .as_any() .downcast_ref::() .unwrap(); - assert_eq!(new_chain.iter().collect::>(), vec![Some("eth")]); + let expected_new_chain = if matches!(kind, NestedListKind::FixedSizeList) { + vec![Some("eth"), Some("doge")] + } else { + vec![Some("eth")] + }; + assert_eq!(new_chain.iter().collect::>(), expected_new_chain); let projected = ctx .sql( @@ -863,12 +928,12 @@ async fn test_struct_schema_evolution_projection_and_filter() -> Result<()> { Ok(()) } -/// Macro to generate paired test functions for List and LargeList variants. -/// Expands to two `#[tokio::test]` functions with the specified names. -macro_rules! test_struct_schema_evolution_pair { +/// Macro to generate schema evolution tests for list-like variants. +macro_rules! test_struct_schema_evolution_variants { ( list: $list_test:ident, large_list: $large_list_test:ident, + fixed_size_list: $fixed_size_list_test:ident, fn: $assertion_fn:path $(, args: $($arg:expr),+)? ) => { #[tokio::test] @@ -880,10 +945,16 @@ macro_rules! test_struct_schema_evolution_pair { async fn $large_list_test() { $assertion_fn(NestedListKind::LargeList $(, $($arg),+)?).await; } + + #[tokio::test] + async fn $fixed_size_list_test() { + $assertion_fn(NestedListKind::FixedSizeList $(, $($arg),+)?).await; + } }; ( list: $list_test:ident, large_list: $large_list_test:ident, + fixed_size_list: $fixed_size_list_test:ident, fn_result: $assertion_fn:path ) => { #[tokio::test] @@ -895,31 +966,34 @@ macro_rules! test_struct_schema_evolution_pair { async fn $large_list_test() -> Result<()> { $assertion_fn(NestedListKind::LargeList).await } + + #[tokio::test] + async fn $fixed_size_list_test() -> Result<()> { + $assertion_fn(NestedListKind::FixedSizeList).await + } }; } -test_struct_schema_evolution_pair!( +test_struct_schema_evolution_variants!( list: test_list_struct_schema_evolution_end_to_end, large_list: test_large_list_struct_schema_evolution_end_to_end, + fixed_size_list: test_fixed_size_list_struct_schema_evolution_end_to_end, fn_result: assert_nested_list_struct_schema_evolution ); async fn assert_nested_list_struct_schema_evolution_errors( kind: NestedListKind, + source_includes_chain: bool, chain_type: DataType, chain_nullable: bool, expected_error: &str, ) { + let messages = error_messages(kind); let batch = nested_messages_batch( kind, 1, - &[NestedMessageRow { - id: 10, - name: "alpha", - chain: Some("eth"), - ignored: None, - }], - &message_fields(DataType::Utf8, true, true, false), + &messages, + &message_fields(DataType::Utf8, true, source_includes_chain, false), ); let table_schema = @@ -949,6 +1023,7 @@ async fn assert_nested_list_struct_schema_evolution_errors( async fn assert_non_nullable_missing_chain_field_fails(kind: NestedListKind) { assert_nested_list_struct_schema_evolution_errors( kind, + false, DataType::Utf8, false, "non-nullable", @@ -959,6 +1034,7 @@ async fn assert_non_nullable_missing_chain_field_fails(kind: NestedListKind) { async fn assert_incompatible_chain_field_fails(kind: NestedListKind) { assert_nested_list_struct_schema_evolution_errors( kind, + true, incompatible_chain_type(), true, "Cannot cast struct field 'chain'", @@ -970,15 +1046,17 @@ fn incompatible_chain_type() -> DataType { DataType::Struct(vec![Arc::new(Field::new("value", DataType::Utf8, true))].into()) } -test_struct_schema_evolution_pair!( +test_struct_schema_evolution_variants!( list: test_list_struct_schema_evolution_non_nullable_missing_field_fails, large_list: test_large_list_struct_schema_evolution_non_nullable_missing_field_fails, + fixed_size_list: test_fixed_size_list_struct_schema_evolution_non_nullable_missing_field_fails, fn: assert_non_nullable_missing_chain_field_fails ); -test_struct_schema_evolution_pair!( +test_struct_schema_evolution_variants!( list: test_list_struct_schema_evolution_incompatible_field_fails, large_list: test_large_list_struct_schema_evolution_incompatible_field_fails, + fixed_size_list: test_fixed_size_list_struct_schema_evolution_incompatible_field_fails, fn: assert_incompatible_chain_field_fails );