diff --git a/src/iceberg/schema_field.cc b/src/iceberg/schema_field.cc index 6c8d10d97..80ea61bce 100644 --- a/src/iceberg/schema_field.cc +++ b/src/iceberg/schema_field.cc @@ -22,9 +22,11 @@ #include #include #include +#include #include "iceberg/expression/literal.h" #include "iceberg/type.h" +#include "iceberg/util/checked_cast.h" #include "iceberg/util/formatter.h" // IWYU pragma: keep #include "iceberg/util/macros.h" @@ -32,8 +34,7 @@ namespace iceberg { namespace { -// A null default value is modeled as the absence of a default (matching Java), so it is -// not stored. +// Treat null defaults as absent. std::shared_ptr DropNullDefault(std::shared_ptr value) { if (value != nullptr && value->IsNull()) { return nullptr; @@ -65,6 +66,55 @@ SchemaField SchemaField::MakeRequired(int32_t field_id, std::string_view name, return {field_id, name, std::move(type), false, doc}; } +SchemaField SchemaField::WithName(std::string_view name) const { + return {field_id_, name, type_, optional_, doc_, initial_default_, write_default_}; +} + +SchemaField SchemaField::WithType(std::shared_ptr type) const { + return {field_id_, name_, std::move(type), optional_, doc_, + initial_default_, write_default_}; +} + +SchemaField SchemaField::WithDoc(std::string_view doc) const { + return {field_id_, name_, type_, optional_, doc, initial_default_, write_default_}; +} + +SchemaField SchemaField::WithInitialDefault( + std::shared_ptr initial_default) const { + return {field_id_, name_, type_, optional_, doc_, std::move(initial_default), + write_default_}; +} + +SchemaField SchemaField::WithWriteDefault( + std::shared_ptr write_default) const { + return {field_id_, + name_, + type_, + optional_, + doc_, + initial_default_, + std::move(write_default)}; +} + +Result SchemaField::CastDefaultValue( + const Literal& value, const std::shared_ptr& target_type) { + if (target_type->type_id() == TypeId::kDecimal && + std::holds_alternative(value.value())) { + const auto& source_type = internal::checked_cast(*value.type()); + const auto& decimal_type = internal::checked_cast(*target_type); + if (source_type.scale() == decimal_type.scale()) { + const auto& decimal_value = std::get(value.value()); + if (!decimal_value.FitsInPrecision(decimal_type.precision())) { + return InvalidArgument("Cannot cast default value to {}: {}", *target_type, + value); + } + return Literal::Decimal(decimal_value.value(), decimal_type.precision(), + decimal_type.scale()); + } + } + return value.CastTo(target_type); +} + int32_t SchemaField::field_id() const { return field_id_; } std::string_view SchemaField::name() const { return name_; } @@ -87,8 +137,7 @@ namespace { Status ValidateDefault(const SchemaField& field, const Literal& value, std::string_view kind) { - // A null default is modeled as absence and dropped at construction, so it never reaches - // here; only the out-of-range cast sentinels need rejecting. + // Null defaults are dropped at construction. if (value.IsAboveMax() || value.IsBelowMin()) { return InvalidSchema("Invalid {} value for {}: value is out of range", kind, field.name()); @@ -97,8 +146,7 @@ Status ValidateDefault(const SchemaField& field, const Literal& value, return InvalidSchema("Invalid {} value for {}: field has no type", kind, field.name()); } - // The spec requires unknown/variant/geometry/geography columns to default to null, so a - // non-null default on them is invalid (a null default was already dropped as absence). + // These types can only use null defaults. switch (field.type()->type_id()) { case TypeId::kUnknown: case TypeId::kVariant: @@ -109,17 +157,13 @@ Status ValidateDefault(const SchemaField& field, const Literal& value, default: break; } - // Defaults are otherwise only supported on primitive fields. The spec also permits JSON - // single-value defaults for struct/list/map (e.g. an empty struct `{}` whose sub-field - // defaults live in field metadata); that matches the current Java model's gap and is - // left as a follow-up. + // Nested defaults are not supported yet. if (!field.type()->is_primitive()) { return InvalidSchema( "Invalid {} value for {}: default values are only supported for primitive types", kind, field.name()); } - // Defaults are stored verbatim (no implicit cast), so a default whose literal type does - // not match the field type is invalid. + // Stored defaults must already match the field type. if (*value.type() != *field.type()) { return InvalidSchema("{} of field {} has type {} but expected {}", kind, field.name(), *value.type(), *field.type()); diff --git a/src/iceberg/schema_field.h b/src/iceberg/schema_field.h index 8066a6406..86bfb92b6 100644 --- a/src/iceberg/schema_field.h +++ b/src/iceberg/schema_field.h @@ -46,10 +46,8 @@ class ICEBERG_EXPORT SchemaField : public iceberg::util::Formattable { /// \param[in] type The field type. /// \param[in] optional Whether values of this field are required or nullable. /// \param[in] doc Optional documentation string for the field. - /// \param[in] initial_default The v3 `initial-default` value, or null if absent. The - /// field shares ownership of the (immutable) value. - /// \param[in] write_default The v3 `write-default` value, or null if absent. The field - /// shares ownership of the (immutable) value. + /// \param[in] initial_default The v3 `initial-default`, or null if absent. + /// \param[in] write_default The v3 `write-default`, or null if absent. SchemaField(int32_t field_id, std::string_view name, std::shared_ptr type, bool optional, std::string_view doc = {}, std::shared_ptr initial_default = nullptr, @@ -62,14 +60,33 @@ class ICEBERG_EXPORT SchemaField : public iceberg::util::Formattable { static SchemaField MakeRequired(int32_t field_id, std::string_view name, std::shared_ptr type, std::string_view doc = {}); + /// \brief Return a copy with a new name. + SchemaField WithName(std::string_view name) const; + + /// \brief Return a copy with a new type. + SchemaField WithType(std::shared_ptr type) const; + + /// \brief Return a copy with new documentation. + SchemaField WithDoc(std::string_view doc) const; + + /// \brief Return a copy with a new `initial-default`. + SchemaField WithInitialDefault(std::shared_ptr initial_default) const; + + /// \brief Return a copy with a new `write-default`. + SchemaField WithWriteDefault(std::shared_ptr write_default) const; + + /// \brief Cast a default literal to a field type. + static Result CastDefaultValue( + const Literal& value, const std::shared_ptr& target_type); + /// \brief Get the field ID. - [[nodiscard]] int32_t field_id() const; + int32_t field_id() const; /// \brief Get the field name. - [[nodiscard]] std::string_view name() const; + std::string_view name() const; /// \brief Get the field type. - [[nodiscard]] const std::shared_ptr& type() const; + const std::shared_ptr& type() const; /// \brief Get whether the field is optional. [[nodiscard]] bool optional() const; @@ -77,15 +94,13 @@ class ICEBERG_EXPORT SchemaField : public iceberg::util::Formattable { /// \brief Get the field documentation. std::string_view doc() const; - /// \brief Get the owning pointer to the default value for this field used when reading - /// rows written before the field existed (v3 `initial-default`), or null if absent. + /// \brief Get the v3 `initial-default`, or null if absent. const std::shared_ptr& initial_default() const; - /// \brief Get the owning pointer to the default value for this field used when a writer - /// does not supply a value (v3 `write-default`), or null if absent. + /// \brief Get the v3 `write-default`, or null if absent. const std::shared_ptr& write_default() const; - [[nodiscard]] std::string ToString() const override; + std::string ToString() const override; Status Validate() const; @@ -107,14 +122,13 @@ class ICEBERG_EXPORT SchemaField : public iceberg::util::Formattable { private: /// \brief Compare two fields for equality. - [[nodiscard]] bool Equals(const SchemaField& other) const; + bool Equals(const SchemaField& other) const; int32_t field_id_; std::string name_; std::shared_ptr type_; bool optional_; std::string doc_; - // Immutable default values, shared (not deep-copied) across field copies, like `type_`. std::shared_ptr initial_default_; std::shared_ptr write_default_; }; diff --git a/src/iceberg/test/meson.build b/src/iceberg/test/meson.build index ba0f6230d..12460237d 100644 --- a/src/iceberg/test/meson.build +++ b/src/iceberg/test/meson.build @@ -59,6 +59,7 @@ iceberg_tests = { 'table_requirements_test.cc', 'table_test.cc', 'table_update_test.cc', + 'update_schema_test.cc', ), }, 'logging_test': { diff --git a/src/iceberg/test/schema_field_test.cc b/src/iceberg/test/schema_field_test.cc index c2cd6a758..d757cb0b2 100644 --- a/src/iceberg/test/schema_field_test.cc +++ b/src/iceberg/test/schema_field_test.cc @@ -24,6 +24,7 @@ #include +#include "iceberg/expression/literal.h" #include "iceberg/type.h" #include "iceberg/util/formatter.h" // IWYU pragma: keep @@ -107,4 +108,66 @@ TEST(SchemaFieldTest, WithDoc) { } } +TEST(SchemaFieldTest, WithFieldMetadata) { + auto initial_default = std::make_shared(Literal::Int(1)); + auto write_default = std::make_shared(Literal::Int(2)); + SchemaField field(1, "foo", int32(), false, "doc", initial_default, write_default); + + auto renamed = field.WithName("bar"); + EXPECT_EQ(renamed.name(), "bar"); + EXPECT_EQ(renamed.field_id(), field.field_id()); + EXPECT_EQ(renamed.type(), field.type()); + EXPECT_EQ(renamed.doc(), field.doc()); + EXPECT_EQ(renamed.initial_default(), field.initial_default()); + EXPECT_EQ(renamed.write_default(), field.write_default()); + + auto retyped = field.WithType(int64()); + EXPECT_EQ(retyped.type(), int64()); + EXPECT_EQ(retyped.name(), field.name()); + EXPECT_EQ(retyped.initial_default(), field.initial_default()); + EXPECT_EQ(retyped.write_default(), field.write_default()); + + auto documented = field.WithDoc("new doc"); + EXPECT_EQ(documented.doc(), "new doc"); + EXPECT_EQ(documented.name(), field.name()); + EXPECT_EQ(documented.initial_default(), field.initial_default()); + EXPECT_EQ(documented.write_default(), field.write_default()); + + auto new_initial = std::make_shared(Literal::Int(3)); + auto with_initial = field.WithInitialDefault(new_initial); + EXPECT_EQ(with_initial.initial_default(), new_initial); + EXPECT_EQ(with_initial.write_default(), field.write_default()); + + auto new_write = std::make_shared(Literal::Int(4)); + auto with_write = field.WithWriteDefault(new_write); + EXPECT_EQ(with_write.initial_default(), field.initial_default()); + EXPECT_EQ(with_write.write_default(), new_write); + EXPECT_EQ(field.name(), "foo"); + EXPECT_EQ(*field.write_default(), Literal::Int(2)); +} + +TEST(SchemaFieldTest, CastDefaultValue) { + { + auto result = SchemaField::CastDefaultValue(Literal::Int(5), int64()); + ASSERT_TRUE(result.has_value()); + EXPECT_EQ(result.value(), Literal::Long(5)); + } + { + auto result = + SchemaField::CastDefaultValue(Literal::Decimal(1234, 9, 2), decimal(18, 2)); + ASSERT_TRUE(result.has_value()); + EXPECT_EQ(result.value(), Literal::Decimal(1234, 18, 2)); + } + { + auto result = + SchemaField::CastDefaultValue(Literal::Decimal(1234, 9, 3), decimal(18, 2)); + EXPECT_FALSE(result.has_value()); + } + { + auto result = + SchemaField::CastDefaultValue(Literal::Decimal(1234567, 9, 2), decimal(4, 2)); + EXPECT_FALSE(result.has_value()); + } +} + } // namespace iceberg diff --git a/src/iceberg/test/update_schema_test.cc b/src/iceberg/test/update_schema_test.cc index 07872e69a..db6b1d02e 100644 --- a/src/iceberg/test/update_schema_test.cc +++ b/src/iceberg/test/update_schema_test.cc @@ -19,23 +19,88 @@ #include "iceberg/update/update_schema.h" +#include +#include #include +#include +#include +#include #include +#include #include +#include "iceberg/catalog/memory/in_memory_catalog.h" +#include "iceberg/expression/literal.h" #include "iceberg/schema.h" #include "iceberg/schema_field.h" +#include "iceberg/table.h" +#include "iceberg/table_identifier.h" +#include "iceberg/table_metadata.h" #include "iceberg/test/matchers.h" -#include "iceberg/test/update_test_base.h" +#include "iceberg/test/mock_io.h" +#include "iceberg/test/test_resource.h" #include "iceberg/type.h" #include "iceberg/util/checked_cast.h" +#include "iceberg/util/uuid.h" namespace iceberg { using internal::checked_cast; -class UpdateSchemaTest : public UpdateTestBase {}; +class UpdateSchemaTest : public ::testing::Test { + protected: + virtual std::string MetadataResource() const { return "TableMetadataV2Valid.json"; } + virtual std::string TableName() const { return "test_table"; } + + void SetUp() override { + table_ident_ = TableIdentifier{.name = TableName()}; + table_location_ = "/warehouse/" + TableName(); + + InitializeFileIO(); + RegisterTableFromResource(MetadataResource()); + } + + void InitializeFileIO() { + auto mock_file_io = std::make_shared<::testing::NiceMock>(); + auto* mock_file_io_ptr = mock_file_io.get(); + ON_CALL(*mock_file_io, ReadFile(::testing::_, ::testing::_)) + .WillByDefault([mock_file_io_ptr](const std::string& file_location, + std::optional length) { + return mock_file_io_ptr->FileIO::ReadFile(file_location, length); + }); + ON_CALL(*mock_file_io, WriteFile(::testing::_, ::testing::_)) + .WillByDefault([mock_file_io_ptr](const std::string& file_location, + std::string_view content) { + return mock_file_io_ptr->FileIO::WriteFile(file_location, content); + }); + file_io_ = std::move(mock_file_io); + catalog_ = + InMemoryCatalog::Make("test_catalog", file_io_, "/warehouse/", /*properties=*/{}); + } + + void RegisterTable(std::unique_ptr metadata) { + auto metadata_location = std::format("{}/metadata/00001-{}.metadata.json", + table_location_, Uuid::GenerateV7().ToString()); + metadata->location = table_location_; + ASSERT_THAT(TableMetadataUtil::Write(*file_io_, metadata_location, *metadata), + IsOk()); + + ICEBERG_UNWRAP_OR_FAIL(table_, + catalog_->RegisterTable(table_ident_, metadata_location)); + } + + void RegisterTableFromResource(const std::string& resource_name) { + ICEBERG_UNWRAP_OR_FAIL(auto metadata, ReadTableMetadataFromResource(resource_name)); + RegisterTable(std::move(metadata)); + } + + TableIdentifier table_ident_; + std::string table_location_; + std::shared_ptr file_io_; + std::shared_ptr catalog_; + std::shared_ptr table_; +}; TEST_F(UpdateSchemaTest, AddOptionalColumn) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); @@ -82,6 +147,347 @@ TEST_F(UpdateSchemaTest, AddRequiredColumnWithAllowIncompatible) { EXPECT_EQ(new_field.doc(), "A required string column"); } +/// Uses v3 metadata for default-value tests. +class UpdateSchemaDefaultValueTest : public UpdateSchemaTest { + protected: + void SetUp() override { + table_ident_ = TableIdentifier{.name = TableName()}; + table_location_ = "/warehouse/" + TableName(); + + InitializeFileIO(); + + ICEBERG_UNWRAP_OR_FAIL(auto metadata, + ReadTableMetadataFromResource("TableMetadataV2Valid.json")); + metadata->format_version = TableMetadata::kSupportedTableFormatVersion; + metadata->next_row_id = TableMetadata::kInitialRowId; + RegisterTable(std::move(metadata)); + } +}; + +TEST_F(UpdateSchemaTest, AddColumnWithDefaultValueRequiresV3) { + ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); + update->AddColumn("new_col", int32(), "An integer column", Literal::Int(42)); + + auto result = update->Apply(); + EXPECT_THAT(result, IsError(ErrorKind::kInvalidSchema)); + EXPECT_THAT(result, HasErrorMessage("is not supported until v3")); +} + +TEST_F(UpdateSchemaDefaultValueTest, AddColumnWithDefaultValue) { + ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); + update->AddColumn("new_col", int32(), "An integer column", Literal::Int(42)); + + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto new_field_opt, result.schema->FindFieldByName("new_col")); + ASSERT_TRUE(new_field_opt.has_value()); + + const auto& new_field = new_field_opt->get(); + ASSERT_NE(new_field.initial_default(), nullptr); + EXPECT_EQ(*new_field.initial_default(), Literal::Int(42)); + ASSERT_NE(new_field.write_default(), nullptr); + EXPECT_EQ(*new_field.write_default(), Literal::Int(42)); +} + +TEST_F(UpdateSchemaDefaultValueTest, AddRequiredColumnWithDefaultValue) { + ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); + update->AddRequiredColumn("required_col", string(), "A required string column", + Literal::String("n/a")); + + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto new_field_opt, + result.schema->FindFieldByName("required_col")); + ASSERT_TRUE(new_field_opt.has_value()); + + const auto& new_field = new_field_opt->get(); + EXPECT_FALSE(new_field.optional()); + ASSERT_NE(new_field.initial_default(), nullptr); + EXPECT_EQ(*new_field.initial_default(), Literal::String("n/a")); + ASSERT_NE(new_field.write_default(), nullptr); + EXPECT_EQ(*new_field.write_default(), Literal::String("n/a")); +} + +TEST_F(UpdateSchemaDefaultValueTest, AddColumnWithMismatchedDefaultValueFails) { + ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); + update->AddColumn("new_col", int32(), "An integer column", Literal::String("oops")); + + auto result = update->Apply(); + EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); + EXPECT_THAT(result, HasErrorMessage("Cannot cast default value")); +} + +TEST_F(UpdateSchemaDefaultValueTest, AddColumnWithNarrowingDefaultValueFails) { + ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); + update->AddColumn("new_col", int32(), "An integer column", + Literal::Long(std::numeric_limits::max())); + + auto result = update->Apply(); + EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); + EXPECT_THAT(result, HasErrorMessage("Cannot cast default value")); +} + +TEST_F(UpdateSchemaDefaultValueTest, UpdateColumnDefault) { + ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); + update->AddColumn("new_col", int32(), "An integer column", Literal::Int(42)) + .UpdateColumnDefault("new_col", Literal::Int(7)); + + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto new_field_opt, result.schema->FindFieldByName("new_col")); + ASSERT_TRUE(new_field_opt.has_value()); + + const auto& new_field = new_field_opt->get(); + ASSERT_NE(new_field.initial_default(), nullptr); + EXPECT_EQ(*new_field.initial_default(), Literal::Int(42)); + ASSERT_NE(new_field.write_default(), nullptr); + EXPECT_EQ(*new_field.write_default(), Literal::Int(7)); +} + +TEST_F(UpdateSchemaDefaultValueTest, UpdateColumnDefaultOnExistingColumn) { + ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); + update->UpdateColumnDefault("x", Literal::Long(0)); + + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto field_opt, result.schema->FindFieldByName("x")); + ASSERT_TRUE(field_opt.has_value()); + + const auto& field = field_opt->get(); + EXPECT_EQ(field.initial_default(), nullptr); + ASSERT_NE(field.write_default(), nullptr); + EXPECT_EQ(*field.write_default(), Literal::Long(0)); +} + +TEST_F(UpdateSchemaDefaultValueTest, UpdateColumnDefaultClearsWithNullopt) { + ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); + update->AddColumn("new_col", int32(), "An integer column", Literal::Int(42)) + .UpdateColumnDefault("new_col", std::nullopt); + + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto field_opt, result.schema->FindFieldByName("new_col")); + ASSERT_TRUE(field_opt.has_value()); + + const auto& field = field_opt->get(); + ASSERT_NE(field.initial_default(), nullptr); + EXPECT_EQ(*field.initial_default(), Literal::Int(42)); + EXPECT_EQ(field.write_default(), nullptr); +} + +TEST_F(UpdateSchemaDefaultValueTest, AddNestedColumnPreservesNestedDefaults) { + auto nested_type = std::make_shared(std::vector{ + SchemaField(/*field_id=*/100, "inner", int32(), /*optional=*/false, /*doc=*/{}, + std::make_shared(Literal::Int(5)), + std::make_shared(Literal::Int(9)))}); + + ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); + update->AddColumn("outer", nested_type, "A nested column"); + + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto outer_opt, result.schema->FindFieldByName("outer")); + ASSERT_TRUE(outer_opt.has_value()); + + const auto& outer_struct = + internal::checked_cast(*outer_opt->get().type()); + ASSERT_EQ(outer_struct.fields().size(), 1); + const SchemaField& inner = outer_struct.fields()[0]; + ASSERT_NE(inner.initial_default(), nullptr); + EXPECT_EQ(*inner.initial_default(), Literal::Int(5)); + ASSERT_NE(inner.write_default(), nullptr); + EXPECT_EQ(*inner.write_default(), Literal::Int(9)); +} + +TEST_F(UpdateSchemaDefaultValueTest, UpdateColumnDefaultCastsToColumnType) { + ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); + update->UpdateColumnDefault("x", Literal::Int(5)); + + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto field_opt, result.schema->FindFieldByName("x")); + ASSERT_TRUE(field_opt.has_value()); + + const auto& field = field_opt->get(); + ASSERT_NE(field.write_default(), nullptr); + EXPECT_EQ(*field.write_default(), Literal::Long(5)); +} + +TEST_F(UpdateSchemaDefaultValueTest, RequireColumnAddedWithDefault) { + ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); + update->AddColumn("new_col", int32(), "An integer column", Literal::Int(42)) + .RequireColumn("new_col"); + + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto new_field_opt, result.schema->FindFieldByName("new_col")); + ASSERT_TRUE(new_field_opt.has_value()); + EXPECT_FALSE(new_field_opt->get().optional()); +} + +TEST_F(UpdateSchemaDefaultValueTest, RequireNestedMapListColumnAddedWithDefault) { + // Map/list paths omit value/element. + auto map_key_struct = std::make_shared( + std::vector{SchemaField(20, "address", string(), false)}); + auto map_value_struct = std::make_shared( + std::vector{SchemaField(12, "lat", float32(), false), + SchemaField(13, "long", float32(), false)}); + auto map_type = + std::make_shared(SchemaField(10, "key", map_key_struct, false), + SchemaField(11, "value", map_value_struct, false)); + auto list_element_struct = std::make_shared(std::vector{ + SchemaField(15, "x", int64(), false), SchemaField(16, "y", int64(), false)}); + auto list_type = + std::make_shared(SchemaField(14, "element", list_element_struct, true)); + + ICEBERG_UNWRAP_OR_FAIL(auto setup_update, table_->NewUpdateSchema()); + setup_update->AddColumn("locations", map_type, "map of address to coordinate") + .AddColumn("points", list_type, "2-D cartesian points"); + EXPECT_THAT(setup_update->Commit(), IsOk()); + + ICEBERG_UNWRAP_OR_FAIL(auto reloaded, catalog_->LoadTable(table_ident_)); + ICEBERG_UNWRAP_OR_FAIL(auto update, reloaded->NewUpdateSchema()); + update->AddColumn("locations", "alt", float32(), "altitude", Literal::Float(0.0f)) + .RequireColumn("locations.alt") + .AddColumn("points", "z", int64(), "z coordinate", Literal::Long(0)) + .RequireColumn("points.z"); + + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + + ICEBERG_UNWRAP_OR_FAIL(auto locations_opt, result.schema->FindFieldByName("locations")); + ASSERT_TRUE(locations_opt.has_value()); + const auto& map = checked_cast(*locations_opt->get().type()); + const auto& value_struct = checked_cast(*map.value().type()); + ICEBERG_UNWRAP_OR_FAIL(auto alt_opt, value_struct.GetFieldByName("alt")); + ASSERT_TRUE(alt_opt.has_value()); + EXPECT_FALSE(alt_opt->get().optional()); + ASSERT_NE(alt_opt->get().initial_default(), nullptr); + + ICEBERG_UNWRAP_OR_FAIL(auto points_opt, result.schema->FindFieldByName("points")); + ASSERT_TRUE(points_opt.has_value()); + const auto& list = checked_cast(*points_opt->get().type()); + const auto& element_struct = checked_cast(*list.element().type()); + ICEBERG_UNWRAP_OR_FAIL(auto z_opt, element_struct.GetFieldByName("z")); + ASSERT_TRUE(z_opt.has_value()); + EXPECT_FALSE(z_opt->get().optional()); + ASSERT_NE(z_opt->get().initial_default(), nullptr); +} + +TEST_F(UpdateSchemaDefaultValueTest, UpdateColumnDocPreservesDefaultValues) { + ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); + update->AddColumn("new_col", int32(), "An integer column", Literal::Int(42)) + .UpdateColumnDoc("new_col", "updated doc"); + + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto field_opt, result.schema->FindFieldByName("new_col")); + ASSERT_TRUE(field_opt.has_value()); + + const auto& field = field_opt->get(); + EXPECT_EQ(field.doc(), "updated doc"); + ASSERT_NE(field.initial_default(), nullptr); + EXPECT_EQ(*field.initial_default(), Literal::Int(42)); + ASSERT_NE(field.write_default(), nullptr); + EXPECT_EQ(*field.write_default(), Literal::Int(42)); +} + +TEST_F(UpdateSchemaDefaultValueTest, UpdateColumnTypePromotesDefaultValues) { + ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); + update->AddColumn("new_col", int32(), "An integer column", Literal::Int(42)) + .UpdateColumn("new_col", int64()); + + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto field_opt, result.schema->FindFieldByName("new_col")); + ASSERT_TRUE(field_opt.has_value()); + + const auto& field = field_opt->get(); + EXPECT_EQ(field.type(), int64()); + ASSERT_NE(field.initial_default(), nullptr); + EXPECT_EQ(*field.initial_default(), Literal::Long(42)); + ASSERT_NE(field.write_default(), nullptr); + EXPECT_EQ(*field.write_default(), Literal::Long(42)); +} + +TEST_F(UpdateSchemaDefaultValueTest, UpdateColumnTypePromotesDecimalDefault) { + ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); + update + ->AddColumn("new_col", decimal(9, 2), "A decimal column", + Literal::Decimal(1234, 9, 2)) + .UpdateColumn("new_col", decimal(18, 2)); + + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto field_opt, result.schema->FindFieldByName("new_col")); + ASSERT_TRUE(field_opt.has_value()); + + const auto& field = field_opt->get(); + EXPECT_EQ(field.type()->ToString(), decimal(18, 2)->ToString()); + ASSERT_NE(field.initial_default(), nullptr); + EXPECT_EQ(*field.initial_default(), Literal::Decimal(1234, 18, 2)); + ASSERT_NE(field.write_default(), nullptr); + EXPECT_EQ(*field.write_default(), Literal::Decimal(1234, 18, 2)); +} + +TEST_F(UpdateSchemaDefaultValueTest, AddColumnWithWiderPrecisionDecimalDefault) { + ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); + update->AddColumn("new_col", decimal(18, 2), "A decimal column", + Literal::Decimal(1234, 9, 2)); + + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto field_opt, result.schema->FindFieldByName("new_col")); + ASSERT_TRUE(field_opt.has_value()); + + const auto& field = field_opt->get(); + ASSERT_NE(field.initial_default(), nullptr); + EXPECT_EQ(*field.initial_default(), Literal::Decimal(1234, 18, 2)); + ASSERT_NE(field.write_default(), nullptr); + EXPECT_EQ(*field.write_default(), Literal::Decimal(1234, 18, 2)); +} + +TEST_F(UpdateSchemaDefaultValueTest, UpdateColumnDefaultWiderPrecisionDecimal) { + ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); + update->AddColumn("new_col", decimal(18, 2), "A decimal column") + .UpdateColumnDefault("new_col", Literal::Decimal(1234, 9, 2)); + + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto field_opt, result.schema->FindFieldByName("new_col")); + ASSERT_TRUE(field_opt.has_value()); + + const auto& field = field_opt->get(); + ASSERT_NE(field.write_default(), nullptr); + EXPECT_EQ(*field.write_default(), Literal::Decimal(1234, 18, 2)); +} + +TEST_F(UpdateSchemaDefaultValueTest, AddColumnWithDifferentScaleDecimalDefaultFails) { + ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); + update->AddColumn("new_col", decimal(18, 2), "A decimal column", + Literal::Decimal(1234, 9, 3)); + + auto result = update->Apply(); + EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); + EXPECT_THAT(result, HasErrorMessage("Cannot cast default value")); +} + +TEST_F(UpdateSchemaDefaultValueTest, UpdateColumnDefaultDifferentScaleDecimalFails) { + ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); + update->AddColumn("new_col", decimal(18, 2), "A decimal column") + .UpdateColumnDefault("new_col", Literal::Decimal(1234, 9, 3)); + + auto result = update->Apply(); + EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); + EXPECT_THAT(result, HasErrorMessage("Cannot cast default value")); +} + +TEST_F(UpdateSchemaDefaultValueTest, AddColumnWithOutOfPrecisionDecimalDefaultFails) { + ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); + update->AddColumn("new_col", decimal(4, 2), "A decimal column", + Literal::Decimal(1234567, 9, 2)); + + auto result = update->Apply(); + EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); + EXPECT_THAT(result, HasErrorMessage("Cannot cast default value")); +} + +TEST_F(UpdateSchemaDefaultValueTest, AddColumnWithTypedNullDecimalDefaultFails) { + ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); + update->AddColumn("new_col", decimal(18, 2), "A decimal column", + Literal::Null(decimal(18, 2))); + + auto result = update->Apply(); + EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); + EXPECT_THAT(result, HasErrorMessage("Cannot cast default value")); +} + TEST_F(UpdateSchemaTest, AddMultipleColumns) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); update->AddColumn("col1", int32(), "First column") diff --git a/src/iceberg/type_fwd.h b/src/iceberg/type_fwd.h index 35a493b48..91e18b24c 100644 --- a/src/iceberg/type_fwd.h +++ b/src/iceberg/type_fwd.h @@ -104,6 +104,7 @@ class GeometryType; /// \brief Data values. class Decimal; +class Literal; class Uuid; /// \brief Schema. diff --git a/src/iceberg/update/update_schema.cc b/src/iceberg/update/update_schema.cc index 5c50ee41d..56e167e10 100644 --- a/src/iceberg/update/update_schema.cc +++ b/src/iceberg/update/update_schema.cc @@ -29,6 +29,7 @@ #include #include +#include "iceberg/expression/literal.h" #include "iceberg/json_serde_internal.h" #include "iceberg/name_mapping.h" #include "iceberg/schema.h" @@ -39,6 +40,7 @@ #include "iceberg/type.h" #include "iceberg/util/checked_cast.h" #include "iceberg/util/error_collector.h" +#include "iceberg/util/formatter.h" // IWYU pragma: keep #include "iceberg/util/macros.h" #include "iceberg/util/string_util.h" #include "iceberg/util/type_util.h" @@ -214,11 +216,9 @@ class ApplyChangesVisitor { if (update_it != updates_.end()) { const auto& update_field = update_it->second; - return SchemaField(field_id, update_field->name(), std::move(result_type), - update_field->optional(), update_field->doc()); + return update_field->WithType(std::move(result_type)); } else if (result_type != field.type()) { - return SchemaField(field_id, field.name(), std::move(result_type), field.optional(), - field.doc()); + return field.WithType(std::move(result_type)); } else { return field; } @@ -282,6 +282,10 @@ std::vector ApplyChangesVisitor::MoveFields( return reordered; } +bool ValidateDefaultLiteral(const Literal& value) { + return !value.IsNull() && !value.IsAboveMax() && !value.IsBelowMin(); +} + } // namespace Result> UpdateSchema::Make( @@ -345,39 +349,43 @@ UpdateSchema& UpdateSchema::CaseSensitive(bool case_sensitive) { } UpdateSchema& UpdateSchema::AddColumn(std::string_view name, std::shared_ptr type, - std::string_view doc) { + std::string_view doc, + std::optional default_value) { ICEBERG_BUILDER_CHECK(!name.contains('.'), "Cannot add column with ambiguous name: {}, use " "AddColumn(parent, name, type, doc)", name); - return AddColumnInternal(std::nullopt, name, /*is_optional=*/true, std::move(type), - doc); + return AddColumnInternal(/*parent=*/std::nullopt, name, /*is_optional=*/true, + std::move(type), doc, std::move(default_value)); } UpdateSchema& UpdateSchema::AddColumn(std::optional parent, std::string_view name, std::shared_ptr type, - std::string_view doc) { + std::string_view doc, + std::optional default_value) { return AddColumnInternal(std::move(parent), name, /*is_optional=*/true, std::move(type), - doc); + doc, std::move(default_value)); } UpdateSchema& UpdateSchema::AddRequiredColumn(std::string_view name, std::shared_ptr type, - std::string_view doc) { + std::string_view doc, + std::optional default_value) { ICEBERG_BUILDER_CHECK(!name.contains('.'), "Cannot add column with ambiguous name: {}, use " "AddRequiredColumn(parent, name, type, doc)", name); return AddColumnInternal(std::nullopt, name, /*is_optional=*/false, std::move(type), - doc); + doc, std::move(default_value)); } UpdateSchema& UpdateSchema::AddRequiredColumn(std::optional parent, std::string_view name, std::shared_ptr type, - std::string_view doc) { + std::string_view doc, + std::optional default_value) { return AddColumnInternal(std::move(parent), name, /*is_optional=*/false, - std::move(type), doc); + std::move(type), doc, std::move(default_value)); } UpdateSchema& UpdateSchema::UpdateColumn(std::string_view name, @@ -399,8 +407,25 @@ UpdateSchema& UpdateSchema::UpdateColumn(std::string_view name, "Cannot change column type: {}: {} -> {}", name, field.type()->ToString(), new_type->ToString()); - updates_[field_id] = std::make_shared( - field.field_id(), field.name(), new_type, field.optional(), field.doc()); + // Defaults must follow the promoted type. + auto promote_default = [&](const std::shared_ptr& value) + -> Result> { + if (value == nullptr) { + return nullptr; + } + ICEBERG_ASSIGN_OR_RAISE(Literal promoted, + SchemaField::CastDefaultValue(*value, new_type)); + return std::make_shared(std::move(promoted)); + }; + ICEBERG_BUILDER_ASSIGN_OR_RETURN(std::shared_ptr initial_default, + promote_default(field.initial_default())); + ICEBERG_BUILDER_ASSIGN_OR_RETURN(std::shared_ptr write_default, + promote_default(field.write_default())); + + updates_[field_id] = + std::make_shared(field.WithType(new_type) + .WithInitialDefault(std::move(initial_default)) + .WithWriteDefault(std::move(write_default))); return *this; } @@ -420,9 +445,44 @@ UpdateSchema& UpdateSchema::UpdateColumnDoc(std::string_view name, return *this; } - updates_[field_id] = - std::make_shared(field.field_id(), field.name(), field.type(), - field.optional(), std::string(new_doc)); + updates_[field_id] = std::make_shared(field.WithDoc(new_doc)); + + return *this; +} + +UpdateSchema& UpdateSchema::UpdateColumnDefault(std::string_view name, + std::optional new_default) { + ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto field_opt, FindFieldForUpdate(name)); + ICEBERG_BUILDER_CHECK(field_opt.has_value(), "Cannot update missing column: {}", name); + + const auto& field = field_opt->get(); + int32_t field_id = field.field_id(); + + ICEBERG_BUILDER_CHECK(!deletes_.contains(field_id), + "Cannot update a column that will be deleted: {}", field.name()); + + if (!new_default.has_value()) { + updates_[field_id] = std::make_shared(field.WithWriteDefault(nullptr)); + return *this; + } + + ICEBERG_BUILDER_CHECK(field.type()->is_primitive(), + "Invalid default value for {}: {} (must be null)", *field.type(), + *new_default); + ICEBERG_BUILDER_ASSIGN_OR_RETURN_WITH_ERROR( + Literal typed_default, + SchemaField::CastDefaultValue( + *new_default, internal::checked_pointer_cast(field.type())), + "Cannot cast default value to {}: {}", *field.type(), *new_default); + ICEBERG_BUILDER_CHECK(ValidateDefaultLiteral(typed_default), + "Cannot cast default value to {}: {}", *field.type(), + *new_default); + if (field.write_default() != nullptr && *field.write_default() == typed_default) { + return *this; + } + + updates_[field_id] = std::make_shared( + field.WithWriteDefault(std::make_shared(std::move(typed_default)))); return *this; } @@ -443,9 +503,7 @@ UpdateSchema& UpdateSchema::RenameColumn(std::string_view name, const SchemaField& base_field = update_it != updates_.end() ? *update_it->second : field; - updates_[field_id] = std::make_shared( - base_field.field_id(), std::string(new_name), base_field.type(), - base_field.optional(), base_field.doc()); + updates_[field_id] = std::make_shared(base_field.WithName(new_name)); auto it = std::ranges::find(identifier_field_names_, name); if (it != identifier_field_names_.end()) { @@ -474,9 +532,9 @@ UpdateSchema& UpdateSchema::UpdateColumnRequirementInternal(std::string_view nam return *this; } - // TODO(GuotaoYu): support added column with default value - // bool is_defaulted_add = IsAdded(name) && field.initial_default() != null; - bool is_defaulted_add = false; + // Defaulted adds can become required. + bool is_defaulted_add = added_name_to_id_.contains(CaseSensitivityAwareName(name)) && + field.initial_default() != nullptr; ICEBERG_BUILDER_CHECK(is_optional || is_defaulted_add || allow_incompatible_changes_, "Cannot change column nullability: {}: optional -> required", @@ -625,13 +683,15 @@ Result UpdateSchema::Apply() { .updated_props = std::move(updated_props)}; } -// TODO(Guotao Yu): v3 default value is not yet supported UpdateSchema& UpdateSchema::AddColumnInternal(std::optional parent, std::string_view name, bool is_optional, std::shared_ptr type, - std::string_view doc) { + std::string_view doc, + std::optional default_value) { int32_t parent_id = kTableRootId; std::string full_name; + // For map/list adds, this omits synthetic value/element path segments. + std::string short_name; if (parent.has_value()) { ICEBERG_BUILDER_CHECK(!parent->empty(), "Parent name cannot be empty"); @@ -661,13 +721,13 @@ UpdateSchema& UpdateSchema::AddColumnInternal(std::optional pa ICEBERG_BUILDER_CHECK(!deletes_.contains(parent_id), "Cannot add to a column that will be deleted: {}", *parent); - auto current_name = std::format("{}.{}", *parent, name); - ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto current_field, FindField(current_name)); + short_name = std::format("{}.{}", *parent, name); + ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto current_field, FindField(short_name)); ICEBERG_BUILDER_CHECK( !current_field.has_value() || deletes_.contains(current_field->get().field_id()), "Cannot add column, name already exists: {}.{}", *parent, name); - // Build full name using canonical name of parent + // Use the canonical parent name for schema updates. ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto parent_name_opt, schema_->FindColumnNameById(parent_id)); ICEBERG_BUILDER_CHECK(parent_name_opt.has_value(), @@ -680,16 +740,19 @@ UpdateSchema& UpdateSchema::AddColumnInternal(std::optional pa "Cannot add column, name already exists: {}", name); full_name = std::string(name); + short_name = full_name; } ICEBERG_BUILDER_CHECK( - is_optional || allow_incompatible_changes_, + is_optional || default_value.has_value() || allow_incompatible_changes_, "Incompatible change: cannot add required column without a default value: {}", full_name); int32_t new_id = AssignNewColumnId(); added_name_to_id_[CaseSensitivityAwareName(full_name)] = new_id; + // Needed for same-update lookups by user-facing path. + added_name_to_id_[CaseSensitivityAwareName(short_name)] = new_id; if (parent_id != kTableRootId) { id_to_parent_[new_id] = parent_id; } @@ -697,11 +760,30 @@ UpdateSchema& UpdateSchema::AddColumnInternal(std::optional pa AssignFreshIdVisitor id_assigner([this]() { return AssignNewColumnId(); }); auto type_with_fresh_ids = id_assigner.Visit(type); - auto new_field = std::make_shared(new_id, std::string(name), - std::move(type_with_fresh_ids), - is_optional, std::string(doc)); + std::shared_ptr initial_default; + std::shared_ptr write_default; + if (default_value.has_value()) { + ICEBERG_BUILDER_CHECK(type_with_fresh_ids->is_primitive(), + "Invalid default value for {}: {} (must be null)", + *type_with_fresh_ids, *default_value); + ICEBERG_BUILDER_ASSIGN_OR_RETURN_WITH_ERROR( + Literal typed_default, + SchemaField::CastDefaultValue( + *default_value, + internal::checked_pointer_cast(type_with_fresh_ids)), + "Cannot cast default value to {}: {}", *type_with_fresh_ids, *default_value); + ICEBERG_BUILDER_CHECK(ValidateDefaultLiteral(typed_default), + "Cannot cast default value to {}: {}", *type_with_fresh_ids, + *default_value); + // Add-column defaults set both default kinds. + auto shared_default = std::make_shared(std::move(typed_default)); + initial_default = shared_default; + write_default = std::move(shared_default); + } - updates_[new_id] = std::move(new_field); + updates_[new_id] = std::make_shared( + new_id, name, std::move(type_with_fresh_ids), is_optional, doc, + std::move(initial_default), std::move(write_default)); parent_to_added_ids_[parent_id].push_back(new_id); return *this; diff --git a/src/iceberg/update/update_schema.h b/src/iceberg/update/update_schema.h index 2be3732a0..d0dd944ef 100644 --- a/src/iceberg/update/update_schema.h +++ b/src/iceberg/update/update_schema.h @@ -30,6 +30,7 @@ #include #include +#include "iceberg/expression/literal.h" #include "iceberg/iceberg_export.h" #include "iceberg/result.h" #include "iceberg/type_fwd.h" @@ -41,10 +42,6 @@ namespace iceberg { /// /// When committing, these changes will be applied to the current table metadata. /// Commit conflicts will not be resolved and will result in a CommitFailed error. -/// -/// TODO(Guotao Yu): Add support for V3 default values when adding columns. Currently, all -/// added columns use null as the default value, but Iceberg V3 supports custom -/// default values for new columns. class ICEBERG_EXPORT UpdateSchema : public PendingUpdate { public: static Result> Make( @@ -78,15 +75,17 @@ class ICEBERG_EXPORT UpdateSchema : public PendingUpdate { /// If type is a nested type, its field IDs are reassigned when added to the /// existing schema. /// - /// The added column will be optional with a null default value. + /// v3+: `default_value` sets `initial-default` and `write-default`. /// /// \param name Name for the new column. /// \param type Type for the new column. /// \param doc Documentation string for the new column. + /// \param default_value Optional default value for the new column (v3+). /// \return Reference to this for method chaining. /// \note InvalidArgument will be reported if name contains ".". UpdateSchema& AddColumn(std::string_view name, std::shared_ptr type, - std::string_view doc = ""); + std::string_view doc = "", + std::optional default_value = std::nullopt); /// \brief Add a new optional column to a nested struct with documentation. /// @@ -102,22 +101,23 @@ class ICEBERG_EXPORT UpdateSchema : public PendingUpdate { /// If type is a nested type, its field IDs are reassigned when added to the /// existing schema. /// - /// The added column will be optional with a null default value. + /// v3+: `default_value` sets `initial-default` and `write-default`. /// /// \param parent Name of the parent struct to which the column will be added. /// \param name Name for the new column. /// \param type Type for the new column. /// \param doc Documentation string for the new column. + /// \param default_value Optional default value for the new column (v3+). /// \return Reference to this for method chaining. /// \note InvalidArgument will be reported if parent doesn't identify a struct. UpdateSchema& AddColumn(std::optional parent, std::string_view name, - std::shared_ptr type, std::string_view doc = ""); + std::shared_ptr type, std::string_view doc = "", + std::optional default_value = std::nullopt); /// \brief Add a new required top-level column with documentation. /// - /// Adding a required column without a default is an incompatible change that can - /// break reading older data. To suppress exceptions thrown when an incompatible - /// change is detected, call AllowIncompatibleChanges(). + /// Required adds need AllowIncompatibleChanges() unless `default_value` is set. + /// v3+: `default_value` sets `initial-default` and `write-default`. /// /// Because "." may be interpreted as a column path separator or may be used in /// field names, it is not allowed in names passed to this method. To add to nested @@ -130,16 +130,17 @@ class ICEBERG_EXPORT UpdateSchema : public PendingUpdate { /// \param name Name for the new column. /// \param type Type for the new column. /// \param doc Documentation string for the new column. + /// \param default_value Optional default value for the new column (v3+). /// \return Reference to this for method chaining. /// \note InvalidArgument will be reported if name contains ".". UpdateSchema& AddRequiredColumn(std::string_view name, std::shared_ptr type, - std::string_view doc = ""); + std::string_view doc = "", + std::optional default_value = std::nullopt); /// \brief Add a new required column to a nested struct with documentation. /// - /// Adding a required column without a default is an incompatible change that can - /// break reading older data. To suppress exceptions thrown when an incompatible - /// change is detected, call AllowIncompatibleChanges(). + /// Required adds need AllowIncompatibleChanges() unless `default_value` is set. + /// v3+: `default_value` sets `initial-default` and `write-default`. /// /// The parent name is used to find the parent using Schema::FindFieldByName(). If /// the parent name is null or empty, the new column will be added to the root as a @@ -157,11 +158,13 @@ class ICEBERG_EXPORT UpdateSchema : public PendingUpdate { /// \param name Name for the new column. /// \param type Type for the new column. /// \param doc Documentation string for the new column. + /// \param default_value Optional default value for the new column (v3+). /// \return Reference to this for method chaining. /// \note InvalidArgument will be reported if parent doesn't identify a struct. UpdateSchema& AddRequiredColumn(std::optional parent, std::string_view name, std::shared_ptr type, - std::string_view doc = ""); + std::string_view doc = "", + std::optional default_value = std::nullopt); /// \brief Rename a column in the schema. /// @@ -210,6 +213,20 @@ class ICEBERG_EXPORT UpdateSchema : public PendingUpdate { /// the column will be deleted. UpdateSchema& UpdateColumnDoc(std::string_view name, std::string_view new_doc); + /// \brief Update the `write-default` value for a column (v3+). + /// + /// The name is used to find the column to update using Schema::FindFieldByName(). + /// Only `write-default` changes; `initial-default` is fixed. + /// + /// \param name Name of the column to update the default value for. + /// \param new_default Replacement `write-default` value for the column, or + /// `std::nullopt` to clear it. + /// \return Reference to this for method chaining. + /// \note InvalidArgument will be reported if name doesn't identify a column in the + /// schema or if the column will be deleted. + UpdateSchema& UpdateColumnDefault(std::string_view name, + std::optional new_default); + /// \brief Update a column to be optional. /// /// \param name Name of the column to mark optional. @@ -364,10 +381,12 @@ class ICEBERG_EXPORT UpdateSchema : public PendingUpdate { /// \param is_optional Whether the column is optional. /// \param type Type for the new column. /// \param doc Optional documentation string. + /// \param default_value Optional default value for the new column (v3+). /// \return Reference to this for method chaining. UpdateSchema& AddColumnInternal(std::optional parent, std::string_view name, bool is_optional, - std::shared_ptr type, std::string_view doc); + std::shared_ptr type, std::string_view doc, + std::optional default_value); /// \brief Internal implementation for updating column requirement (optional/required). /// diff --git a/src/iceberg/util/type_util.cc b/src/iceberg/util/type_util.cc index 1f1e02747..6656c5636 100644 --- a/src/iceberg/util/type_util.cc +++ b/src/iceberg/util/type_util.cc @@ -385,9 +385,9 @@ std::shared_ptr AssignFreshIdVisitor::Visit(const StructType& type) std::vector fresh_fields; for (size_t i = 0; i < type.fields().size(); ++i) { const auto& field = type.fields()[i]; - fresh_fields.emplace_back(fresh_ids[i], std::string(field.name()), - Visit(field.type()), field.optional(), - std::string(field.doc())); + fresh_fields.emplace_back( + fresh_ids[i], std::string(field.name()), Visit(field.type()), field.optional(), + std::string(field.doc()), field.initial_default(), field.write_default()); } return std::make_shared(std::move(fresh_fields)); } @@ -397,7 +397,8 @@ std::shared_ptr AssignFreshIdVisitor::Visit(const ListType& type) cons int32_t fresh_id = next_id_(); SchemaField fresh_elem_field(fresh_id, std::string(elem_field.name()), Visit(elem_field.type()), elem_field.optional(), - std::string(elem_field.doc())); + std::string(elem_field.doc()), + elem_field.initial_default(), elem_field.write_default()); return std::make_shared(std::move(fresh_elem_field)); } @@ -410,10 +411,12 @@ std::shared_ptr AssignFreshIdVisitor::Visit(const MapType& type) const SchemaField fresh_key_field(fresh_key_id, std::string(key_field.name()), Visit(key_field.type()), key_field.optional(), - std::string(key_field.doc())); - SchemaField fresh_value_field(fresh_value_id, std::string(value_field.name()), - Visit(value_field.type()), value_field.optional(), - std::string(value_field.doc())); + std::string(key_field.doc()), key_field.initial_default(), + key_field.write_default()); + SchemaField fresh_value_field( + fresh_value_id, std::string(value_field.name()), Visit(value_field.type()), + value_field.optional(), std::string(value_field.doc()), + value_field.initial_default(), value_field.write_default()); return std::make_shared(std::move(fresh_key_field), std::move(fresh_value_field)); }