This is an automated email from the ASF dual-hosted git repository.
SteNicholas pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/paimon-cpp.git
The following commit(s) were added to refs/heads/main by this push:
new 81608c7 fix: support nullable map key schemas and reject null key
data (#176)
81608c7 is described below
commit 81608c7579e908a3b694beeab6dfe95234d39fa0
Author: lxy <[email protected]>
AuthorDate: Tue Aug 4 16:49:12 2026 +0800
fix: support nullable map key schemas and reject null key data (#176)
* fix: support nullable map key schemas and reject null key data
* add validate for avro
---
docs/source/user_guide/data_types.rst | 7 ++--
docs/source/user_guide/schema.rst | 32 ++-------------
src/paimon/common/types/data_type_json_parser.cpp | 14 +++----
.../common/types/data_type_json_parser_test.cpp | 4 +-
src/paimon/core/schema/arrow_schema_validator.cpp | 4 --
src/paimon/core/schema/schema_validation_test.cpp | 10 +++--
src/paimon/core/schema/table_schema_test.cpp | 17 +++++---
src/paimon/format/avro/avro_file_batch_reader.cpp | 1 +
src/paimon/format/orc/orc_adapter.cpp | 1 +
.../format/parquet/parquet_file_batch_reader.cpp | 1 +
test/inte/scan_and_read_inte_test.cpp | 44 +++++++++++++++++++++
.../append_multiple/schema/schema-0 | 2 +-
.../append_simple.db/append_simple/schema/schema-0 | 2 +-
.../append_with_multiple_map/schema/schema-0 | 18 ++++-----
.../pk_with_multiple_type/schema/schema-0 | 2 +-
.../schema/schema-0 | 2 +-
.../append_table_with_nested_type/schema/schema-0 | 2 +-
.../nullable_map_key.db/nullable_map_key/README | 13 ++++++
...data-7e6899e9-f3cd-4d2e-86bc-7a5f383d5922-0.orc | Bin 0 -> 562 bytes
...manifest-a956b9da-04c5-41b6-b6f0-20efa3974bd4-0 | Bin 0 -> 2089 bytes
...est-list-67c61c9b-ab3c-4564-a5f2-612aa684a8f2-0 | Bin 0 -> 1006 bytes
...est-list-67c61c9b-ab3c-4564-a5f2-612aa684a8f2-1 | Bin 0 -> 1110 bytes
.../nullable_map_key/schema/schema-0 | 25 ++++++++++++
.../nullable_map_key/snapshot/EARLIEST | 1 +
.../nullable_map_key/snapshot/LATEST | 1 +
.../nullable_map_key/snapshot/snapshot-1 | 17 ++++++++
.../pk_table_nested_type/schema/schema-0 | 2 +-
.../schema/schema-0 | 2 +-
.../parquet_append_table/schema/schema-0 | 2 +-
.../pk_table_nested_type/schema/schema-0 | 2 +-
30 files changed, 154 insertions(+), 74 deletions(-)
diff --git a/docs/source/user_guide/data_types.rst
b/docs/source/user_guide/data_types.rst
index f532642..3d52933 100644
--- a/docs/source/user_guide/data_types.rst
+++ b/docs/source/user_guide/data_types.rst
@@ -188,15 +188,14 @@ and `Arrow DataTypes
<https://arrow.apache.org/docs/format/Columnar.html#data-ty
* - ``MAP<kt, vt>``
- Map
- - Data type of an associative array that maps keys to values (including
NULL). A map cannot contain duplicate keys; each key can map to at most one
value.
+ - Data type of an associative array that maps keys (including NULL) to
values (including NULL). A map cannot contain duplicate keys; each key can map
to at most one value.
There is no restriction of element types; it is the responsibility of
the user to ensure uniqueness.
The type can be declared using ``MAP<kt, vt>`` where kt is the data
type of the key elements and vt is the data type of the value elements.
- **Note:** In Paimon C++, map keys must be explicitly marked as ``NOT
NULL``.
- Apache Arrow does not support nullable map keys. If the key type is not
- marked as ``NOT NULL`` in the schema, parsing will fail with an error.
+ **Note:** Paimon C++ accepts nullable map key declarations for schema
compatibility.
+ A query fails only if the data actually contains a NULL map key.
* - ``MULTISET<t>``
- Not Supported
diff --git a/docs/source/user_guide/schema.rst
b/docs/source/user_guide/schema.rst
index 0fe8e72..5db69ef 100644
--- a/docs/source/user_guide/schema.rst
+++ b/docs/source/user_guide/schema.rst
@@ -84,35 +84,11 @@ DataField represents a column of the table.
3. ``type``: data type, very similar to SQL type string.
4. ``description``: string.
-Limitations
------------
+Nullable MAP Keys
+-----------------
-MAP Key Must Be NOT NULL
-^^^^^^^^^^^^^^^^^^^^^^^^
-
-Apache Arrow does not support nullable map keys. When defining a ``MAP`` type
in the schema,
-the key must be explicitly marked as ``NOT NULL``. If the key is not marked as
``NOT NULL``,
-schema parsing will fail with an error.
-
-For example, the following is **valid**:
-
-.. code-block:: json
-
- {
- "type": "MAP",
- "key": "TINYINT NOT NULL",
- "value": "SMALLINT"
- }
-
-The following is **invalid** and will be rejected:
-
-.. code-block:: json
-
- {
- "type": "MAP",
- "key": "TINYINT",
- "value": "SMALLINT"
- }
+Paimon C++ accepts nullable ``MAP`` key declarations for compatibility with
existing schemas.
+A query fails only when the data actually contains a NULL map key.
Update Schema
-------------
diff --git a/src/paimon/common/types/data_type_json_parser.cpp
b/src/paimon/common/types/data_type_json_parser.cpp
index 950308a..33308ab 100644
--- a/src/paimon/common/types/data_type_json_parser.cpp
+++ b/src/paimon/common/types/data_type_json_parser.cpp
@@ -689,17 +689,13 @@ Result<std::shared_ptr<arrow::Field>>
DataTypeJsonParser::ParseMapType(
PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<arrow::Field> key,
ParseType("key", type_json_value["key"]));
+
// NOTE: Unlike Java Paimon, this C++ implementation does not support
nullable keys in
// MapType. This is a limitation of Apache Arrow, which does not allow
null keys in its
- // MapType. As a result, we validate `nullable = false` for the map key.
- if (key->nullable()) {
- return Status::Invalid(fmt::format(
- "Map field '{}' has a nullable key."
- "Map keys must be explicitly marked as NOT NULL in the schema for
paimon-cpp "
- "because Apache Arrow does not support nullable map keys. "
- "Please add 'NOT NULL' to the key type definition.",
- name));
- }
+ // MapType. As a result, we explicitly set `nullable = false` for the map
key, regardless of
+ // the original schema. Please be aware of this behavioral difference when
migrating or
+ // interoperating with Java Paimon.
+ key = key->WithNullable(false);
PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<arrow::Field> value,
ParseType("value", type_json_value["value"]));
return arrow::field(name, std::make_shared<arrow::MapType>(key, value),
nullable);
diff --git a/src/paimon/common/types/data_type_json_parser_test.cpp
b/src/paimon/common/types/data_type_json_parser_test.cpp
index bbb4fb5..bfa69c3 100644
--- a/src/paimon/common/types/data_type_json_parser_test.cpp
+++ b/src/paimon/common/types/data_type_json_parser_test.cpp
@@ -51,7 +51,7 @@ TEST(DataTypeJsonParserTest, ParseTypeMapTypeSuccess) {
const std::string name = "map_field";
const char* json = R"({
"type": "MAP",
- "key": "STRING NOT NULL",
+ "key": "STRING",
"value": "INT"
})";
rapidjson::Document doc;
@@ -60,6 +60,8 @@ TEST(DataTypeJsonParserTest, ParseTypeMapTypeSuccess) {
ASSERT_OK_AND_ASSIGN(std::shared_ptr<arrow::Field> field,
DataTypeJsonParser::ParseType(name, doc));
ASSERT_NE(field, nullptr);
+ auto map_type = std::static_pointer_cast<arrow::MapType>(field->type());
+ ASSERT_FALSE(map_type->key_field()->nullable());
}
TEST(DataTypeJsonParserTest, ParseTypeRowTypeSuccess) {
diff --git a/src/paimon/core/schema/arrow_schema_validator.cpp
b/src/paimon/core/schema/arrow_schema_validator.cpp
index b9ea5da..e01afe0 100644
--- a/src/paimon/core/schema/arrow_schema_validator.cpp
+++ b/src/paimon/core/schema/arrow_schema_validator.cpp
@@ -231,10 +231,6 @@ Status ArrowSchemaValidator::ValidateField(const
std::shared_ptr<arrow::Field>&
arrow::internal::checked_cast<const
arrow::MapType&>(*field->type()).key_field();
const auto& item_field =
arrow::internal::checked_cast<const
arrow::MapType&>(*field->type()).item_field();
- if (key_field->nullable()) {
- return Status::Invalid(
- fmt::format("Map field '{}' has a nullable key.",
field->name()));
- }
PAIMON_RETURN_NOT_OK(ValidateField(key_field,
/*allow_blob=*/false));
PAIMON_RETURN_NOT_OK(ValidateField(item_field,
/*allow_blob=*/false));
break;
diff --git a/src/paimon/core/schema/schema_validation_test.cpp
b/src/paimon/core/schema/schema_validation_test.cpp
index 57179e8..3c3054a 100644
--- a/src/paimon/core/schema/schema_validation_test.cpp
+++ b/src/paimon/core/schema/schema_validation_test.cpp
@@ -938,7 +938,7 @@ TEST(SchemaValidationTest, TestMapStorageLayout) {
}
}
-TEST(SchemaValidationTest, TestMapRequiresNonNullableKey) {
+TEST(SchemaValidationTest, TestMapSharedShreddingRequiresNonNullableKey) {
auto nullable_key_map =
std::make_shared<arrow::MapType>(arrow::field("key", arrow::utf8(),
/*nullable=*/true),
arrow::field("value",
arrow::int64()));
@@ -951,9 +951,11 @@ TEST(SchemaValidationTest, TestMapRequiresNonNullableKey) {
{Options::BUCKET_KEY, "f0"},
{"fields.f1.map.storage-layout", "shared-shredding"},
};
- ASSERT_NOK_WITH_MSG(TableSchema::Create(/*schema_id=*/0, schema,
/*partition_keys=*/{},
- /*primary_keys=*/{}, options),
- "Map field 'f1' has a nullable key.");
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<TableSchema> table_schema,
+ TableSchema::Create(/*schema_id=*/0, schema,
/*partition_keys=*/{},
+ /*primary_keys=*/{}, options));
+ ASSERT_NOK_WITH_MSG(SchemaValidation::ValidateTableSchema(*table_schema),
+ "map key type is nullable");
}
TEST(SchemaValidationTest, TestMapSharedShreddingRejectsBlobValue) {
diff --git a/src/paimon/core/schema/table_schema_test.cpp
b/src/paimon/core/schema/table_schema_test.cpp
index fd0ac42..1a46eaa 100644
--- a/src/paimon/core/schema/table_schema_test.cpp
+++ b/src/paimon/core/schema/table_schema_test.cpp
@@ -1297,7 +1297,7 @@ TEST_F(TableSchemaTest, SetFieldIdNestedListInStruct) {
}
}
-TEST_F(TableSchemaTest, MapKeyMustBeNotNull) {
+TEST_F(TableSchemaTest, NullableMapKeySchemaIsSupported) {
std::string table_schema_str = R"({
"version" : 3,
"id" : 0,
@@ -1316,16 +1316,21 @@ TEST_F(TableSchemaTest, MapKeyMustBeNotNull) {
"options" : {},
"timeMillis" : 1721614341162
})";
- ASSERT_NOK_WITH_MSG(TableSchema::CreateFromJson(table_schema_str),
- "Map field 'f0' has a nullable key.");
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<TableSchema> table_schema,
+ TableSchema::CreateFromJson(table_schema_str));
+ auto json_map_type =
std::static_pointer_cast<arrow::MapType>(table_schema->Fields()[0].Type());
+ ASSERT_FALSE(json_map_type->key_field()->nullable());
auto nullable_key_map =
std::make_shared<arrow::MapType>(arrow::field("key", arrow::int8(),
/*nullable=*/true),
arrow::field("value",
arrow::int16()));
- ASSERT_NOK_WITH_MSG(
+ ASSERT_OK_AND_ASSIGN(
+ std::shared_ptr<TableSchema> direct_table_schema,
TableSchema::Create(/*schema_id=*/0, arrow::schema({arrow::field("f0",
nullable_key_map)}),
- /*partition_keys=*/{}, /*primary_keys=*/{},
/*options=*/{}),
- "Map field 'f0' has a nullable key.");
+ /*partition_keys=*/{}, /*primary_keys=*/{},
/*options=*/{}));
+ auto direct_map_type =
+
std::static_pointer_cast<arrow::MapType>(direct_table_schema->Fields()[0].Type());
+ ASSERT_TRUE(direct_map_type->key_field()->nullable());
}
TEST_F(TableSchemaTest, MapKeysSortedIsNormalized) {
diff --git a/src/paimon/format/avro/avro_file_batch_reader.cpp
b/src/paimon/format/avro/avro_file_batch_reader.cpp
index 92ac769..f48ec4c 100644
--- a/src/paimon/format/avro/avro_file_batch_reader.cpp
+++ b/src/paimon/format/avro/avro_file_batch_reader.cpp
@@ -123,6 +123,7 @@ Result<BatchReader::ReadBatch>
AvroFileBatchReader::NextBatch() {
}
PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Array> array,
array_builder_->Finish());
+ PAIMON_RETURN_NOT_OK_FROM_ARROW(array->Validate());
std::unique_ptr<ArrowArray> c_array = std::make_unique<ArrowArray>();
std::unique_ptr<ArrowSchema> c_schema =
std::make_unique<ArrowSchema>();
PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*array,
c_array.get(), c_schema.get()));
diff --git a/src/paimon/format/orc/orc_adapter.cpp
b/src/paimon/format/orc/orc_adapter.cpp
index 992387d..4773f2e 100644
--- a/src/paimon/format/orc/orc_adapter.cpp
+++ b/src/paimon/format/orc/orc_adapter.cpp
@@ -945,6 +945,7 @@ Result<std::shared_ptr<arrow::Array>>
OrcAdapter::AppendBatch(
MakeArrowBuilder(type, batch, pool));
std::shared_ptr<arrow::Array> array;
PAIMON_RETURN_NOT_OK_FROM_ARROW(builder->Finish(&array));
+ PAIMON_RETURN_NOT_OK_FROM_ARROW(array->Validate());
return array;
}
diff --git a/src/paimon/format/parquet/parquet_file_batch_reader.cpp
b/src/paimon/format/parquet/parquet_file_batch_reader.cpp
index 4f1d013..fcc47c1 100644
--- a/src/paimon/format/parquet/parquet_file_batch_reader.cpp
+++ b/src/paimon/format/parquet/parquet_file_batch_reader.cpp
@@ -537,6 +537,7 @@ Result<BatchReader::ReadBatch>
ParquetFileBatchReader::NextBatch() {
}
PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Array> array,
batch->ToStructArray());
+ PAIMON_RETURN_NOT_OK_FROM_ARROW(array->Validate());
PAIMON_ASSIGN_OR_RAISE(bool need_cast,
ParquetTimestampConverter::NeedCastArrayForTimestamp(
array->type(),
read_data_type_));
if (need_cast) {
diff --git a/test/inte/scan_and_read_inte_test.cpp
b/test/inte/scan_and_read_inte_test.cpp
index a9294ef..2338463 100644
--- a/test/inte/scan_and_read_inte_test.cpp
+++ b/test/inte/scan_and_read_inte_test.cpp
@@ -2819,6 +2819,50 @@ TEST_P(ScanAndReadInteTest,
TestWithPKBucketSelectByPredicate) {
ASSERT_TRUE(expected->Equals(read_result)) << read_result->ToString();
}
+TEST_P(ScanAndReadInteTest, TestReadNullableMapKey) {
+ auto file_format = FileFormat();
+ if (file_format != "orc") {
+ return;
+ }
+ // Java Parquet does not support writing null MAP keys; see
+ // ParquetRowDataWriter.MapWriter#writeMapData in Apache Paimon.
Therefore, this test uses an
+ // ORC table. The table contains three rows: the first two rows have
non-null MAP keys, while
+ // the third row has a null MAP key and is expected to fail during reading.
+ const std::string table_path =
+ paimon::test::GetDataDir() +
"orc/nullable_map_key.db/nullable_map_key";
+
+ ScanContextBuilder scan_context_builder(table_path);
+ ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish());
+ ASSERT_OK_AND_ASSIGN(auto table_scan,
TableScan::Create(std::move(scan_context)));
+ ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan());
+ ASSERT_TRUE(result_plan->SnapshotId());
+ ASSERT_EQ(result_plan->SnapshotId().value(), 1);
+ ASSERT_EQ(result_plan->Splits().size(), 1);
+
+ ReadContextBuilder read_context_builder(table_path);
+ read_context_builder.AddOption(Options::READ_BATCH_SIZE, "1");
+ ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish());
+ ASSERT_OK_AND_ASSIGN(auto table_read,
TableRead::Create(std::move(read_context)));
+ ASSERT_OK_AND_ASSIGN(auto batch_reader,
table_read->CreateReader(result_plan->Splits()));
+
+ const std::vector<std::string> expected_rows = {R"([[0, 1, [["one",
10]]]])",
+ R"([[0, 2, [["two",
20]]]])"};
+ for (int32_t i = 0; i < 2; ++i) {
+ ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch batch,
batch_reader->NextBatch());
+ ASSERT_FALSE(BatchReader::IsEofBatch(batch));
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<arrow::Array> array,
+ ReadResultCollector::GetArray(std::move(batch)));
+ ASSERT_EQ(array->length(), 1);
+
+ std::shared_ptr<arrow::Array> expected_array =
+ arrow::ipc::internal::json::ArrayFromJSON(array->type(),
expected_rows[i]).ValueOrDie();
+ ASSERT_TRUE(array->Equals(expected_array))
+ << "actual: " << array->ToString() << ", expected: " <<
expected_array->ToString();
+ }
+ ASSERT_NOK_WITH_MSG(batch_reader->NextBatch(), "Map array keys array
should have no nulls");
+ batch_reader->Close();
+}
+
TEST_P(ScanAndReadInteTest, TestCountRowsEmptySplits) {
auto file_format = FileFormat();
std::string table_path = paimon::test::GetDataDir() + file_format +
diff --git
a/test/test_data/avro/append_multiple.db/append_multiple/schema/schema-0
b/test/test_data/avro/append_multiple.db/append_multiple/schema/schema-0
index e3b8f71..c0d6d75 100644
--- a/test/test_data/avro/append_multiple.db/append_multiple/schema/schema-0
+++ b/test/test_data/avro/append_multiple.db/append_multiple/schema/schema-0
@@ -93,7 +93,7 @@
"name": "f0",
"type": {
"type": "MAP",
- "key": "STRING NOT NULL",
+ "key": "STRING",
"value": "INT"
}
},
diff --git a/test/test_data/avro/append_simple.db/append_simple/schema/schema-0
b/test/test_data/avro/append_simple.db/append_simple/schema/schema-0
index c8b4be2..be12e26 100644
--- a/test/test_data/avro/append_simple.db/append_simple/schema/schema-0
+++ b/test/test_data/avro/append_simple.db/append_simple/schema/schema-0
@@ -23,7 +23,7 @@
"name" : "f0",
"type" : {
"type" : "MAP",
- "key" : "STRING NOT NULL",
+ "key" : "STRING",
"value" : "INT"
}
}, {
diff --git
a/test/test_data/avro/append_with_multiple_map.db/append_with_multiple_map/schema/schema-0
b/test/test_data/avro/append_with_multiple_map.db/append_with_multiple_map/schema/schema-0
index 7237632..b118e22 100644
---
a/test/test_data/avro/append_with_multiple_map.db/append_with_multiple_map/schema/schema-0
+++
b/test/test_data/avro/append_with_multiple_map.db/append_with_multiple_map/schema/schema-0
@@ -6,7 +6,7 @@
"name" : "f0",
"type" : {
"type" : "MAP",
- "key" : "INT NOT NULL",
+ "key" : "INT",
"value" : "INT"
}
}, {
@@ -14,7 +14,7 @@
"name" : "f1",
"type" : {
"type" : "MAP",
- "key" : "DOUBLE NOT NULL",
+ "key" : "DOUBLE",
"value" : "DOUBLE"
}
}, {
@@ -22,7 +22,7 @@
"name" : "f2",
"type" : {
"type" : "MAP",
- "key" : "STRING NOT NULL",
+ "key" : "STRING",
"value" : "STRING"
}
}, {
@@ -30,7 +30,7 @@
"name" : "f3",
"type" : {
"type" : "MAP",
- "key" : "STRING NOT NULL",
+ "key" : "STRING",
"value" : "BINARY(6)"
}
}, {
@@ -38,7 +38,7 @@
"name" : "f4",
"type" : {
"type" : "MAP",
- "key" : "TIMESTAMP(6) NOT NULL",
+ "key" : "TIMESTAMP(6)",
"value" : "TIMESTAMP(6)"
}
}, {
@@ -46,7 +46,7 @@
"name" : "f5",
"type" : {
"type" : "MAP",
- "key" : "STRING NOT NULL",
+ "key" : "STRING",
"value" : {
"type" : "ARRAY",
"element" : "DOUBLE"
@@ -57,10 +57,10 @@
"name" : "f6",
"type" : {
"type" : "MAP",
- "key" : "STRING NOT NULL",
+ "key" : "STRING",
"value" : {
"type" : "MAP",
- "key" : "DOUBLE NOT NULL",
+ "key" : "DOUBLE",
"value" : "STRING"
}
}
@@ -69,7 +69,7 @@
"name" : "f7",
"type" : {
"type" : "MAP",
- "key" : "BIGINT NOT NULL",
+ "key" : "BIGINT",
"value" : {
"type" : "ROW",
"fields" : [ {
diff --git
a/test/test_data/avro/pk_with_multiple_type.db/pk_with_multiple_type/schema/schema-0
b/test/test_data/avro/pk_with_multiple_type.db/pk_with_multiple_type/schema/schema-0
index 5533e14..608d0df 100644
---
a/test/test_data/avro/pk_with_multiple_type.db/pk_with_multiple_type/schema/schema-0
+++
b/test/test_data/avro/pk_with_multiple_type.db/pk_with_multiple_type/schema/schema-0
@@ -55,7 +55,7 @@
"name" : "f0",
"type" : {
"type" : "MAP",
- "key" : "STRING NOT NULL",
+ "key" : "STRING",
"value" : "INT"
}
}, {
diff --git
a/test/test_data/orc/append_complex_build_in_fieldid.db/append_complex_build_in_fieldid/schema/schema-0
b/test/test_data/orc/append_complex_build_in_fieldid.db/append_complex_build_in_fieldid/schema/schema-0
index 699682f..f973a64 100644
---
a/test/test_data/orc/append_complex_build_in_fieldid.db/append_complex_build_in_fieldid/schema/schema-0
+++
b/test/test_data/orc/append_complex_build_in_fieldid.db/append_complex_build_in_fieldid/schema/schema-0
@@ -6,7 +6,7 @@
"name" : "f1",
"type" : {
"type" : "MAP",
- "key" : "TINYINT NOT NULL",
+ "key" : "TINYINT",
"value" : "SMALLINT"
}
}, {
diff --git
a/test/test_data/orc/append_table_with_nested_type.db/append_table_with_nested_type/schema/schema-0
b/test/test_data/orc/append_table_with_nested_type.db/append_table_with_nested_type/schema/schema-0
index c754fec..57cd38a 100644
---
a/test/test_data/orc/append_table_with_nested_type.db/append_table_with_nested_type/schema/schema-0
+++
b/test/test_data/orc/append_table_with_nested_type.db/append_table_with_nested_type/schema/schema-0
@@ -64,7 +64,7 @@
"type" : {
"type" : "MAP",
"key" : {
- "type" : "ROW NOT NULL",
+ "type" : "ROW",
"fields" : [ {
"id" : 13,
"name" : "sub1",
diff --git a/test/test_data/orc/nullable_map_key.db/nullable_map_key/README
b/test/test_data/orc/nullable_map_key.db/nullable_map_key/README
new file mode 100644
index 0000000..02b0d4e
--- /dev/null
+++ b/test/test_data/orc/nullable_map_key.db/nullable_map_key/README
@@ -0,0 +1,13 @@
+id:int attrs:map(string, int) (all can be null)
+no partition key
+no primary key
+bucket count: -1
+file format: orc
+
+Rows (snapshot-1):
+Add: [1, {"one": 10}]
+Add: [2, {"two": 20}]
+Add: [3, {null: 30}]
+NoCompact
+
+The third row contains a null MAP key and is expected to fail during reading.
diff --git
a/test/test_data/orc/nullable_map_key.db/nullable_map_key/bucket-0/data-7e6899e9-f3cd-4d2e-86bc-7a5f383d5922-0.orc
b/test/test_data/orc/nullable_map_key.db/nullable_map_key/bucket-0/data-7e6899e9-f3cd-4d2e-86bc-7a5f383d5922-0.orc
new file mode 100644
index 0000000..89c6f60
Binary files /dev/null and
b/test/test_data/orc/nullable_map_key.db/nullable_map_key/bucket-0/data-7e6899e9-f3cd-4d2e-86bc-7a5f383d5922-0.orc
differ
diff --git
a/test/test_data/orc/nullable_map_key.db/nullable_map_key/manifest/manifest-a956b9da-04c5-41b6-b6f0-20efa3974bd4-0
b/test/test_data/orc/nullable_map_key.db/nullable_map_key/manifest/manifest-a956b9da-04c5-41b6-b6f0-20efa3974bd4-0
new file mode 100644
index 0000000..e96a4bb
Binary files /dev/null and
b/test/test_data/orc/nullable_map_key.db/nullable_map_key/manifest/manifest-a956b9da-04c5-41b6-b6f0-20efa3974bd4-0
differ
diff --git
a/test/test_data/orc/nullable_map_key.db/nullable_map_key/manifest/manifest-list-67c61c9b-ab3c-4564-a5f2-612aa684a8f2-0
b/test/test_data/orc/nullable_map_key.db/nullable_map_key/manifest/manifest-list-67c61c9b-ab3c-4564-a5f2-612aa684a8f2-0
new file mode 100644
index 0000000..0b80517
Binary files /dev/null and
b/test/test_data/orc/nullable_map_key.db/nullable_map_key/manifest/manifest-list-67c61c9b-ab3c-4564-a5f2-612aa684a8f2-0
differ
diff --git
a/test/test_data/orc/nullable_map_key.db/nullable_map_key/manifest/manifest-list-67c61c9b-ab3c-4564-a5f2-612aa684a8f2-1
b/test/test_data/orc/nullable_map_key.db/nullable_map_key/manifest/manifest-list-67c61c9b-ab3c-4564-a5f2-612aa684a8f2-1
new file mode 100644
index 0000000..47fc5e9
Binary files /dev/null and
b/test/test_data/orc/nullable_map_key.db/nullable_map_key/manifest/manifest-list-67c61c9b-ab3c-4564-a5f2-612aa684a8f2-1
differ
diff --git
a/test/test_data/orc/nullable_map_key.db/nullable_map_key/schema/schema-0
b/test/test_data/orc/nullable_map_key.db/nullable_map_key/schema/schema-0
new file mode 100644
index 0000000..e1c711c
--- /dev/null
+++ b/test/test_data/orc/nullable_map_key.db/nullable_map_key/schema/schema-0
@@ -0,0 +1,25 @@
+{
+ "version" : 3,
+ "id" : 0,
+ "fields" : [ {
+ "id" : 0,
+ "name" : "id",
+ "type" : "INT"
+ }, {
+ "id" : 1,
+ "name" : "attrs",
+ "type" : {
+ "type" : "MAP",
+ "key" : "STRING",
+ "value" : "INT"
+ }
+ } ],
+ "highestFieldId" : 1,
+ "partitionKeys" : [ ],
+ "primaryKeys" : [ ],
+ "options" : {
+ "bucket" : "-1",
+ "file.format" : "orc"
+ },
+ "timeMillis" : 1785815466343
+}
\ No newline at end of file
diff --git
a/test/test_data/orc/nullable_map_key.db/nullable_map_key/snapshot/EARLIEST
b/test/test_data/orc/nullable_map_key.db/nullable_map_key/snapshot/EARLIEST
new file mode 100644
index 0000000..56a6051
--- /dev/null
+++ b/test/test_data/orc/nullable_map_key.db/nullable_map_key/snapshot/EARLIEST
@@ -0,0 +1 @@
+1
\ No newline at end of file
diff --git
a/test/test_data/orc/nullable_map_key.db/nullable_map_key/snapshot/LATEST
b/test/test_data/orc/nullable_map_key.db/nullable_map_key/snapshot/LATEST
new file mode 100644
index 0000000..56a6051
--- /dev/null
+++ b/test/test_data/orc/nullable_map_key.db/nullable_map_key/snapshot/LATEST
@@ -0,0 +1 @@
+1
\ No newline at end of file
diff --git
a/test/test_data/orc/nullable_map_key.db/nullable_map_key/snapshot/snapshot-1
b/test/test_data/orc/nullable_map_key.db/nullable_map_key/snapshot/snapshot-1
new file mode 100644
index 0000000..78c3467
--- /dev/null
+++
b/test/test_data/orc/nullable_map_key.db/nullable_map_key/snapshot/snapshot-1
@@ -0,0 +1,17 @@
+{
+ "version" : 3,
+ "uuid" : "67b9daa5-c055-45f9-ac6a-d87d017490b1",
+ "id" : 1,
+ "schemaId" : 0,
+ "baseManifestList" : "manifest-list-67c61c9b-ab3c-4564-a5f2-612aa684a8f2-0",
+ "baseManifestListSize" : 1006,
+ "deltaManifestList" : "manifest-list-67c61c9b-ab3c-4564-a5f2-612aa684a8f2-1",
+ "deltaManifestListSize" : 1110,
+ "commitUser" : "8bd3a648-0532-4a3e-8815-9e1da2e6d7f9",
+ "commitIdentifier" : 0,
+ "commitKind" : "APPEND",
+ "timeMillis" : 1785815467254,
+ "totalRecordCount" : 3,
+ "deltaRecordCount" : 3,
+ "nextRowId" : 0
+}
\ No newline at end of file
diff --git
a/test/test_data/orc/pk_table_nested_type.db/pk_table_nested_type/schema/schema-0
b/test/test_data/orc/pk_table_nested_type.db/pk_table_nested_type/schema/schema-0
index 7a3fdc5..7f38243 100644
---
a/test/test_data/orc/pk_table_nested_type.db/pk_table_nested_type/schema/schema-0
+++
b/test/test_data/orc/pk_table_nested_type.db/pk_table_nested_type/schema/schema-0
@@ -44,7 +44,7 @@
"name" : "col2",
"type" : {
"type" : "MAP",
- "key" : "INT NOT NULL",
+ "key" : "INT",
"value" : "INT"
}
} ],
diff --git
a/test/test_data/parquet/append_complex_build_in_fieldid.db/append_complex_build_in_fieldid/schema/schema-0
b/test/test_data/parquet/append_complex_build_in_fieldid.db/append_complex_build_in_fieldid/schema/schema-0
index 4bf9bca..296bbca 100644
---
a/test/test_data/parquet/append_complex_build_in_fieldid.db/append_complex_build_in_fieldid/schema/schema-0
+++
b/test/test_data/parquet/append_complex_build_in_fieldid.db/append_complex_build_in_fieldid/schema/schema-0
@@ -6,7 +6,7 @@
"name" : "f1",
"type" : {
"type" : "MAP",
- "key" : "TINYINT NOT NULL",
+ "key" : "TINYINT",
"value" : "SMALLINT"
}
}, {
diff --git
a/test/test_data/parquet/parquet_append_table.db/parquet_append_table/schema/schema-0
b/test/test_data/parquet/parquet_append_table.db/parquet_append_table/schema/schema-0
index 524117c..ae19f8a 100644
---
a/test/test_data/parquet/parquet_append_table.db/parquet_append_table/schema/schema-0
+++
b/test/test_data/parquet/parquet_append_table.db/parquet_append_table/schema/schema-0
@@ -43,7 +43,7 @@
"type" : {
"type" : "MAP",
"key" : {
- "type" : "ARRAY NOT NULL",
+ "type" : "ARRAY",
"element" : "FLOAT"
},
"value" : {
diff --git
a/test/test_data/parquet/pk_table_nested_type.db/pk_table_nested_type/schema/schema-0
b/test/test_data/parquet/pk_table_nested_type.db/pk_table_nested_type/schema/schema-0
index 22968d4..db05092 100644
---
a/test/test_data/parquet/pk_table_nested_type.db/pk_table_nested_type/schema/schema-0
+++
b/test/test_data/parquet/pk_table_nested_type.db/pk_table_nested_type/schema/schema-0
@@ -44,7 +44,7 @@
"name" : "col2",
"type" : {
"type" : "MAP",
- "key" : "INT NOT NULL",
+ "key" : "INT",
"value" : "INT"
}
} ],
