lxy-9602 commented on code in PR #295:
URL: https://github.com/apache/paimon-cpp/pull/295#discussion_r3964060283
##########
test/inte/data_evolution_table_test.cpp:
##########
@@ -1651,6 +1651,251 @@ TEST_P(DataEvolutionTableTest,
TestPartitionWithPredicate) {
}
}
+TEST_P(DataEvolutionTableTest, TestVectorReadWrite) {
+ if (FileFormat() != "parquet") {
+ GTEST_SKIP() << "VECTOR currently only supports Parquet data files";
+ }
+ auto vector_type =
+ arrow::fixed_size_list(arrow::field("item", arrow::float32(),
/*nullable=*/false), 3);
+ arrow::FieldVector fields = {arrow::field("id", arrow::int32()),
+ arrow::field("embedding", vector_type),
+ arrow::field("tag", arrow::utf8())};
+ std::map<std::string, std::string> options = {
+ {Options::FILE_FORMAT, "parquet"},
+ {Options::FILE_SYSTEM, "local"},
+ {Options::BUCKET, "-1"},
+ {Options::ROW_TRACKING_ENABLED, "true"},
+ {Options::DATA_EVOLUTION_ENABLED, "true"},
+ {Options::READ_BATCH_SIZE, "2"},
+ };
+ CreateTable(fields, /*partition_keys=*/{}, options);
+ std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar");
+ auto field_names = arrow::schema(fields)->field_names();
+ auto initial_array = std::dynamic_pointer_cast<arrow::StructArray>(
+ arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([
+ [1, [1, 2, 3], "a"],
Review Comment:
Consider use `checked_pointer_cast`
##########
src/paimon/core/utils/nested_projection_utils.cpp:
##########
@@ -193,10 +200,31 @@ Result<bool> EqualWithFieldIds(const
std::shared_ptr<arrow::DataType>& a,
return true;
}
+Status ValidateVectorEvolution(const std::shared_ptr<arrow::DataType>&
read_type,
+ const std::shared_ptr<arrow::DataType>&
data_type) {
+ bool read_is_vector = read_type->id() == arrow::Type::FIXED_SIZE_LIST;
+ bool data_is_vector = data_type->id() == arrow::Type::FIXED_SIZE_LIST;
+ if (!read_is_vector && !data_is_vector) {
+ return Status::OK();
+ }
+ if (read_is_vector && data_is_vector) {
+ const auto& read_vector = checked_cast<const
arrow::FixedSizeListType&>(*read_type);
+ const auto& data_vector = checked_cast<const
arrow::FixedSizeListType&>(*data_type);
+ if (read_vector.list_size() == data_vector.list_size() &&
+ read_vector.value_type()->Equals(data_vector.value_type())) {
+ return Status::OK();
+ }
+ }
+ return Status::Invalid(
+ fmt::format("VECTOR type mismatch during schema evolution: data {} vs
read {}",
+ data_type->ToString(), read_type->ToString()));
+}
+
Review Comment:
Should we consider moving this function into `VectorUtils`?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
