lxy-9602 commented on code in PR #172:
URL: https://github.com/apache/paimon-cpp/pull/172#discussion_r3710586701
##########
test/inte/write_and_read_inte_test.cpp:
##########
@@ -806,6 +806,161 @@ TEST_P(WriteAndReadInteTest, TestPkTimestampType) {
ASSERT_TRUE(success);
}
+/// End-to-end coverage for second-precision timestamps nested inside
list/struct/map.
+/// Parquet has no second-precision timestamp, so the writer stores those
leaves as milli and
+/// the reader has to convert milli back to second for every nested leaf.
+TEST_P(WriteAndReadInteTest, TestAppendNestedTimestampSecondPrecision) {
+ auto [file_format, file_system] = GetParam();
+ TimezoneGuard timezone_guard("Asia/Shanghai");
+ auto timezone = DateTimeUtils::GetLocalTimezoneName();
+ auto event_type = arrow::struct_({
+ arrow::field("name", arrow::utf8()),
+ arrow::field("ts_sec", arrow::timestamp(arrow::TimeUnit::SECOND)),
+ arrow::field("ts_ltz_sec", arrow::timestamp(arrow::TimeUnit::SECOND,
timezone)),
+ });
+ arrow::FieldVector fields = {
+ arrow::field("events", arrow::list(arrow::field("element",
event_type))),
+ arrow::field("marks", arrow::map(arrow::utf8(),
arrow::timestamp(arrow::TimeUnit::SECOND))),
+ };
+ std::map<std::string, std::string> options = {
+ {Options::MANIFEST_FORMAT, "avro"}, {Options::FILE_FORMAT,
file_format},
+ {Options::TARGET_FILE_SIZE, "1024"}, {Options::BUCKET, "-1"},
+ {Options::FILE_SYSTEM, file_system}, {"orc.timestamp-ltz.legacy.type",
"false"}};
+ if (file_system == "jindo") {
+ options = AddOptionsForJindo(options);
+ }
+ ASSERT_OK_AND_ASSIGN(auto helper, TestHelper::Create(test_dir_,
arrow::schema(fields),
+
/*partition_keys=*/{}, /*primary_keys=*/{},
+ options,
/*is_streaming_mode=*/false));
+ std::string data = R"([
+ [[["e-1", "1970-01-01 00:00:01", "1970-01-01 00:00:02"]],
+ [["begin", "1970-01-01 00:00:03"]]],
+ [[["e-2", "1970-01-01 00:00:04", null], ["e-3", null, "1970-01-01
00:00:05"]], []],
+ [[null], null]
+ ])";
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch,
+ TestHelper::MakeRecordBatch(arrow::struct_(fields),
data,
+ /*partition_map=*/{},
/*bucket=*/0, {}));
+ ASSERT_OK(helper->WriteAndCommit(std::move(batch), /*commit_identifier=*/0,
+
/*expected_commit_messages=*/std::nullopt));
+
+ if (file_format == "parquet") {
+ // Verify the precision downgrade really happened, so the read below
is exercising an
+ // actual milli to second conversion instead of a no-op.
Review Comment:
Please rm file verify in E2E tests.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
