SteNicholas commented on code in PR #479:
URL: https://github.com/apache/paimon-cpp/pull/479#discussion_r4415698682
##########
test/inte/write_and_read_inte_test.cpp:
##########
@@ -6027,6 +6028,115 @@ TEST(TimeColumnWriteInteTest,
TestAppendTimeColumnFinalize) {
}
#endif
+#ifdef PAIMON_ENABLE_LANCE
+class LanceTimestampLtzInteTest
+ : public ::testing::Test,
+ public ::testing::WithParamInterface<std::tuple<std::string, bool>> {};
+
+TEST_P(LanceTimestampLtzInteTest, TestWriteAndReadInPinnedTimezone) {
+ TimezoneGuard timezone_guard(std::get<0>(GetParam()));
+ const std::string timezone = DateTimeUtils::GetLocalTimezoneName();
+ const bool with_primary_key = std::get<1>(GetParam());
+ auto dir = UniqueTestDirectory::Create("local");
+ ASSERT_NE(dir, nullptr);
+
+ auto ltz = [&timezone](arrow::TimeUnit::type unit) { return
arrow::timestamp(unit, timezone); };
+ arrow::FieldVector fields = {
+ arrow::field("id", arrow::int32()),
+ arrow::field("ltz_0", ltz(arrow::TimeUnit::SECOND)),
+ arrow::field("ltz_9", ltz(arrow::TimeUnit::NANO)),
+ arrow::field("ltz_array", arrow::list(ltz(arrow::TimeUnit::MICRO))),
+ arrow::field(
+ "ltz_row",
+ arrow::struct_({arrow::field("ltz_3", ltz(arrow::TimeUnit::MILLI)),
+ arrow::field("ntz",
arrow::timestamp(arrow::TimeUnit::MICRO))})),
+ };
+ std::map<std::string, std::string> options = {{Options::FILE_FORMAT,
"lance"},
+ {Options::BUCKET,
with_primary_key ? "1" : "-1"},
+ {Options::FILE_SYSTEM,
"local"}};
+ std::vector<std::string> primary_keys;
+ if (with_primary_key) {
+ primary_keys.push_back("id");
+ }
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<TestHelper> helper,
+ TestHelper::Create(dir->Str(), arrow::schema(fields),
+ /*partition_keys=*/{},
primary_keys, options,
+ /*is_streaming_mode=*/false));
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch_v0,
+ TestHelper::MakeRecordBatch(arrow::struct_(fields),
R"([
+ [1, "1969-12-31 23:59:59", "1969-12-31 23:59:59.999999999",
+ ["2024-10-04 12:34:56.123456", null],
+ ["1970-01-01 00:00:00.001", "1970-01-01 00:00:00.000001"]],
+ [2, null, null, null, [null, null]],
+ [3, "2024-10-04 12:34:56", "2024-10-04 12:34:56.123456789", [],
+ ["2024-10-04 12:34:56.123", null]]
+ ])",
+ /*partition_map=*/{},
/*bucket=*/0, {}));
+ ASSERT_OK(helper->WriteAndCommit(std::move(batch_v0),
/*commit_identifier=*/0,
+
/*expected_commit_messages=*/std::nullopt));
+
+ const std::string table_path = PathUtil::JoinPath(dir->Str(),
"foo.db/bar");
+ SchemaManager schema_manager(dir->GetFileSystem(), table_path);
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<TableSchema> schema_v0,
schema_manager.ReadSchema(0));
+ const int32_t added_field_id = schema_v0->HighestFieldId() + 1;
+ std::shared_ptr<arrow::Field> added_field =
+ arrow::field("ltz_added", ltz(arrow::TimeUnit::MICRO));
+ std::vector<DataField> data_fields_v1 = schema_v0->Fields();
+ data_fields_v1.emplace_back(added_field_id, added_field);
+ ASSERT_OK(TestHelper::WriteNextSchema(dir->GetFileSystem(), table_path,
data_fields_v1,
+ added_field_id, options));
+ arrow::FieldVector fields_v1 = fields;
Review Comment:
Thanks! Extended `TestWriteAndReadInPinnedTimezone`: `ntz_to_ltz` changes
from `TIMESTAMP(6)` to `TIMESTAMP_LTZ(6)` and `ltz_to_ntz` the other way,
keeping their field ids. Files written before the change are read under `UTC`,
`Asia/Shanghai` and `Pacific/Honolulu`, for both append and primary-key tables.
The expected values follow Java's rule: `TimestampToTimestampCastRule`
converts with `TimeZone.getDefault()`, so a `TIMESTAMP` becomes the instant its
wall clock denotes in the default zone, and a `TIMESTAMP_LTZ` becomes the wall
clock it shows there. `ScanAndReadInteTest.TestCastTimestampType` already
checks the same conversion on Parquet and ORC files written by Java under
`Asia/Shanghai`. There is no Java-written Lance fixture with a type change; I
can add one if you'd like that as well.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
