This is an automated email from the ASF dual-hosted git repository.
lxy-9602 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/paimon-cpp.git
The following commit(s) were added to refs/heads/main by this push:
new b04d6e1c feat(lance): support TIMESTAMP_LTZ type (#479)
b04d6e1c is described below
commit b04d6e1cf8c89fb941a29aec96ab2dea4eb2647d
Author: Nicholas Jiang <[email protected]>
AuthorDate: Fri Oct 30 08:09:57 2026 +0700
feat(lance): support TIMESTAMP_LTZ type (#479)
---
docs/source/user_guide/data_types.rst | 5 +
src/paimon/core/schema/schema_validation.cpp | 11 +-
src/paimon/core/schema/schema_validation_test.cpp | 20 ++-
src/paimon/core/utils/nested_projection_utils.cpp | 95 ++++++++++++++
src/paimon/core/utils/nested_projection_utils.h | 10 ++
.../core/utils/nested_projection_utils_test.cpp | 88 +++++++++++++
.../format/lance/lance_file_batch_reader.cpp | 11 +-
src/paimon/format/lance/lance_file_format_test.cpp | 34 ++++-
src/paimon/format/lance/lance_format_writer.cpp | 35 +++++-
.../format/vortex/vortex_file_batch_reader.cpp | 65 +---------
test/inte/scan_and_read_inte_test.cpp | 67 ++++++++++
test/inte/write_and_read_inte_test.cpp | 138 ++++++++++++++++++++-
.../append_ltz_java_compat/README.md | 40 ++++++
...ta-4fcb4a78-f17b-4a9c-9454-ea37ee78e0c5-0.lance | Bin 0 -> 3094 bytes
...ta-e9ac3746-5001-45e0-8060-d0e84efae172-0.lance | Bin 0 -> 2555 bytes
...manifest-1e2d27b3-41db-46d5-9c0e-507c8ffe695a-0 | Bin 0 -> 2173 bytes
...manifest-fe3d8af9-46b4-4157-b164-7a0660316cf5-0 | Bin 0 -> 2172 bytes
...est-list-1a908b65-80a0-47d5-9dcd-81c70689d863-0 | Bin 0 -> 1158 bytes
...est-list-1a908b65-80a0-47d5-9dcd-81c70689d863-1 | Bin 0 -> 1267 bytes
...est-list-366b231f-4abf-408d-bd62-e7de9c6c1f0f-0 | Bin 0 -> 1267 bytes
...est-list-366b231f-4abf-408d-bd62-e7de9c6c1f0f-1 | Bin 0 -> 1267 bytes
.../append_ltz_java_compat/schema/schema-0 | 56 +++++++++
.../append_ltz_java_compat/snapshot/EARLIEST | 1 +
.../append_ltz_java_compat/snapshot/LATEST | 1 +
.../append_ltz_java_compat/snapshot/snapshot-1 | 18 +++
.../append_ltz_java_compat/snapshot/snapshot-2 | 18 +++
.../append_ltz_python_compat/README.md | 43 +++++++
...ta-3dd49f37-400b-4d87-ab60-dda8278083c8-0.lance | Bin 0 -> 2854 bytes
...ta-cb6cba6a-5d96-42f4-b8b9-2b5b4616c60f-0.lance | Bin 0 -> 2315 bytes
...manifest-2630e922-a399-4613-9181-54cc65803426-0 | Bin 0 -> 2208 bytes
...manifest-db163726-5626-4b91-a607-33f79a420829-0 | Bin 0 -> 2206 bytes
...est-list-92cdba02-3525-463d-981d-381562e9cb9d-0 | Bin 0 -> 897 bytes
...est-list-92cdba02-3525-463d-981d-381562e9cb9d-1 | Bin 0 -> 897 bytes
...est-list-df1a6587-9d9b-43e9-a73a-18245a5217b2-0 | Bin 0 -> 799 bytes
...est-list-df1a6587-9d9b-43e9-a73a-18245a5217b2-1 | Bin 0 -> 897 bytes
.../append_ltz_python_compat/schema/schema-0 | 71 +++++++++++
.../append_ltz_python_compat/snapshot/LATEST | 1 +
.../append_ltz_python_compat/snapshot/snapshot-1 | 15 +++
.../append_ltz_python_compat/snapshot/snapshot-2 | 15 +++
39 files changed, 775 insertions(+), 83 deletions(-)
diff --git a/docs/source/user_guide/data_types.rst
b/docs/source/user_guide/data_types.rst
index 92b1dda1..19de6ac2 100644
--- a/docs/source/user_guide/data_types.rst
+++ b/docs/source/user_guide/data_types.rst
@@ -194,6 +194,11 @@ and `Arrow DataTypes
<https://arrow.apache.org/docs/format/Columnar.html#data-ty
Paimon C++ supports precision values 0 (seconds), 3 (milliseconds), 6
(microseconds), and 9 (nanoseconds).
+ As in Paimon Java, Lance data files store this type as an Arrow
+ timestamp with the ``UTC`` time zone whatever the local time zone is,
+ and reads return it in the local time zone. Only the time zone of the
+ Arrow type changes; the values since the epoch are kept as they are.
+
* - ``ARRAY<t>``
- List
- Data type of an array of elements with same subtype.
diff --git a/src/paimon/core/schema/schema_validation.cpp
b/src/paimon/core/schema/schema_validation.cpp
index 7717b27f..63801ba9 100644
--- a/src/paimon/core/schema/schema_validation.cpp
+++ b/src/paimon/core/schema/schema_validation.cpp
@@ -1002,19 +1002,14 @@ Status SchemaValidation::ValidateLanceDataField(const
std::shared_ptr<arrow::Fie
case arrow::Type::DOUBLE:
case arrow::Type::DATE32:
case arrow::Type::TIME32:
+ // Lance cannot parse a time zone id containing ':', but
LanceFormatWriter, like Paimon
+ // Java, writes every zoned timestamp with the "UTC" time zone.
+ case arrow::Type::TIMESTAMP:
case arrow::Type::STRING:
case arrow::Type::BINARY:
case arrow::Type::DECIMAL128:
case arrow::Type::FIXED_SIZE_LIST:
return Status::OK();
- case arrow::Type::TIMESTAMP: {
- const auto& timestamp_type = checked_cast<const
arrow::TimestampType&>(*type);
- if (!timestamp_type.timezone().empty()) {
- return Status::Invalid(
- "Lance file format does not support type
LOCAL_ZONED_TIMESTAMP");
- }
- return Status::OK();
- }
case arrow::Type::LIST:
return ValidateLanceDataField(type->field(0));
case arrow::Type::STRUCT:
diff --git a/src/paimon/core/schema/schema_validation_test.cpp
b/src/paimon/core/schema/schema_validation_test.cpp
index 4f7b4116..2b9c6957 100644
--- a/src/paimon/core/schema/schema_validation_test.cpp
+++ b/src/paimon/core/schema/schema_validation_test.cpp
@@ -439,6 +439,14 @@ TEST(SchemaValidationTest, TestLanceDataTypes) {
arrow::field("date", arrow::date32()),
arrow::field("time", arrow::time32(arrow::TimeUnit::MILLI)),
arrow::field("timestamp", arrow::timestamp(arrow::TimeUnit::MICRO)),
+ arrow::field("ltz_0", arrow::timestamp(arrow::TimeUnit::SECOND,
"UTC")),
+ arrow::field("ltz_3", arrow::timestamp(arrow::TimeUnit::MILLI,
"Asia/Shanghai")),
+ arrow::field("ltz_6", arrow::timestamp(arrow::TimeUnit::MICRO,
"GMT-10:00")),
+ arrow::field("ltz_9", arrow::timestamp(arrow::TimeUnit::NANO,
"+08:00")),
+ arrow::field("ltz_array",
arrow::list(arrow::timestamp(arrow::TimeUnit::MICRO, "UTC"))),
+ arrow::field("nested_ltz",
+ arrow::struct_({arrow::field(
+ "values",
arrow::list(arrow::timestamp(arrow::TimeUnit::MILLI, "UTC")))})),
arrow::field("decimal", arrow::decimal128(38, 2)),
arrow::field("array", arrow::list(arrow::float32())),
arrow::field("nested_time",
@@ -474,11 +482,9 @@ TEST(SchemaValidationTest, TestLanceDataTypes) {
arrow::FieldVector unsupported_fields = {
arrow::field("map", arrow::map(arrow::int32(), arrow::utf8())),
- arrow::field("ltz", arrow::timestamp(arrow::TimeUnit::MICRO, "UTC")),
VariantTypeUtils::ToArrowField("variant"),
};
- std::vector<std::string> expected_errors = {"type MAP",
"LOCAL_ZONED_TIMESTAMP",
- "type VARIANT"};
+ std::vector<std::string> expected_errors = {"type MAP", "type VARIANT"};
for (size_t i = 0; i < unsupported_fields.size(); ++i) {
ASSERT_OK_AND_ASSIGN(
table_schema,
@@ -504,6 +510,14 @@ TEST(SchemaValidationTest, TestLanceDataTypes) {
{Options::CHANGELOG_FILE_FORMAT, "lance"}}) {
std::map<std::string, std::string> alternate_format_options = {
{Options::BUCKET, "-1"}, {Options::FILE_FORMAT, "parquet"},
{option_key, option_value}};
+ ASSERT_OK_AND_ASSIGN(
+ table_schema,
+ TableSchema::Create(
+ /*schema_id=*/0,
+ arrow::schema({arrow::field(
+ "ltz",
arrow::list(arrow::timestamp(arrow::TimeUnit::MICRO, "GMT-10:00")))}),
+ /*partition_keys=*/{}, /*primary_keys=*/{},
alternate_format_options));
+ ASSERT_OK(SchemaValidation::ValidateTableSchema(*table_schema));
ASSERT_OK_AND_ASSIGN(
table_schema,
TableSchema::Create(
diff --git a/src/paimon/core/utils/nested_projection_utils.cpp
b/src/paimon/core/utils/nested_projection_utils.cpp
index ff6368a7..ff069029 100644
--- a/src/paimon/core/utils/nested_projection_utils.cpp
+++ b/src/paimon/core/utils/nested_projection_utils.cpp
@@ -727,6 +727,92 @@ std::shared_ptr<arrow::DataType>
NormalizeLeafRepresentation(
}
return t;
}
+
+std::shared_ptr<arrow::ArrayData> AlignTimestampTimezonesData(
+ const std::shared_ptr<arrow::ArrayData>& data,
+ const std::shared_ptr<arrow::DataType>& read_type) {
+ const std::shared_ptr<arrow::DataType>& type = data->type;
+ if (type->id() == arrow::Type::TIMESTAMP && read_type->id() ==
arrow::Type::TIMESTAMP) {
+ const auto& stored = checked_cast<const arrow::TimestampType&>(*type);
+ const auto& wanted = checked_cast<const
arrow::TimestampType&>(*read_type);
+ if (stored.unit() != wanted.unit() || stored.timezone().empty() ||
+ wanted.timezone().empty() || stored.timezone() ==
wanted.timezone()) {
+ return data;
+ }
+ std::shared_ptr<arrow::ArrayData> retyped = data->Copy();
+ retyped->type = arrow::timestamp(wanted.unit(), wanted.timezone());
+ return retyped;
+ }
+ if (type->id() == arrow::Type::STRUCT && read_type->id() ==
arrow::Type::STRUCT) {
+ const auto& wanted_struct = checked_cast<const
arrow::StructType&>(*read_type);
+ arrow::FieldVector fields = type->fields();
+ std::shared_ptr<arrow::ArrayData> retyped;
+ for (int32_t i = 0; i < static_cast<int32_t>(data->child_data.size());
++i) {
+ const int32_t index =
wanted_struct.GetFieldIndex(fields[i]->name());
+ if (index < 0) {
+ continue;
+ }
+ std::shared_ptr<arrow::ArrayData> child =
AlignTimestampTimezonesData(
+ data->child_data[i], wanted_struct.field(index)->type());
+ if (child == data->child_data[i]) {
+ continue;
+ }
+ if (retyped == nullptr) {
+ retyped = data->Copy();
+ }
+ retyped->child_data[i] = child;
+ fields[i] = fields[i]->WithType(child->type);
+ }
+ if (retyped == nullptr) {
+ return data;
+ }
+ retyped->type = arrow::struct_(fields);
+ return retyped;
+ }
+ if ((type->id() == arrow::Type::LIST || type->id() ==
arrow::Type::FIXED_SIZE_LIST) &&
+ type->id() == read_type->id() && !data->child_data.empty()) {
+ std::shared_ptr<arrow::ArrayData> values =
+ AlignTimestampTimezonesData(data->child_data[0],
read_type->field(0)->type());
+ if (values == data->child_data[0]) {
+ return data;
+ }
+ std::shared_ptr<arrow::ArrayData> retyped = data->Copy();
+ retyped->child_data[0] = values;
+ const std::shared_ptr<arrow::Field> value_field =
type->field(0)->WithType(values->type);
+ retyped->type = type->id() == arrow::Type::LIST
+ ? arrow::list(value_field)
+ : arrow::fixed_size_list(
+ value_field,
+ checked_cast<const
arrow::FixedSizeListType&>(*type).list_size());
+ return retyped;
+ }
+ if (type->id() == arrow::Type::MAP && read_type->id() == arrow::Type::MAP
&&
+ !data->child_data.empty()) {
+ const auto& stored_map = checked_cast<const arrow::MapType&>(*type);
+ const auto& wanted_map = checked_cast<const
arrow::MapType&>(*read_type);
+ const std::shared_ptr<arrow::ArrayData>& entries = data->child_data[0];
+ std::shared_ptr<arrow::ArrayData> keys =
+ AlignTimestampTimezonesData(entries->child_data[0],
wanted_map.key_type());
+ std::shared_ptr<arrow::ArrayData> items =
+ AlignTimestampTimezonesData(entries->child_data[1],
wanted_map.item_type());
+ if (keys == entries->child_data[0] && items == entries->child_data[1])
{
+ return data;
+ }
+ const std::shared_ptr<arrow::Field> key_field =
+ stored_map.key_field()->WithType(keys->type);
+ const std::shared_ptr<arrow::Field> item_field =
+ stored_map.item_field()->WithType(items->type);
+ std::shared_ptr<arrow::ArrayData> retyped_entries = entries->Copy();
+ retyped_entries->type = arrow::struct_({key_field, item_field});
+ retyped_entries->child_data = {keys, items};
+ std::shared_ptr<arrow::ArrayData> retyped = data->Copy();
+ retyped->type =
+ std::make_shared<arrow::MapType>(key_field, item_field,
stored_map.keys_sorted());
+ retyped->child_data = {retyped_entries};
+ return retyped;
+ }
+ return data;
+}
} // namespace
Result<std::shared_ptr<arrow::Array>>
NestedProjectionUtils::AlignArrayToReadType(
@@ -840,4 +926,13 @@ Result<std::shared_ptr<arrow::Array>>
NestedProjectionUtils::AlignArrayToReadTyp
}
}
+std::shared_ptr<arrow::Array> NestedProjectionUtils::AlignTimestampTimezones(
+ const std::shared_ptr<arrow::Array>& array, const
std::shared_ptr<arrow::DataType>& read_type) {
+ std::shared_ptr<arrow::ArrayData> data =
AlignTimestampTimezonesData(array->data(), read_type);
+ if (data == array->data()) {
+ return array;
+ }
+ return arrow::MakeArray(data);
+}
+
} // namespace paimon
diff --git a/src/paimon/core/utils/nested_projection_utils.h
b/src/paimon/core/utils/nested_projection_utils.h
index 083e878e..b25aef5d 100644
--- a/src/paimon/core/utils/nested_projection_utils.h
+++ b/src/paimon/core/utils/nested_projection_utils.h
@@ -128,6 +128,16 @@ class PAIMON_EXPORT NestedProjectionUtils {
const std::shared_ptr<arrow::Array>& array,
const std::shared_ptr<arrow::DataType>& read_type, arrow::MemoryPool*
pool);
+ /// Relabel a TIMESTAMP in `array` with the zone `read_type` asks for,
only when both sides
+ /// have a zone and the same unit. A zoned value is an instant since the
epoch, so only the
+ /// type changes. STRUCT children are matched by name, and
LIST/FIXED_SIZE_LIST elements and
+ /// MAP keys and values are followed; every other difference, a unit
change included, is left
+ /// as is.
+ /// @return `array` itself when no TIMESTAMP is relabelled, otherwise a
shallow copy.
+ static std::shared_ptr<arrow::Array> AlignTimestampTimezones(
+ const std::shared_ptr<arrow::Array>& array,
+ const std::shared_ptr<arrow::DataType>& read_type);
+
private:
static Result<bool> HasNestedSubfieldProjectionType(
const std::shared_ptr<arrow::DataType>& file_type,
diff --git a/src/paimon/core/utils/nested_projection_utils_test.cpp
b/src/paimon/core/utils/nested_projection_utils_test.cpp
index e052c521..9ca69e91 100644
--- a/src/paimon/core/utils/nested_projection_utils_test.cpp
+++ b/src/paimon/core/utils/nested_projection_utils_test.cpp
@@ -520,6 +520,94 @@ TEST(NestedProjectionUtilsTest,
AlignArrayToReadTypeKeepsNestedLargeBinaryBlob)
ASSERT_TRUE(blob_out->Equals(*blob));
}
+TEST(NestedProjectionUtilsTest,
AlignTimestampTimezonesRelabelsZonedLeavesOfSameUnit) {
+ auto make_type = [](const std::string& zone, arrow::TimeUnit::type
other_unit) {
+ return arrow::struct_({
+ arrow::field("ltz", arrow::timestamp(arrow::TimeUnit::MICRO,
zone)),
+ arrow::field("row", arrow::struct_({arrow::field(
+ "ltz",
arrow::timestamp(arrow::TimeUnit::MILLI, zone))})),
+ arrow::field("list",
arrow::list(arrow::timestamp(arrow::TimeUnit::MICRO, zone))),
+ arrow::field("fixed_list",
+
arrow::fixed_size_list(arrow::timestamp(arrow::TimeUnit::MICRO, zone), 2)),
+ arrow::field("map",
arrow::map(arrow::timestamp(arrow::TimeUnit::MILLI, zone),
+
arrow::timestamp(arrow::TimeUnit::MICRO, zone))),
+ arrow::field("other_unit", arrow::timestamp(other_unit, zone)),
+ arrow::field("ntz", arrow::timestamp(arrow::TimeUnit::MICRO)),
+ });
+ };
+ const std::string json = R"([[1, [2], [3, null], [16, null], [[11, 12],
[13, null]], 4, 5],
+ [null, [null], null, null, null, null, null],
+ [6, [7], [8], [17, 18], [[14, 15]], 9, 10]])";
+ std::shared_ptr<arrow::DataType> data_type = make_type("UTC",
arrow::TimeUnit::MICRO);
+ std::shared_ptr<arrow::Array> array =
+ arrow::ipc::internal::json::ArrayFromJSON(data_type,
json).ValueOrDie();
+ ASSERT_EQ(NestedProjectionUtils::AlignTimestampTimezones(array,
array->type()).get(),
+ array.get());
+
+ std::shared_ptr<arrow::DataType> read_type = make_type("Asia/Shanghai",
arrow::TimeUnit::MILLI);
+ std::shared_ptr<arrow::Array> aligned =
+ NestedProjectionUtils::AlignTimestampTimezones(array, read_type);
+ auto expected_type = arrow::struct_({
+ arrow::field("ltz", arrow::timestamp(arrow::TimeUnit::MICRO,
"Asia/Shanghai")),
+ arrow::field("row",
+ arrow::struct_({arrow::field(
+ "ltz", arrow::timestamp(arrow::TimeUnit::MILLI,
"Asia/Shanghai"))})),
+ arrow::field("list",
+ arrow::list(arrow::timestamp(arrow::TimeUnit::MICRO,
"Asia/Shanghai"))),
+ arrow::field(
+ "fixed_list",
+ arrow::fixed_size_list(arrow::timestamp(arrow::TimeUnit::MICRO,
"Asia/Shanghai"), 2)),
+ arrow::field("map",
arrow::map(arrow::timestamp(arrow::TimeUnit::MILLI, "Asia/Shanghai"),
+
arrow::timestamp(arrow::TimeUnit::MICRO, "Asia/Shanghai"))),
+ arrow::field("other_unit", arrow::timestamp(arrow::TimeUnit::MICRO,
"UTC")),
+ arrow::field("ntz", arrow::timestamp(arrow::TimeUnit::MICRO)),
+ });
+ std::shared_ptr<arrow::Array> expected =
+ arrow::ipc::internal::json::ArrayFromJSON(expected_type,
json).ValueOrDie();
+ ASSERT_TRUE(aligned->type()->Equals(expected_type)) <<
aligned->type()->ToString();
+ ASSERT_TRUE(aligned->Equals(*expected));
+ ASSERT_TRUE(array->type()->Equals(data_type)) << array->type()->ToString();
+
+ std::shared_ptr<arrow::Array> aligned_slice =
+ NestedProjectionUtils::AlignTimestampTimezones(array->Slice(1),
read_type);
+ ASSERT_TRUE(aligned_slice->Equals(*expected->Slice(1)));
+
+ std::shared_ptr<arrow::Array> zoned =
+
arrow::ipc::internal::json::ArrayFromJSON(arrow::timestamp(arrow::TimeUnit::MICRO,
"UTC"),
+ "[1, null]")
+ .ValueOrDie();
+ ASSERT_EQ(NestedProjectionUtils::AlignTimestampTimezones(
+ zoned, arrow::timestamp(arrow::TimeUnit::MICRO))
+ .get(),
+ zoned.get());
+ std::shared_ptr<arrow::Array> unzoned =
+
arrow::ipc::internal::json::ArrayFromJSON(arrow::timestamp(arrow::TimeUnit::MICRO),
+ "[1, null]")
+ .ValueOrDie();
+ ASSERT_EQ(NestedProjectionUtils::AlignTimestampTimezones(
+ unzoned, arrow::timestamp(arrow::TimeUnit::MICRO, "UTC"))
+ .get(),
+ unzoned.get());
+}
+
+TEST(NestedProjectionUtilsTest,
AlignTimestampTimezonesMatchesStructChildrenByName) {
+ auto utc = arrow::timestamp(arrow::TimeUnit::MICRO, "UTC");
+ auto shanghai = arrow::timestamp(arrow::TimeUnit::MICRO, "Asia/Shanghai");
+ const std::string json = "[[1, 2], [null, null]]";
+ std::shared_ptr<arrow::Array> array =
+ arrow::ipc::internal::json::ArrayFromJSON(
+ arrow::struct_({arrow::field("unread", utc), arrow::field("read",
utc)}), json)
+ .ValueOrDie();
+ std::shared_ptr<arrow::Array> aligned =
NestedProjectionUtils::AlignTimestampTimezones(
+ array, arrow::struct_({arrow::field("read", shanghai)}));
+ auto expected_type =
+ arrow::struct_({arrow::field("unread", utc), arrow::field("read",
shanghai)});
+ std::shared_ptr<arrow::Array> expected =
+ arrow::ipc::internal::json::ArrayFromJSON(expected_type,
json).ValueOrDie();
+ ASSERT_TRUE(aligned->type()->Equals(expected_type)) <<
aligned->type()->ToString();
+ ASSERT_TRUE(aligned->Equals(*expected));
+}
+
TEST(NestedProjectionUtilsTest, HasNestedSubfieldProjectionNoProjection) {
auto file_schema = arrow::schema({
MakeField("f0", arrow::int32(), 1),
diff --git a/src/paimon/format/lance/lance_file_batch_reader.cpp
b/src/paimon/format/lance/lance_file_batch_reader.cpp
index b1e0163e..c2606afa 100644
--- a/src/paimon/format/lance/lance_file_batch_reader.cpp
+++ b/src/paimon/format/lance/lance_file_batch_reader.cpp
@@ -128,9 +128,14 @@ Result<BatchReader::ReadBatch>
LanceFileBatchReader::AlignBatch(ReadBatch batch)
if (array->type_id() != arrow::Type::STRUCT) {
return Status::Invalid("Lance reader returned a non-struct Arrow
batch");
}
- PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<arrow::Array> aligned,
- NestedProjectionUtils::AlignArrayToReadType(
- array, arrow::struct_(read_schema_->fields()),
arrow_pool_.get()));
+ std::shared_ptr<arrow::DataType> read_type =
arrow::struct_(read_schema_->fields());
+ // Zoned timestamps come back as UTC, as LanceFormatWriter and Paimon Java
store them, while
+ // paimon asks for the local zone; the instants are the same.
+ std::shared_ptr<arrow::Array> relabelled =
+ NestedProjectionUtils::AlignTimestampTimezones(array, read_type);
+ PAIMON_ASSIGN_OR_RAISE(
+ std::shared_ptr<arrow::Array> aligned,
+ NestedProjectionUtils::AlignArrayToReadType(relabelled, read_type,
arrow_pool_.get()));
auto out_array = std::make_unique<::ArrowArray>();
auto out_schema = std::make_unique<::ArrowSchema>();
PAIMON_RETURN_NOT_OK_FROM_ARROW(
diff --git a/src/paimon/format/lance/lance_file_format_test.cpp
b/src/paimon/format/lance/lance_file_format_test.cpp
index 0805d49e..cdee8831 100644
--- a/src/paimon/format/lance/lance_file_format_test.cpp
+++ b/src/paimon/format/lance/lance_file_format_test.cpp
@@ -182,6 +182,19 @@ TEST_F(LanceFileFormatTest, WriteThenReadSupportedTypes) {
name, arrow::time32(arrow::TimeUnit::MILLI), /*nullable=*/true,
arrow::key_value_metadata({DataType::kTimePrecision},
{std::to_string(precision)}));
};
+ auto ltz_fields = [](const std::string& zone) {
+ return arrow::FieldVector{
+ arrow::field("ltz_0_col",
arrow::timestamp(arrow::TimeUnit::SECOND, zone)),
+ arrow::field("ltz_3_col", arrow::timestamp(arrow::TimeUnit::MILLI,
zone)),
+ arrow::field("ltz_6_col", arrow::timestamp(arrow::TimeUnit::MICRO,
zone)),
+ arrow::field("ltz_9_col", arrow::timestamp(arrow::TimeUnit::NANO,
zone)),
+ arrow::field("ltz_array_col",
+ arrow::list(arrow::timestamp(arrow::TimeUnit::MICRO,
zone))),
+ arrow::field("ltz_row_col",
+ arrow::struct_({arrow::field(
+ "value", arrow::timestamp(arrow::TimeUnit::MILLI,
zone))})),
+ };
+ };
arrow::FieldVector fields = {
arrow::field("bool_col", arrow::boolean()),
arrow::field("i8_col", arrow::int8()),
@@ -206,17 +219,21 @@ TEST_F(LanceFileFormatTest, WriteThenReadSupportedTypes) {
arrow::field("time_row_array_col",
arrow::struct_({arrow::field("values",
arrow::list(time_field("item", 9)))})),
};
+ arrow::FieldVector local_ltz_fields = ltz_fields("GMT-10:00");
+ fields.insert(fields.end(), local_ltz_fields.begin(),
local_ltz_fields.end());
std::shared_ptr<arrow::Schema> schema = arrow::schema(fields);
std::shared_ptr<arrow::Array> expected =
arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([
[true,-1,-2,-3,-4,1.5,2.5,"one","abc",1,0,
"1970-01-01 00:00:00.000001","12.34",[1.0,2.0],[7],[3.0,4.0],
- [0,null,45296000],[45296123],[[3723123,null]]],
+ [0,null,45296000],[45296123],[[3723123,null]],
+ 1700000000,1700000000123,1700000000123456,1700000000123456789,
+ [1700000000123456,null,-1],[1700000000123]],
[false,1,2,3,4,3.5,4.5,"two","xyz",2,86399999,
"2030-12-31 23:59:59.999999","-12.34",[],[8],[5.0,6.0],
- [],[86399999],[[]]],
+ [],[86399999],[[]],-1,-1,-1,-1,[],[-1]],
[null,null,null,null,null,null,null,null,null,null,null,null,null,null,[null],null,
- null,[null],[null]]
+ null,[null],[null],null,null,null,null,null,[null]]
])")
.ValueOrDie();
std::string path = PathUtil::JoinPath(directory_->Str(),
"supported-types.lance");
@@ -246,6 +263,17 @@ TEST_F(LanceFileFormatTest, WriteThenReadSupportedTypes) {
std::shared_ptr<arrow::Array> batch_array =
std::move(batch_result).MoveValueUnsafe();
ASSERT_TRUE(batch_array->type()->Equals(arrow::struct_(fields),
/*check_metadata=*/true))
<< batch_array->type()->ToString(/*show_metadata=*/true);
+
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<::ArrowSchema> c_file_schema,
reader->GetFileSchema());
+ arrow::Result<std::shared_ptr<arrow::Schema>> file_schema =
+ arrow::ImportSchema(c_file_schema.get());
+ ASSERT_TRUE(file_schema.ok()) << file_schema.status().ToString();
+ for (const std::shared_ptr<arrow::Field>& utc_field : ltz_fields("UTC")) {
+ std::shared_ptr<arrow::Field> file_field =
+ file_schema.ValueUnsafe()->GetFieldByName(utc_field->name());
+ ASSERT_NE(file_field, nullptr) << utc_field->name();
+ ASSERT_TRUE(file_field->type()->Equals(utc_field->type())) <<
file_field->ToString();
+ }
}
TEST_F(LanceFileFormatTest, RejectsNullTopLevelRows) {
diff --git a/src/paimon/format/lance/lance_format_writer.cpp
b/src/paimon/format/lance/lance_format_writer.cpp
index 6fb0c163..0e7776c4 100644
--- a/src/paimon/format/lance/lance_format_writer.cpp
+++ b/src/paimon/format/lance/lance_format_writer.cpp
@@ -26,6 +26,7 @@
#include "arrow/array/array_nested.h"
#include "arrow/c/bridge.h"
#include "arrow/c/helpers.h"
+#include "arrow/type.h"
#include "paimon/common/metrics/metrics_impl.h"
#include "paimon/common/utils/arrow/arrow_utils.h"
#include "paimon/common/utils/arrow/status_utils.h"
@@ -64,12 +65,44 @@ Status ValidateRowValues(const
std::shared_ptr<arrow::Array>& array) {
return Status::OK();
}
+// Lance 0.39 stores a zoned timestamp as `timestamp:<unit>:<zone>` and cannot
parse a zone id
+// containing ':' (e.g. GMT-10:00), so store every zone as UTC, as Paimon Java
does. The values
+// are instants since the epoch whatever the zone, so only the type changes.
+std::shared_ptr<arrow::DataType> ToLanceType(const
std::shared_ptr<arrow::DataType>& type) {
+ switch (type->id()) {
+ case arrow::Type::TIMESTAMP: {
+ const auto& timestamp_type = checked_cast<const
arrow::TimestampType&>(*type);
+ if (timestamp_type.timezone().empty()) {
+ return type;
+ }
+ return arrow::timestamp(timestamp_type.unit(), "UTC");
+ }
+ case arrow::Type::LIST:
+ return
arrow::list(type->field(0)->WithType(ToLanceType(type->field(0)->type())));
+ case arrow::Type::STRUCT: {
+ arrow::FieldVector fields;
+ fields.reserve(type->num_fields());
+ for (const std::shared_ptr<arrow::Field>& field : type->fields()) {
+ fields.push_back(field->WithType(ToLanceType(field->type())));
+ }
+ return arrow::struct_(fields);
+ }
+ default:
+ return type;
+ }
+}
+
+std::shared_ptr<arrow::Schema> ToLanceSchema(const
std::shared_ptr<arrow::Schema>& schema) {
+ return
arrow::schema(ToLanceType(arrow::struct_(schema->fields()))->fields(),
+ schema->metadata());
+}
+
} // namespace
LanceFormatWriter::LanceFormatWriter(const std::shared_ptr<arrow::Schema>&
schema,
const std::shared_ptr<arrow::MemoryPool>&
arrow_pool,
PaimonLanceWriter* writer)
- : schema_(schema),
+ : schema_(ToLanceSchema(schema)),
arrow_pool_(arrow_pool),
writer_(writer),
metrics_(std::make_shared<MetricsImpl>()) {}
diff --git a/src/paimon/format/vortex/vortex_file_batch_reader.cpp
b/src/paimon/format/vortex/vortex_file_batch_reader.cpp
index 6fa537b9..e171589e 100644
--- a/src/paimon/format/vortex/vortex_file_batch_reader.cpp
+++ b/src/paimon/format/vortex/vortex_file_batch_reader.cpp
@@ -168,68 +168,6 @@ Result<std::shared_ptr<arrow::Array>>
NormalizeViewArray(const std::shared_ptr<a
}
}
-// Vortex reports the file's zone and paimon asks for the reader's; the
instants are the same.
-std::shared_ptr<arrow::ArrayData> AlignTimestampTimezones(
- const std::shared_ptr<arrow::ArrayData>& data,
- const std::shared_ptr<arrow::DataType>& read_type) {
- const std::shared_ptr<arrow::DataType>& type = data->type;
- if (type->id() == arrow::Type::TIMESTAMP && read_type->id() ==
arrow::Type::TIMESTAMP) {
- const auto& stored = checked_cast<const arrow::TimestampType&>(*type);
- const auto& wanted = checked_cast<const
arrow::TimestampType&>(*read_type);
- if (stored.unit() != wanted.unit() || stored.timezone().empty() ||
- wanted.timezone().empty() || stored.timezone() ==
wanted.timezone()) {
- return data;
- }
- std::shared_ptr<arrow::ArrayData> retyped = data->Copy();
- retyped->type = arrow::timestamp(wanted.unit(), wanted.timezone());
- return retyped;
- }
- if (type->id() == arrow::Type::STRUCT && read_type->id() ==
arrow::Type::STRUCT) {
- const auto& wanted_struct = checked_cast<const
arrow::StructType&>(*read_type);
- arrow::FieldVector fields = type->fields();
- std::shared_ptr<arrow::ArrayData> retyped;
- for (int32_t i = 0; i < static_cast<int32_t>(data->child_data.size());
++i) {
- const int32_t index =
wanted_struct.GetFieldIndex(fields[i]->name());
- if (index < 0) {
- continue;
- }
- std::shared_ptr<arrow::ArrayData> child =
- AlignTimestampTimezones(data->child_data[i],
wanted_struct.field(index)->type());
- if (child == data->child_data[i]) {
- continue;
- }
- if (retyped == nullptr) {
- retyped = data->Copy();
- }
- retyped->child_data[i] = child;
- fields[i] = fields[i]->WithType(child->type);
- }
- if (retyped == nullptr) {
- return data;
- }
- retyped->type = arrow::struct_(fields);
- return retyped;
- }
- if ((type->id() == arrow::Type::LIST || type->id() ==
arrow::Type::FIXED_SIZE_LIST) &&
- type->id() == read_type->id() && !data->child_data.empty()) {
- std::shared_ptr<arrow::ArrayData> values =
- AlignTimestampTimezones(data->child_data[0],
read_type->field(0)->type());
- if (values == data->child_data[0]) {
- return data;
- }
- std::shared_ptr<arrow::ArrayData> retyped = data->Copy();
- retyped->child_data[0] = values;
- const std::shared_ptr<arrow::Field> value_field =
type->field(0)->WithType(values->type);
- retyped->type = type->id() == arrow::Type::LIST
- ? arrow::list(value_field)
- : arrow::fixed_size_list(
- value_field,
- checked_cast<const
arrow::FixedSizeListType&>(*type).list_size());
- return retyped;
- }
- return data;
-}
-
} // namespace
VortexFileBatchReader::VortexFileBatchReader(
@@ -395,8 +333,9 @@ Result<std::shared_ptr<arrow::Array>>
VortexFileBatchReader::AlignToReadSchema(
}
// TODO(zhouyc-ali): push top-level column pruning into the Vortex scan
std::shared_ptr<arrow::DataType> read_type =
arrow::struct_(read_schema_->fields());
+ // Vortex reports the file's zone and paimon asks for the reader's; the
instants are the same.
return NestedProjectionUtils::AlignArrayToReadType(
- arrow::MakeArray(AlignTimestampTimezones(array->data(), read_type)),
read_type,
+ NestedProjectionUtils::AlignTimestampTimezones(array, read_type),
read_type,
arrow_pool_.get());
}
diff --git a/test/inte/scan_and_read_inte_test.cpp
b/test/inte/scan_and_read_inte_test.cpp
index 3bfb58c7..17807e25 100644
--- a/test/inte/scan_and_read_inte_test.cpp
+++ b/test/inte/scan_and_read_inte_test.cpp
@@ -3637,6 +3637,73 @@ TEST_F(ScanAndReadInteTest,
TestLancePythonCompatibility) {
check(1, {"f_struct", "id", "f_vector", "f_array_int"},
projected->Slice(0, 2));
check(2, {"f_struct", "id", "f_vector", "f_array_int"}, projected);
}
+
+TEST_F(ScanAndReadInteTest, TestLanceLtzCompatibility) {
+ TimezoneGuard timezone_guard("Asia/Shanghai");
+ const std::string zone = DateTimeUtils::GetLocalTimezoneName();
+ arrow::FieldVector fields = {
+ arrow::field("_VALUE_KIND", arrow::int8()),
+ arrow::field("id", arrow::int32()),
+ arrow::field("f_ltz_0", arrow::timestamp(arrow::TimeUnit::SECOND,
zone)),
+ arrow::field("f_ltz_3", arrow::timestamp(arrow::TimeUnit::MILLI,
zone)),
+ arrow::field("f_ltz_6", arrow::timestamp(arrow::TimeUnit::MICRO,
zone)),
+ arrow::field("f_ltz_9", arrow::timestamp(arrow::TimeUnit::NANO, zone)),
+ arrow::field("f_ltz_array",
arrow::list(arrow::timestamp(arrow::TimeUnit::MICRO, zone))),
+ arrow::field(
+ "f_ltz_struct",
+ arrow::struct_({arrow::field("number", arrow::int32()),
+ arrow::field("ts",
arrow::timestamp(arrow::TimeUnit::MILLI, zone))})),
+ };
+ auto expected = std::make_shared<arrow::ChunkedArray>(
+ arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([
+[0, 1, "1970-01-01 00:00:01", "1970-01-01 00:00:01.123", "1970-01-01
00:00:01.123456", "1970-01-01 00:00:01.123456789", ["1970-01-01
00:00:01.123456", null, "1969-12-31 23:59:59.999999"], [1, "1970-01-01
00:00:01.123"]],
+[0, 2, null, null, null, null, null, [null, null]],
+[0, 10, "1969-12-31 00:00:00", "1969-12-31 23:59:59.999", "1969-12-31
23:59:59.999999", "1969-12-31 23:59:59.999999999", [], [10, "1969-12-31
23:59:59.999"]],
+[0, 11, "2024-10-04 12:34:56", "2024-10-04 12:34:56.123", "2024-10-04
12:34:56.123456", "2024-10-04 12:34:56.123456789", ["2024-10-04
12:34:56.123456"], [11, "2024-10-04 12:34:56.123"]]
+])")
+ .ValueOrDie());
+ auto projected = std::make_shared<arrow::ChunkedArray>(
+ arrow::ipc::internal::json::ArrayFromJSON(
+ arrow::struct_({fields[0], fields[7], fields[1], fields[5],
fields[6]}), R"([
+[0, [1, "1970-01-01 00:00:01.123"], 1, "1970-01-01 00:00:01.123456789",
["1970-01-01 00:00:01.123456", null, "1969-12-31 23:59:59.999999"]],
+[0, [null, null], 2, null, null],
+[0, [10, "1969-12-31 23:59:59.999"], 10, "1969-12-31 23:59:59.999999999", []],
+[0, [11, "2024-10-04 12:34:56.123"], 11, "2024-10-04 12:34:56.123456789",
["2024-10-04 12:34:56.123456"]]
+])")
+ .ValueOrDie());
+ auto check = [&](const std::string& table_path, int64_t snapshot_id,
+ const std::vector<std::string>& projection,
+ const std::shared_ptr<arrow::ChunkedArray>&
expected_result) {
+ SCOPED_TRACE("snapshot " + std::to_string(snapshot_id));
+ ScanContextBuilder scan_builder(table_path);
+ scan_builder.AddOption(Options::SCAN_SNAPSHOT_ID,
std::to_string(snapshot_id));
+ ASSERT_OK_AND_ASSIGN(auto scan_context, scan_builder.Finish());
+ ASSERT_OK_AND_ASSIGN(auto scan,
TableScan::Create(std::move(scan_context)));
+ ASSERT_OK_AND_ASSIGN(auto plan, scan->CreatePlan());
+ ASSERT_EQ(plan->SnapshotId(), std::optional<int64_t>(snapshot_id));
+ ASSERT_FALSE(plan->Splits().empty());
+
+ ReadContextBuilder read_builder(table_path);
+ read_builder.AddOption("read.batch-size", "1");
+ if (!projection.empty()) {
+ read_builder.SetReadFieldNames(projection);
+ }
+ ASSERT_OK_AND_ASSIGN(auto read_context, read_builder.Finish());
+ ASSERT_OK_AND_ASSIGN(auto read,
TableRead::Create(std::move(read_context)));
+ ASSERT_OK_AND_ASSIGN(auto reader, read->CreateReader(plan->Splits()));
+ ASSERT_OK_AND_ASSIGN(auto actual,
ReadResultCollector::CollectResult(std::move(reader)));
+ ASSERT_TRUE(actual);
+ ASSERT_TRUE(expected_result->Equals(actual))
+ << "actual: " << actual->ToString() << "\nexpected: " <<
expected_result->ToString();
+ };
+ for (const std::string table_name : {"append_ltz_java_compat",
"append_ltz_python_compat"}) {
+ SCOPED_TRACE(table_name);
+ std::string table_path = GetDataDir() + "/lance/" + table_name +
".db/" + table_name;
+ check(table_path, 1, /*projection=*/{}, expected->Slice(0, 2));
+ check(table_path, 2, /*projection=*/{}, expected);
+ check(table_path, 2, {"f_ltz_struct", "id", "f_ltz_9", "f_ltz_array"},
projected);
+ }
+}
#endif
#ifdef PAIMON_ENABLE_MOSAIC
diff --git a/test/inte/write_and_read_inte_test.cpp
b/test/inte/write_and_read_inte_test.cpp
index 8b36d3c7..dbbdf63f 100644
--- a/test/inte/write_and_read_inte_test.cpp
+++ b/test/inte/write_and_read_inte_test.cpp
@@ -22,6 +22,7 @@
#include <memory>
#include <optional>
#include <string>
+#include <tuple>
#include <utility>
#include <vector>
@@ -2589,7 +2590,7 @@ TEST_P(WriteAndReadInteTest, TestAppendTimestampType) {
};
auto schema = arrow::schema(fields);
auto [file_format, file_system] = GetParam();
- if (file_format == "mosaic" || file_format == "lance") {
+ if (file_format == "mosaic") {
return;
}
std::map<std::string, std::string> options = {{Options::FILE_FORMAT,
file_format},
@@ -2646,7 +2647,7 @@ TEST_P(WriteAndReadInteTest, TestPkTimestampType) {
};
auto schema = arrow::schema(fields);
auto [file_format, file_system] = GetParam();
- if (file_format == "mosaic" || file_format == "lance") {
+ if (file_format == "mosaic") {
return;
}
std::map<std::string, std::string> options = {{Options::FILE_FORMAT,
file_format},
@@ -6027,6 +6028,139 @@ TEST(TimeColumnWriteInteTest,
TestAppendTimeColumnFinalize) {
}
#endif
+#ifdef PAIMON_ENABLE_LANCE
+class LanceTimestampLtzInteTest
+ : public ::testing::Test,
+ public ::testing::WithParamInterface<std::tuple<std::string, bool>> {};
+
+TEST_P(LanceTimestampLtzInteTest, TestWriteAndReadInPinnedTimezone) {
+ TimezoneGuard timezone_guard(std::get<0>(GetParam()));
+ const std::string timezone = DateTimeUtils::GetLocalTimezoneName();
+ const bool with_primary_key = std::get<1>(GetParam());
+ auto dir = UniqueTestDirectory::Create("local");
+ ASSERT_NE(dir, nullptr);
+
+ auto ltz = [&timezone](arrow::TimeUnit::type unit) { return
arrow::timestamp(unit, timezone); };
+ arrow::FieldVector fields = {
+ arrow::field("id", arrow::int32()),
+ arrow::field("ltz_0", ltz(arrow::TimeUnit::SECOND)),
+ arrow::field("ltz_9", ltz(arrow::TimeUnit::NANO)),
+ arrow::field("ltz_array", arrow::list(ltz(arrow::TimeUnit::MICRO))),
+ arrow::field(
+ "ltz_row",
+ arrow::struct_({arrow::field("ltz_3", ltz(arrow::TimeUnit::MILLI)),
+ arrow::field("ntz",
arrow::timestamp(arrow::TimeUnit::MICRO))})),
+ arrow::field("ntz_to_ltz", arrow::timestamp(arrow::TimeUnit::MICRO)),
+ arrow::field("ltz_to_ntz", ltz(arrow::TimeUnit::MICRO)),
+ };
+ std::map<std::string, std::string> options = {{Options::FILE_FORMAT,
"lance"},
+ {Options::BUCKET,
with_primary_key ? "1" : "-1"},
+ {Options::FILE_SYSTEM,
"local"}};
+ std::vector<std::string> primary_keys;
+ if (with_primary_key) {
+ primary_keys.push_back("id");
+ }
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<TestHelper> helper,
+ TestHelper::Create(dir->Str(), arrow::schema(fields),
+ /*partition_keys=*/{},
primary_keys, options,
+ /*is_streaming_mode=*/false));
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch_v0,
+ TestHelper::MakeRecordBatch(arrow::struct_(fields),
R"([
+ [1, "1969-12-31 23:59:59", "1969-12-31 23:59:59.999999999",
+ ["2024-10-04 12:34:56.123456", null],
+ ["1970-01-01 00:00:00.001", "1970-01-01 00:00:00.000001"],
+ "1969-12-31 23:59:59.999999", "1970-01-01 00:00:00.000001"],
+ [2, null, null, null, [null, null], null, null],
+ [3, "2024-10-04 12:34:56", "2024-10-04 12:34:56.123456789", [],
+ ["2024-10-04 12:34:56.123", null], "2024-10-04 12:34:56.123456",
+ "2024-10-04 12:34:56.123456"]
+ ])",
+ /*partition_map=*/{},
/*bucket=*/0, {}));
+ ASSERT_OK(helper->WriteAndCommit(std::move(batch_v0),
/*commit_identifier=*/0,
+
/*expected_commit_messages=*/std::nullopt));
+
+ const std::string table_path = PathUtil::JoinPath(dir->Str(),
"foo.db/bar");
+ SchemaManager schema_manager(dir->GetFileSystem(), table_path);
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<TableSchema> schema_v0,
schema_manager.ReadSchema(0));
+ const int32_t added_field_id = schema_v0->HighestFieldId() + 1;
+ std::shared_ptr<arrow::Field> added_field =
+ arrow::field("ltz_added", ltz(arrow::TimeUnit::MICRO));
+ arrow::FieldVector fields_v1 = fields;
+ fields_v1[5] = fields[5]->WithType(ltz(arrow::TimeUnit::MICRO));
+ fields_v1[6] =
fields[6]->WithType(arrow::timestamp(arrow::TimeUnit::MICRO));
+ fields_v1.push_back(added_field);
+ std::vector<DataField> data_fields_v1 = schema_v0->Fields();
+ data_fields_v1[5] = DataField(data_fields_v1[5].Id(), fields_v1[5]);
+ data_fields_v1[6] = DataField(data_fields_v1[6].Id(), fields_v1[6]);
+ data_fields_v1.emplace_back(added_field_id, added_field);
+ ASSERT_OK(TestHelper::WriteNextSchema(dir->GetFileSystem(), table_path,
data_fields_v1,
+ added_field_id, options));
+ helper.reset();
+ ASSERT_OK_AND_ASSIGN(helper,
+ TestHelper::Create(table_path, options,
/*is_streaming_mode=*/false));
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<RecordBatch> batch_v1,
+
TestHelper::MakeRecordBatch(arrow::struct_(fields_v1), R"([
+ [4, "1970-01-01 00:00:00", null, [null], [null, "1969-12-31
23:59:59.999999"],
+ "1969-12-31 23:59:59.999999", "1970-01-01 00:00:00.000001",
+ "2024-10-04 12:34:56.123456"]
+ ])",
+ /*partition_map=*/{},
/*bucket=*/0, {}));
+ ASSERT_OK(helper->WriteAndCommit(std::move(batch_v1),
/*commit_identifier=*/1,
+
/*expected_commit_messages=*/std::nullopt));
+
+ ASSERT_OK_AND_ASSIGN(std::vector<std::shared_ptr<Split>> splits,
+ helper->NewScan(StartupMode::LatestFull(),
/*snapshot_id=*/std::nullopt));
+ ReadContextBuilder read_context_builder(table_path);
+ read_context_builder.SetOptions(options).SetReadFieldNames(
+ {"ltz_row", "ltz_added", "ltz_9", "id", "ltz_array", "ltz_0",
"ltz_to_ntz", "ntz_to_ltz"});
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<ReadContext> read_context,
read_context_builder.Finish());
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<TableRead> table_read,
+ TableRead::Create(std::move(read_context)));
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<BatchReader> batch_reader,
+ table_read->CreateReader(splits));
+ ASSERT_OK_AND_ASSIGN(std::shared_ptr<arrow::ChunkedArray> actual,
+
ReadResultCollector::CollectResult(std::move(batch_reader)));
+ const std::map<std::string, std::vector<std::string>> converted_values = {
+ {"UTC",
+ {"1970-01-01 00:00:00.000001", "1969-12-31 23:59:59.999999",
"2024-10-04 12:34:56.123456",
+ "2024-10-04 12:34:56.123456"}},
+ {"Asia/Shanghai",
+ {"1970-01-01 08:00:00.000001", "1969-12-31 15:59:59.999999",
"2024-10-04 20:34:56.123456",
+ "2024-10-04 04:34:56.123456"}},
+ {"Pacific/Honolulu",
+ {"1969-12-31 14:00:00.000001", "1970-01-01 09:59:59.999999",
"2024-10-04 02:34:56.123456",
+ "2024-10-04 22:34:56.123456"}},
+ };
+ const std::vector<std::string>& converted =
converted_values.at(std::get<0>(GetParam()));
+ auto expected_type = arrow::struct_({arrow::field("_VALUE_KIND",
arrow::int8()), fields_v1[4],
+ fields_v1[7], fields_v1[2],
fields_v1[0], fields_v1[3],
+ fields_v1[1], fields_v1[6],
fields_v1[5]});
+ auto expected = std::make_shared<arrow::ChunkedArray>(
+ arrow::ipc::internal::json::ArrayFromJSON(
+ expected_type, fmt::format(R"([
+ [0, ["1970-01-01 00:00:00.001", "1970-01-01 00:00:00.000001"],
null,
+ "1969-12-31 23:59:59.999999999", 1, ["2024-10-04
12:34:56.123456", null],
+ "1969-12-31 23:59:59", "{}", "{}"],
+ [0, [null, null], null, null, 2, null, null, null, null],
+ [0, ["2024-10-04 12:34:56.123", null], null, "2024-10-04
12:34:56.123456789", 3, [],
+ "2024-10-04 12:34:56", "{}", "{}"],
+ [0, [null, "1969-12-31 23:59:59.999999"], "2024-10-04
12:34:56.123456", null, 4,
+ [null], "1970-01-01 00:00:00", "1970-01-01 00:00:00.000001",
+ "1969-12-31 23:59:59.999999"]
+ ])",
+ converted[0], converted[1],
converted[2], converted[3]))
+ .ValueOrDie());
+ ASSERT_TRUE(expected->Equals(actual))
+ << "actual: " << actual->ToString() << "\nexpected: " <<
expected->ToString();
+}
+
+INSTANTIATE_TEST_SUITE_P(TimezoneAndPrimaryKey, LanceTimestampLtzInteTest,
+
::testing::Combine(::testing::Values(std::string("UTC"),
+
std::string("Asia/Shanghai"),
+
std::string("Pacific/Honolulu")),
+ ::testing::Bool()));
+#endif
+
INSTANTIATE_TEST_SUITE_P(FileFormatAndFileSystem, WriteAndReadInteTest,
::testing::ValuesIn(GetTestValuesForWriteAndReadInteTest()));
diff --git
a/test/test_data/lance/append_ltz_java_compat.db/append_ltz_java_compat/README.md
b/test/test_data/lance/append_ltz_java_compat.db/append_ltz_java_compat/README.md
new file mode 100644
index 00000000..2e808e92
--- /dev/null
+++
b/test/test_data/lance/append_ltz_java_compat.db/append_ltz_java_compat/README.md
@@ -0,0 +1,40 @@
+Table: append_ltz_java_compat
+Writer: Paimon Java 2.2-SNAPSHOT at commit
535a5c944f854cb8b68196d44677e576cf22120b
+Lance version: lance-core 0.39.0
+Arrow Java version: 15.0.0
+JVM default time zone: GMT-10:00
+
+Schema:
+id INT
+f_ltz_0 TIMESTAMP(0) WITH LOCAL TIME ZONE
+f_ltz_3 TIMESTAMP(3) WITH LOCAL TIME ZONE
+f_ltz_6 TIMESTAMP(6) WITH LOCAL TIME ZONE
+f_ltz_9 TIMESTAMP(9) WITH LOCAL TIME ZONE
+f_ltz_array ARRAY<TIMESTAMP(6) WITH LOCAL TIME ZONE>
+f_ltz_struct ROW<number INT, ts TIMESTAMP(3) WITH LOCAL TIME ZONE>
+
+Options:
+bucket = -1
+file.format = lance
+metadata.stats-mode = none
+
+This unpartitioned append-only table has no primary key. The Java catalog and
+BatchTableWrite/BatchTableCommit APIs wrote two rows per commit into two Lance
data files,
+with schema, Avro manifests and two snapshots. All four rows were read back
and verified in Java.
+Values below follow schema order; timestamps are UTC.
+
+Add: (1, 1970-01-01 00:00:01, 1970-01-01 00:00:01.123, 1970-01-01
00:00:01.123456, 1970-01-01 00:00:01.123456789, [1970-01-01 00:00:01.123456,
NULL, 1969-12-31 23:59:59.999999], (1, 1970-01-01 00:00:01.123))
+Add: (2, NULL, NULL, NULL, NULL, NULL, (NULL, NULL))
+
+Commit - snapshot 1 (2 rows)
+
+Add: (10, 1969-12-31 00:00:00, 1969-12-31 23:59:59.999, 1969-12-31
23:59:59.999999, 1969-12-31 23:59:59.999999999, [], (10, 1969-12-31
23:59:59.999))
+Add: (11, 2024-10-04 12:34:56, 2024-10-04 12:34:56.123, 2024-10-04
12:34:56.123456, 2024-10-04 12:34:56.123456789, [2024-10-04 12:34:56.123456],
(11, 2024-10-04 12:34:56.123))
+
+Commit - snapshot 2 (4 rows total)
+
+The writer ran with a default time zone containing ':', which Lance 0.39
cannot parse as a
+time zone id. Java writes every local-zoned timestamp, including the ARRAY
element and the
+ROW child, to Lance as a timestamp with the "UTC" time zone, leaving the
instants unchanged.
+ROW parents are always valid, including id=2 where both children are NULL,
because Lance 0.39
+does not preserve null ROW parents.
diff --git
a/test/test_data/lance/append_ltz_java_compat.db/append_ltz_java_compat/bucket-0/data-4fcb4a78-f17b-4a9c-9454-ea37ee78e0c5-0.lance
b/test/test_data/lance/append_ltz_java_compat.db/append_ltz_java_compat/bucket-0/data-4fcb4a78-f17b-4a9c-9454-ea37ee78e0c5-0.lance
new file mode 100644
index 00000000..d4b8fd8b
Binary files /dev/null and
b/test/test_data/lance/append_ltz_java_compat.db/append_ltz_java_compat/bucket-0/data-4fcb4a78-f17b-4a9c-9454-ea37ee78e0c5-0.lance
differ
diff --git
a/test/test_data/lance/append_ltz_java_compat.db/append_ltz_java_compat/bucket-0/data-e9ac3746-5001-45e0-8060-d0e84efae172-0.lance
b/test/test_data/lance/append_ltz_java_compat.db/append_ltz_java_compat/bucket-0/data-e9ac3746-5001-45e0-8060-d0e84efae172-0.lance
new file mode 100644
index 00000000..b6bdcade
Binary files /dev/null and
b/test/test_data/lance/append_ltz_java_compat.db/append_ltz_java_compat/bucket-0/data-e9ac3746-5001-45e0-8060-d0e84efae172-0.lance
differ
diff --git
a/test/test_data/lance/append_ltz_java_compat.db/append_ltz_java_compat/manifest/manifest-1e2d27b3-41db-46d5-9c0e-507c8ffe695a-0
b/test/test_data/lance/append_ltz_java_compat.db/append_ltz_java_compat/manifest/manifest-1e2d27b3-41db-46d5-9c0e-507c8ffe695a-0
new file mode 100644
index 00000000..1c8b8ffc
Binary files /dev/null and
b/test/test_data/lance/append_ltz_java_compat.db/append_ltz_java_compat/manifest/manifest-1e2d27b3-41db-46d5-9c0e-507c8ffe695a-0
differ
diff --git
a/test/test_data/lance/append_ltz_java_compat.db/append_ltz_java_compat/manifest/manifest-fe3d8af9-46b4-4157-b164-7a0660316cf5-0
b/test/test_data/lance/append_ltz_java_compat.db/append_ltz_java_compat/manifest/manifest-fe3d8af9-46b4-4157-b164-7a0660316cf5-0
new file mode 100644
index 00000000..9fb3d4ed
Binary files /dev/null and
b/test/test_data/lance/append_ltz_java_compat.db/append_ltz_java_compat/manifest/manifest-fe3d8af9-46b4-4157-b164-7a0660316cf5-0
differ
diff --git
a/test/test_data/lance/append_ltz_java_compat.db/append_ltz_java_compat/manifest/manifest-list-1a908b65-80a0-47d5-9dcd-81c70689d863-0
b/test/test_data/lance/append_ltz_java_compat.db/append_ltz_java_compat/manifest/manifest-list-1a908b65-80a0-47d5-9dcd-81c70689d863-0
new file mode 100644
index 00000000..c14fca8c
Binary files /dev/null and
b/test/test_data/lance/append_ltz_java_compat.db/append_ltz_java_compat/manifest/manifest-list-1a908b65-80a0-47d5-9dcd-81c70689d863-0
differ
diff --git
a/test/test_data/lance/append_ltz_java_compat.db/append_ltz_java_compat/manifest/manifest-list-1a908b65-80a0-47d5-9dcd-81c70689d863-1
b/test/test_data/lance/append_ltz_java_compat.db/append_ltz_java_compat/manifest/manifest-list-1a908b65-80a0-47d5-9dcd-81c70689d863-1
new file mode 100644
index 00000000..a86721c2
Binary files /dev/null and
b/test/test_data/lance/append_ltz_java_compat.db/append_ltz_java_compat/manifest/manifest-list-1a908b65-80a0-47d5-9dcd-81c70689d863-1
differ
diff --git
a/test/test_data/lance/append_ltz_java_compat.db/append_ltz_java_compat/manifest/manifest-list-366b231f-4abf-408d-bd62-e7de9c6c1f0f-0
b/test/test_data/lance/append_ltz_java_compat.db/append_ltz_java_compat/manifest/manifest-list-366b231f-4abf-408d-bd62-e7de9c6c1f0f-0
new file mode 100644
index 00000000..555dd609
Binary files /dev/null and
b/test/test_data/lance/append_ltz_java_compat.db/append_ltz_java_compat/manifest/manifest-list-366b231f-4abf-408d-bd62-e7de9c6c1f0f-0
differ
diff --git
a/test/test_data/lance/append_ltz_java_compat.db/append_ltz_java_compat/manifest/manifest-list-366b231f-4abf-408d-bd62-e7de9c6c1f0f-1
b/test/test_data/lance/append_ltz_java_compat.db/append_ltz_java_compat/manifest/manifest-list-366b231f-4abf-408d-bd62-e7de9c6c1f0f-1
new file mode 100644
index 00000000..4fbf2b48
Binary files /dev/null and
b/test/test_data/lance/append_ltz_java_compat.db/append_ltz_java_compat/manifest/manifest-list-366b231f-4abf-408d-bd62-e7de9c6c1f0f-1
differ
diff --git
a/test/test_data/lance/append_ltz_java_compat.db/append_ltz_java_compat/schema/schema-0
b/test/test_data/lance/append_ltz_java_compat.db/append_ltz_java_compat/schema/schema-0
new file mode 100644
index 00000000..af124f64
--- /dev/null
+++
b/test/test_data/lance/append_ltz_java_compat.db/append_ltz_java_compat/schema/schema-0
@@ -0,0 +1,56 @@
+{
+ "version" : 3,
+ "id" : 0,
+ "fields" : [ {
+ "id" : 0,
+ "name" : "id",
+ "type" : "INT"
+ }, {
+ "id" : 1,
+ "name" : "f_ltz_0",
+ "type" : "TIMESTAMP(0) WITH LOCAL TIME ZONE"
+ }, {
+ "id" : 2,
+ "name" : "f_ltz_3",
+ "type" : "TIMESTAMP(3) WITH LOCAL TIME ZONE"
+ }, {
+ "id" : 3,
+ "name" : "f_ltz_6",
+ "type" : "TIMESTAMP(6) WITH LOCAL TIME ZONE"
+ }, {
+ "id" : 4,
+ "name" : "f_ltz_9",
+ "type" : "TIMESTAMP(9) WITH LOCAL TIME ZONE"
+ }, {
+ "id" : 5,
+ "name" : "f_ltz_array",
+ "type" : {
+ "type" : "ARRAY",
+ "element" : "TIMESTAMP(6) WITH LOCAL TIME ZONE"
+ }
+ }, {
+ "id" : 6,
+ "name" : "f_ltz_struct",
+ "type" : {
+ "type" : "ROW",
+ "fields" : [ {
+ "id" : 7,
+ "name" : "number",
+ "type" : "INT"
+ }, {
+ "id" : 8,
+ "name" : "ts",
+ "type" : "TIMESTAMP(3) WITH LOCAL TIME ZONE"
+ } ]
+ }
+ } ],
+ "highestFieldId" : 8,
+ "partitionKeys" : [ ],
+ "primaryKeys" : [ ],
+ "options" : {
+ "bucket" : "-1",
+ "file.format" : "lance",
+ "metadata.stats-mode" : "none"
+ },
+ "timeMillis" : 1793201358194
+}
\ No newline at end of file
diff --git
a/test/test_data/lance/append_ltz_java_compat.db/append_ltz_java_compat/snapshot/EARLIEST
b/test/test_data/lance/append_ltz_java_compat.db/append_ltz_java_compat/snapshot/EARLIEST
new file mode 100644
index 00000000..56a6051c
--- /dev/null
+++
b/test/test_data/lance/append_ltz_java_compat.db/append_ltz_java_compat/snapshot/EARLIEST
@@ -0,0 +1 @@
+1
\ No newline at end of file
diff --git
a/test/test_data/lance/append_ltz_java_compat.db/append_ltz_java_compat/snapshot/LATEST
b/test/test_data/lance/append_ltz_java_compat.db/append_ltz_java_compat/snapshot/LATEST
new file mode 100644
index 00000000..d8263ee9
--- /dev/null
+++
b/test/test_data/lance/append_ltz_java_compat.db/append_ltz_java_compat/snapshot/LATEST
@@ -0,0 +1 @@
+2
\ No newline at end of file
diff --git
a/test/test_data/lance/append_ltz_java_compat.db/append_ltz_java_compat/snapshot/snapshot-1
b/test/test_data/lance/append_ltz_java_compat.db/append_ltz_java_compat/snapshot/snapshot-1
new file mode 100644
index 00000000..e93e1817
--- /dev/null
+++
b/test/test_data/lance/append_ltz_java_compat.db/append_ltz_java_compat/snapshot/snapshot-1
@@ -0,0 +1,18 @@
+{
+ "version" : 3,
+ "uuid" : "cd56a571-077d-420f-ac22-695717c200de",
+ "id" : 1,
+ "schemaId" : 0,
+ "baseManifestList" : "manifest-list-1a908b65-80a0-47d5-9dcd-81c70689d863-0",
+ "baseManifestListSize" : 1158,
+ "deltaManifestList" : "manifest-list-1a908b65-80a0-47d5-9dcd-81c70689d863-1",
+ "deltaManifestListSize" : 1267,
+ "commitUser" : "41aabf22-c463-441a-9c3a-bfc9f6852590",
+ "writerVersion" :
"java-2.2-SNAPSHOT-535a5c944f854cb8b68196d44677e576cf22120b",
+ "commitIdentifier" : 9223372036854775807,
+ "commitKind" : "APPEND",
+ "timeMillis" : 1793201360419,
+ "totalRecordCount" : 2,
+ "deltaRecordCount" : 2,
+ "nextRowId" : 0
+}
\ No newline at end of file
diff --git
a/test/test_data/lance/append_ltz_java_compat.db/append_ltz_java_compat/snapshot/snapshot-2
b/test/test_data/lance/append_ltz_java_compat.db/append_ltz_java_compat/snapshot/snapshot-2
new file mode 100644
index 00000000..ecfa750e
--- /dev/null
+++
b/test/test_data/lance/append_ltz_java_compat.db/append_ltz_java_compat/snapshot/snapshot-2
@@ -0,0 +1,18 @@
+{
+ "version" : 3,
+ "uuid" : "10e95ecd-534f-4016-9644-e37ce4fb51ec",
+ "id" : 2,
+ "schemaId" : 0,
+ "baseManifestList" : "manifest-list-366b231f-4abf-408d-bd62-e7de9c6c1f0f-0",
+ "baseManifestListSize" : 1267,
+ "deltaManifestList" : "manifest-list-366b231f-4abf-408d-bd62-e7de9c6c1f0f-1",
+ "deltaManifestListSize" : 1267,
+ "commitUser" : "44116caa-dcc7-4397-b617-fa43046b55b8",
+ "writerVersion" :
"java-2.2-SNAPSHOT-535a5c944f854cb8b68196d44677e576cf22120b",
+ "commitIdentifier" : 9223372036854775807,
+ "commitKind" : "APPEND",
+ "timeMillis" : 1793201360592,
+ "totalRecordCount" : 4,
+ "deltaRecordCount" : 2,
+ "nextRowId" : 0
+}
\ No newline at end of file
diff --git
a/test/test_data/lance/append_ltz_python_compat.db/append_ltz_python_compat/README.md
b/test/test_data/lance/append_ltz_python_compat.db/append_ltz_python_compat/README.md
new file mode 100644
index 00000000..f48c8511
--- /dev/null
+++
b/test/test_data/lance/append_ltz_python_compat.db/append_ltz_python_compat/README.md
@@ -0,0 +1,43 @@
+Table: append_ltz_python_compat
+Writer: Apache Paimon Python at commit 63ffd403648e1cf9848e32bb29a398c1b3b16ab1
+Source:
https://github.com/apache/paimon/tree/63ffd403648e1cf9848e32bb29a398c1b3b16ab1/paimon-python
+Python version: 3.11.15
+Lance version: pylance 0.39.0
+Arrow Python version: 19.0.1
+
+This unpartitioned append-only table has no primary key. The Python Catalog
and batch
+table-write/commit APIs wrote two rows per commit into two Lance files, with
Avro manifests
+and two snapshots. All values were read back in Python after each commit;
reordered
+projection was also verified after snapshot 2.
+
+Schema (all fields nullable):
+id INT
+f_ltz_0 TIMESTAMP_LTZ(0)
+f_ltz_3 TIMESTAMP_LTZ(3)
+f_ltz_6 TIMESTAMP_LTZ(6)
+f_ltz_9 TIMESTAMP_LTZ(9)
+f_ltz_array ARRAY<TIMESTAMP_LTZ(6)>
+f_ltz_struct ROW<number INT, ts TIMESTAMP_LTZ(3)>
+
+Options:
+bucket = -1
+file.format = lance
+manifest.format = avro
+metadata.stats-mode = none
+
+Values below follow schema order; timestamps are UTC. They match
append_ltz_java_compat.
+
+Add: (1, 1970-01-01 00:00:01, 1970-01-01 00:00:01.123, 1970-01-01
00:00:01.123456, 1970-01-01 00:00:01.123456789, [1970-01-01 00:00:01.123456,
NULL, 1969-12-31 23:59:59.999999], (1, 1970-01-01 00:00:01.123))
+Add: (2, NULL, NULL, NULL, NULL, NULL, (NULL, NULL))
+
+Commit - snapshot 1 (2 rows)
+
+Add: (10, 1969-12-31 00:00:00, 1969-12-31 23:59:59.999, 1969-12-31
23:59:59.999999, 1969-12-31 23:59:59.999999999, [], (10, 1969-12-31
23:59:59.999))
+Add: (11, 2024-10-04 12:34:56, 2024-10-04 12:34:56.123, 2024-10-04
12:34:56.123456, 2024-10-04 12:34:56.123456789, [2024-10-04 12:34:56.123456],
(11, 2024-10-04 12:34:56.123))
+
+Commit - snapshot 2 (4 rows total)
+
+Paimon Python maps a pyarrow timestamp with a time zone to TIMESTAMP_LTZ and
writes every
+TIMESTAMP_LTZ, including the ARRAY element and the ROW child, to Lance as a
timestamp with
+the "UTC" time zone.
+ROW parents are always valid: this fixture does not claim null-parent ROW
compatibility.
diff --git
a/test/test_data/lance/append_ltz_python_compat.db/append_ltz_python_compat/bucket-0/data-3dd49f37-400b-4d87-ab60-dda8278083c8-0.lance
b/test/test_data/lance/append_ltz_python_compat.db/append_ltz_python_compat/bucket-0/data-3dd49f37-400b-4d87-ab60-dda8278083c8-0.lance
new file mode 100644
index 00000000..a88ebd17
Binary files /dev/null and
b/test/test_data/lance/append_ltz_python_compat.db/append_ltz_python_compat/bucket-0/data-3dd49f37-400b-4d87-ab60-dda8278083c8-0.lance
differ
diff --git
a/test/test_data/lance/append_ltz_python_compat.db/append_ltz_python_compat/bucket-0/data-cb6cba6a-5d96-42f4-b8b9-2b5b4616c60f-0.lance
b/test/test_data/lance/append_ltz_python_compat.db/append_ltz_python_compat/bucket-0/data-cb6cba6a-5d96-42f4-b8b9-2b5b4616c60f-0.lance
new file mode 100644
index 00000000..0fbf94bb
Binary files /dev/null and
b/test/test_data/lance/append_ltz_python_compat.db/append_ltz_python_compat/bucket-0/data-cb6cba6a-5d96-42f4-b8b9-2b5b4616c60f-0.lance
differ
diff --git
a/test/test_data/lance/append_ltz_python_compat.db/append_ltz_python_compat/manifest/manifest-2630e922-a399-4613-9181-54cc65803426-0
b/test/test_data/lance/append_ltz_python_compat.db/append_ltz_python_compat/manifest/manifest-2630e922-a399-4613-9181-54cc65803426-0
new file mode 100644
index 00000000..583437a5
Binary files /dev/null and
b/test/test_data/lance/append_ltz_python_compat.db/append_ltz_python_compat/manifest/manifest-2630e922-a399-4613-9181-54cc65803426-0
differ
diff --git
a/test/test_data/lance/append_ltz_python_compat.db/append_ltz_python_compat/manifest/manifest-db163726-5626-4b91-a607-33f79a420829-0
b/test/test_data/lance/append_ltz_python_compat.db/append_ltz_python_compat/manifest/manifest-db163726-5626-4b91-a607-33f79a420829-0
new file mode 100644
index 00000000..54f23577
Binary files /dev/null and
b/test/test_data/lance/append_ltz_python_compat.db/append_ltz_python_compat/manifest/manifest-db163726-5626-4b91-a607-33f79a420829-0
differ
diff --git
a/test/test_data/lance/append_ltz_python_compat.db/append_ltz_python_compat/manifest/manifest-list-92cdba02-3525-463d-981d-381562e9cb9d-0
b/test/test_data/lance/append_ltz_python_compat.db/append_ltz_python_compat/manifest/manifest-list-92cdba02-3525-463d-981d-381562e9cb9d-0
new file mode 100644
index 00000000..03a8e842
Binary files /dev/null and
b/test/test_data/lance/append_ltz_python_compat.db/append_ltz_python_compat/manifest/manifest-list-92cdba02-3525-463d-981d-381562e9cb9d-0
differ
diff --git
a/test/test_data/lance/append_ltz_python_compat.db/append_ltz_python_compat/manifest/manifest-list-92cdba02-3525-463d-981d-381562e9cb9d-1
b/test/test_data/lance/append_ltz_python_compat.db/append_ltz_python_compat/manifest/manifest-list-92cdba02-3525-463d-981d-381562e9cb9d-1
new file mode 100644
index 00000000..78e113ff
Binary files /dev/null and
b/test/test_data/lance/append_ltz_python_compat.db/append_ltz_python_compat/manifest/manifest-list-92cdba02-3525-463d-981d-381562e9cb9d-1
differ
diff --git
a/test/test_data/lance/append_ltz_python_compat.db/append_ltz_python_compat/manifest/manifest-list-df1a6587-9d9b-43e9-a73a-18245a5217b2-0
b/test/test_data/lance/append_ltz_python_compat.db/append_ltz_python_compat/manifest/manifest-list-df1a6587-9d9b-43e9-a73a-18245a5217b2-0
new file mode 100644
index 00000000..d7aaffb2
Binary files /dev/null and
b/test/test_data/lance/append_ltz_python_compat.db/append_ltz_python_compat/manifest/manifest-list-df1a6587-9d9b-43e9-a73a-18245a5217b2-0
differ
diff --git
a/test/test_data/lance/append_ltz_python_compat.db/append_ltz_python_compat/manifest/manifest-list-df1a6587-9d9b-43e9-a73a-18245a5217b2-1
b/test/test_data/lance/append_ltz_python_compat.db/append_ltz_python_compat/manifest/manifest-list-df1a6587-9d9b-43e9-a73a-18245a5217b2-1
new file mode 100644
index 00000000..0f034d16
Binary files /dev/null and
b/test/test_data/lance/append_ltz_python_compat.db/append_ltz_python_compat/manifest/manifest-list-df1a6587-9d9b-43e9-a73a-18245a5217b2-1
differ
diff --git
a/test/test_data/lance/append_ltz_python_compat.db/append_ltz_python_compat/schema/schema-0
b/test/test_data/lance/append_ltz_python_compat.db/append_ltz_python_compat/schema/schema-0
new file mode 100644
index 00000000..ddf0dc4f
--- /dev/null
+++
b/test/test_data/lance/append_ltz_python_compat.db/append_ltz_python_compat/schema/schema-0
@@ -0,0 +1,71 @@
+{
+ "version": 3,
+ "id": 0,
+ "fields": [
+ {
+ "id": 0,
+ "name": "id",
+ "type": "INT"
+ },
+ {
+ "id": 1,
+ "name": "f_ltz_0",
+ "type": "TIMESTAMP_LTZ(0)"
+ },
+ {
+ "id": 2,
+ "name": "f_ltz_3",
+ "type": "TIMESTAMP_LTZ(3)"
+ },
+ {
+ "id": 3,
+ "name": "f_ltz_6",
+ "type": "TIMESTAMP_LTZ(6)"
+ },
+ {
+ "id": 4,
+ "name": "f_ltz_9",
+ "type": "TIMESTAMP_LTZ(9)"
+ },
+ {
+ "id": 5,
+ "name": "f_ltz_array",
+ "type": {
+ "type": "ARRAY",
+ "element": "TIMESTAMP_LTZ(6)",
+ "nullable": true
+ }
+ },
+ {
+ "id": 6,
+ "name": "f_ltz_struct",
+ "type": {
+ "type": "ROW",
+ "fields": [
+ {
+ "id": 7,
+ "name": "number",
+ "type": "INT"
+ },
+ {
+ "id": 8,
+ "name": "ts",
+ "type": "TIMESTAMP_LTZ(3)"
+ }
+ ],
+ "nullable": true
+ }
+ }
+ ],
+ "highestFieldId": 8,
+ "partitionKeys": [],
+ "primaryKeys": [],
+ "options": {
+ "bucket": "-1",
+ "file.format": "lance",
+ "manifest.format": "avro",
+ "metadata.stats-mode": "none"
+ },
+ "comment": null,
+ "timeMillis": 1793201390242
+}
\ No newline at end of file
diff --git
a/test/test_data/lance/append_ltz_python_compat.db/append_ltz_python_compat/snapshot/LATEST
b/test/test_data/lance/append_ltz_python_compat.db/append_ltz_python_compat/snapshot/LATEST
new file mode 100644
index 00000000..d8263ee9
--- /dev/null
+++
b/test/test_data/lance/append_ltz_python_compat.db/append_ltz_python_compat/snapshot/LATEST
@@ -0,0 +1 @@
+2
\ No newline at end of file
diff --git
a/test/test_data/lance/append_ltz_python_compat.db/append_ltz_python_compat/snapshot/snapshot-1
b/test/test_data/lance/append_ltz_python_compat.db/append_ltz_python_compat/snapshot/snapshot-1
new file mode 100644
index 00000000..337aa98c
--- /dev/null
+++
b/test/test_data/lance/append_ltz_python_compat.db/append_ltz_python_compat/snapshot/snapshot-1
@@ -0,0 +1,15 @@
+{
+ "version": 3,
+ "id": 1,
+ "schemaId": 0,
+ "baseManifestList": "manifest-list-df1a6587-9d9b-43e9-a73a-18245a5217b2-0",
+ "deltaManifestList": "manifest-list-df1a6587-9d9b-43e9-a73a-18245a5217b2-1",
+ "totalRecordCount": 2,
+ "deltaRecordCount": 2,
+ "commitUser": "bf1553ec-84d3-4bba-a63f-af57a0821aac",
+ "commitIdentifier": 9223372036854775807,
+ "commitKind": "APPEND",
+ "timeMillis": 1793201390271,
+ "uuid": "146996fb-0a39-443e-8c5b-8716ef072816",
+ "writerVersion": "python-2.1.dev-63ffd403648e1cf9848e32bb29a398c1b3b16ab1"
+}
\ No newline at end of file
diff --git
a/test/test_data/lance/append_ltz_python_compat.db/append_ltz_python_compat/snapshot/snapshot-2
b/test/test_data/lance/append_ltz_python_compat.db/append_ltz_python_compat/snapshot/snapshot-2
new file mode 100644
index 00000000..450f7d73
--- /dev/null
+++
b/test/test_data/lance/append_ltz_python_compat.db/append_ltz_python_compat/snapshot/snapshot-2
@@ -0,0 +1,15 @@
+{
+ "version": 3,
+ "id": 2,
+ "schemaId": 0,
+ "baseManifestList": "manifest-list-92cdba02-3525-463d-981d-381562e9cb9d-0",
+ "deltaManifestList": "manifest-list-92cdba02-3525-463d-981d-381562e9cb9d-1",
+ "totalRecordCount": 4,
+ "deltaRecordCount": 2,
+ "commitUser": "a1338729-f60a-4d94-895b-8c4860517802",
+ "commitIdentifier": 9223372036854775807,
+ "commitKind": "APPEND",
+ "timeMillis": 1793201390281,
+ "uuid": "d721aff9-8b3f-4e4a-82ad-b522916f5e08",
+ "writerVersion": "python-2.1.dev-63ffd403648e1cf9848e32bb29a398c1b3b16ab1"
+}
\ No newline at end of file
