This is an automated email from the ASF dual-hosted git repository.
lxy-9602 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/paimon-cpp.git
The following commit(s) were added to refs/heads/main by this push:
new 1b3c6905 feat(lance): support TIME type (#478)
1b3c6905 is described below
commit 1b3c6905de25f914a8c8cd6ba9200002d5d479a5
Author: Nicholas Jiang <[email protected]>
AuthorDate: Wed Oct 28 15:06:05 2026 +0700
feat(lance): support TIME type (#478)
---
docs/source/user_guide/data_types.rst | 18 +++++-
src/paimon/core/schema/schema_validation.cpp | 1 +
src/paimon/core/schema/schema_validation_test.cpp | 12 ++--
src/paimon/format/lance/lance_file_format_test.cpp | 46 ++++++++++---
src/paimon/format/lance/lance_stats_extractor.cpp | 2 +
test/inte/scan_and_read_inte_test.cpp | 24 ++++---
test/inte/write_and_read_inte_test.cpp | 71 +++++++++++++--------
.../append_java_compat/README.md | 18 ++++--
...a-26e3a06f-8bc4-4bea-99c2-1146b8c0bccf-0.lance} | Bin 8812 -> 9330 bytes
...a-e2d161b4-2446-450b-9b59-2e880ced6aaa-0.lance} | Bin 10275 -> 10947 bytes
...anifest-0dcbe4b0-b6a6-4cba-9c74-b6fa5db0ad61-0} | Bin 2166 -> 2165 bytes
...anifest-428dd1a6-0756-44d3-939c-f39159964fbc-0} | Bin 2164 -> 2164 bytes
...st-list-1a7f860c-eed7-4f3b-9134-dd5f12b8a7d3-0} | Bin 1110 -> 1110 bytes
...st-list-1a7f860c-eed7-4f3b-9134-dd5f12b8a7d3-1} | Bin 1113 -> 1113 bytes
...st-list-24709406-a874-4961-b200-879ceba4452c-0} | Bin 1006 -> 1006 bytes
...st-list-24709406-a874-4961-b200-879ceba4452c-1} | Bin 1110 -> 1110 bytes
.../append_java_compat/schema/schema-0 | 44 +++++++------
.../append_java_compat/snapshot/snapshot-1 | 10 +--
.../append_java_compat/snapshot/snapshot-2 | 10 +--
.../append_python_compat/README.md | 14 ++--
...a-24cc7753-16ed-451d-8899-984adb2de133-0.lance} | Bin 6717 -> 7187 bytes
...a-27b4eee6-062d-4781-b762-d18f2f1a90c8-0.lance} | Bin 8103 -> 8727 bytes
...anifest-63284fb6-53d2-4c5d-8719-a411c920e155-0} | Bin 2208 -> 2209 bytes
...anifest-a10d7552-9602-4543-bc69-152765c1c59c-0} | Bin 2208 -> 2208 bytes
...st-list-cb00c974-4152-4c3c-aa64-7910d413b4d6-0} | Bin 799 -> 799 bytes
...st-list-cb00c974-4152-4c3c-aa64-7910d413b4d6-1} | Bin 897 -> 897 bytes
...st-list-ce87ee4f-fee3-4059-b37f-a1f295a72a7c-0} | Bin 897 -> 897 bytes
...st-list-ce87ee4f-fee3-4059-b37f-a1f295a72a7c-1} | Bin 897 -> 897 bytes
.../append_python_compat/schema/schema-0 | 40 +++++++-----
.../append_python_compat/snapshot/snapshot-1 | 10 +--
.../append_python_compat/snapshot/snapshot-2 | 10 +--
31 files changed, 213 insertions(+), 117 deletions(-)
diff --git a/docs/source/user_guide/data_types.rst
b/docs/source/user_guide/data_types.rst
index 34818ea4..92b1dda1 100644
--- a/docs/source/user_guide/data_types.rst
+++ b/docs/source/user_guide/data_types.rst
@@ -146,13 +146,29 @@ and `Arrow DataTypes
<https://arrow.apache.org/docs/format/Columnar.html#data-ty
* - ``TIME``
``TIME(p)``
- - Not Supported
+ - Time32
- Data type of a time without time zone consisting of
hour:minute:second[.fractional] with up to nanosecond precision and values
ranging from 00:00:00.000000000 to 23:59:59.999999999.
The type can be declared using ``TIME(p)`` where p is the number of
digits of
fractional seconds (precision). p must have a value between 0 and 9
(both inclusive). If no precision is specified, p is equal to 0.
+ Paimon C++ maps every ``TIME(p)`` to an Arrow ``Time32`` in milliseconds
+ (``time32[ms]``) and accepts no other Arrow time type. The declared
+ precision p is kept in the ``paimon.time.precision`` field metadata; a
+ ``Time32`` field without it is ``TIME(0)``. As in Paimon Java, values
are
+ stored as milliseconds of the day whatever p is, so ``TIME(6)`` and
+ ``TIME(9)`` values keep only millisecond precision.
+
+ Paimon C++ reads TIME columns from Parquet, Lance and Vortex data files
+ and writes them to Lance and Vortex data files. ORC and Avro data files
+ do not support TIME.
+
+ **Note:** Only append-only tables support TIME columns. Primary-key
+ tables do not support TIME, whether as a primary key or as a value
+ column. A TIME column also cannot be a partition or bucket key, and
+ predicates cannot reference TIME columns.
+
* - ``TIMESTAMP``
``TIMESTAMP(p)``
diff --git a/src/paimon/core/schema/schema_validation.cpp
b/src/paimon/core/schema/schema_validation.cpp
index 3a73838d..7717b27f 100644
--- a/src/paimon/core/schema/schema_validation.cpp
+++ b/src/paimon/core/schema/schema_validation.cpp
@@ -1001,6 +1001,7 @@ Status SchemaValidation::ValidateLanceDataField(const
std::shared_ptr<arrow::Fie
case arrow::Type::FLOAT:
case arrow::Type::DOUBLE:
case arrow::Type::DATE32:
+ case arrow::Type::TIME32:
case arrow::Type::STRING:
case arrow::Type::BINARY:
case arrow::Type::DECIMAL128:
diff --git a/src/paimon/core/schema/schema_validation_test.cpp
b/src/paimon/core/schema/schema_validation_test.cpp
index d7c380ff..4f7b4116 100644
--- a/src/paimon/core/schema/schema_validation_test.cpp
+++ b/src/paimon/core/schema/schema_validation_test.cpp
@@ -437,9 +437,13 @@ TEST(SchemaValidationTest, TestLanceDataTypes) {
arrow::field("string", arrow::utf8()),
arrow::field("binary", arrow::binary()),
arrow::field("date", arrow::date32()),
+ arrow::field("time", arrow::time32(arrow::TimeUnit::MILLI)),
arrow::field("timestamp", arrow::timestamp(arrow::TimeUnit::MICRO)),
arrow::field("decimal", arrow::decimal128(38, 2)),
arrow::field("array", arrow::list(arrow::float32())),
+ arrow::field("nested_time",
+ arrow::struct_({arrow::field(
+ "values",
arrow::list(arrow::time32(arrow::TimeUnit::MILLI)))})),
arrow::field("row", arrow::struct_({arrow::field("value",
arrow::int32())}),
/*nullable=*/false),
arrow::field("nullable_row", arrow::struct_({arrow::field("value",
arrow::int32())})),
@@ -472,13 +476,9 @@ TEST(SchemaValidationTest, TestLanceDataTypes) {
arrow::field("map", arrow::map(arrow::int32(), arrow::utf8())),
arrow::field("ltz", arrow::timestamp(arrow::TimeUnit::MICRO, "UTC")),
VariantTypeUtils::ToArrowField("variant"),
- arrow::field("time_millis", arrow::time32(arrow::TimeUnit::MILLI)),
- arrow::field("nested_time",
- arrow::struct_({arrow::field(
- "values",
arrow::list(arrow::time32(arrow::TimeUnit::MILLI)))})),
};
- std::vector<std::string> expected_errors = {"type MAP",
"LOCAL_ZONED_TIMESTAMP", "type VARIANT",
- "type time32", "type time32"};
+ std::vector<std::string> expected_errors = {"type MAP",
"LOCAL_ZONED_TIMESTAMP",
+ "type VARIANT"};
for (size_t i = 0; i < unsupported_fields.size(); ++i) {
ASSERT_OK_AND_ASSIGN(
table_schema,
diff --git a/src/paimon/format/lance/lance_file_format_test.cpp
b/src/paimon/format/lance/lance_file_format_test.cpp
index 089f5f32..0805d49e 100644
--- a/src/paimon/format/lance/lance_file_format_test.cpp
+++ b/src/paimon/format/lance/lance_file_format_test.cpp
@@ -29,6 +29,7 @@
#include "arrow/ipc/json_simple.h"
#include "gtest/gtest.h"
#include "paimon/common/types/data_field.h"
+#include "paimon/common/types/data_type.h"
#include "paimon/common/utils/arrow/mem_utils.h"
#include "paimon/common/utils/arrow/status_utils.h"
#include "paimon/common/utils/checked_cast.h"
@@ -176,6 +177,11 @@ TEST(LanceUtilsTest, ConvertsObjectStoreOptions) {
}
TEST_F(LanceFileFormatTest, WriteThenReadSupportedTypes) {
+ auto time_field = [](const std::string& name, int32_t precision) {
+ return arrow::field(
+ name, arrow::time32(arrow::TimeUnit::MILLI), /*nullable=*/true,
+ arrow::key_value_metadata({DataType::kTimePrecision},
{std::to_string(precision)}));
+ };
arrow::FieldVector fields = {
arrow::field("bool_col", arrow::boolean()),
arrow::field("i8_col", arrow::int8()),
@@ -187,6 +193,7 @@ TEST_F(LanceFileFormatTest, WriteThenReadSupportedTypes) {
arrow::field("string_col", arrow::utf8()),
arrow::field("binary_col", arrow::binary()),
arrow::field("date_col", arrow::date32()),
+ time_field("time_col", 3),
arrow::field("timestamp_col",
arrow::timestamp(arrow::TimeUnit::MICRO)),
arrow::field("decimal_col", arrow::decimal128(12, 2)),
arrow::field("array_col", arrow::list(arrow::float32())),
@@ -194,15 +201,22 @@ TEST_F(LanceFileFormatTest, WriteThenReadSupportedTypes) {
arrow::field(
"vector_col",
arrow::fixed_size_list(arrow::field("item", arrow::float32(),
/*nullable=*/false), 2)),
+ arrow::field("time_array_col", arrow::list(time_field("item", 0))),
+ arrow::field("time_row_col", arrow::struct_({time_field("value", 6)})),
+ arrow::field("time_row_array_col",
+ arrow::struct_({arrow::field("values",
arrow::list(time_field("item", 9)))})),
};
std::shared_ptr<arrow::Schema> schema = arrow::schema(fields);
std::shared_ptr<arrow::Array> expected =
arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([
- [true,-1,-2,-3,-4,1.5,2.5,"one","abc",1,
- "1970-01-01 00:00:00.000001","12.34",[1.0,2.0],[7],[3.0,4.0]],
- [false,1,2,3,4,3.5,4.5,"two","xyz",2,
- "2030-12-31 23:59:59.999999","-12.34",[],[8],[5.0,6.0]],
-
[null,null,null,null,null,null,null,null,null,null,null,null,null,[null],null]
+ [true,-1,-2,-3,-4,1.5,2.5,"one","abc",1,0,
+ "1970-01-01 00:00:00.000001","12.34",[1.0,2.0],[7],[3.0,4.0],
+ [0,null,45296000],[45296123],[[3723123,null]]],
+ [false,1,2,3,4,3.5,4.5,"two","xyz",2,86399999,
+ "2030-12-31 23:59:59.999999","-12.34",[],[8],[5.0,6.0],
+ [],[86399999],[[]]],
+
[null,null,null,null,null,null,null,null,null,null,null,null,null,null,[null],null,
+ null,[null],[null]]
])")
.ValueOrDie();
std::string path = PathUtil::JoinPath(directory_->Str(),
"supported-types.lance");
@@ -219,6 +233,19 @@ TEST_F(LanceFileFormatTest, WriteThenReadSupportedTypes) {
ASSERT_TRUE(actual->Equals(arrow::ChunkedArray(expected))) <<
actual->ToString() << "\nvs\n"
<<
expected->ToString();
}
+
+ ASSERT_OK_AND_ASSIGN(std::unique_ptr<FileBatchReader> reader,
OpenReader(path, 2));
+ ::ArrowSchema ffi_schema = {};
+ ASSERT_TRUE(arrow::ExportSchema(*schema, &ffi_schema).ok());
+ ASSERT_OK(reader->SetReadSchema(&ffi_schema, /*predicate=*/nullptr,
+ /*selection_bitmap=*/std::nullopt));
+ ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch batch, reader->NextBatch());
+ arrow::Result<std::shared_ptr<arrow::Array>> batch_result =
+ arrow::ImportArray(batch.first.get(), batch.second.get());
+ ASSERT_TRUE(batch_result.ok()) << batch_result.status().ToString();
+ std::shared_ptr<arrow::Array> batch_array =
std::move(batch_result).MoveValueUnsafe();
+ ASSERT_TRUE(batch_array->type()->Equals(arrow::struct_(fields),
/*check_metadata=*/true))
+ << batch_array->type()->ToString(/*show_metadata=*/true);
}
TEST_F(LanceFileFormatTest, RejectsNullTopLevelRows) {
@@ -351,10 +378,12 @@ TEST_F(LanceFileFormatTest,
EmptyProjectionAndEmptySelection) {
TEST_F(LanceFileFormatTest, ExtractStatistics) {
arrow::FieldVector fields = {arrow::field("id", arrow::int32()),
- arrow::field("name", arrow::utf8())};
+ arrow::field("name", arrow::utf8()),
+ arrow::field("time",
arrow::time32(arrow::TimeUnit::MILLI))};
std::shared_ptr<arrow::Schema> schema = arrow::schema(fields);
std::shared_ptr<arrow::Array> data =
- arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields),
R"([[1,"a"],[2,null]])")
+ arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields),
+
R"([[1,"a",3723123],[2,null,null]])")
.ValueOrDie();
std::string path = PathUtil::JoinPath(directory_->Str(), "stats.lance");
ASSERT_OK(WriteFile(path, schema, data, /*batch_size=*/2));
@@ -365,9 +394,10 @@ TEST_F(LanceFileFormatTest, ExtractStatistics) {
format_->CreateStatsExtractor(&ffi_schema));
ASSERT_OK_AND_ASSIGN(auto result,
extractor->ExtractWithFileInfo(file_system_, path, pool_));
ASSERT_EQ(result.second.GetRowCount(), 2);
- ASSERT_EQ(result.first.size(), 2);
+ ASSERT_EQ(result.first.size(), 3);
ASSERT_EQ(result.first[0]->ToString(), "min null, max null, null count
null");
ASSERT_EQ(result.first[1]->ToString(), "min null, max null, null count
null");
+ ASSERT_EQ(result.first[2]->ToString(), "min null, max null, null count
null");
}
TEST_F(LanceFileFormatTest, RejectsIncompatibleReadTypes) {
diff --git a/src/paimon/format/lance/lance_stats_extractor.cpp
b/src/paimon/format/lance/lance_stats_extractor.cpp
index 0bd39f29..833af6a5 100644
--- a/src/paimon/format/lance/lance_stats_extractor.cpp
+++ b/src/paimon/format/lance/lance_stats_extractor.cpp
@@ -86,6 +86,8 @@ Result<std::unique_ptr<ColumnStats>>
LanceStatsExtractor::CreateEmptyStats(
return ColumnStats::CreateStringColumnStats(std::nullopt,
std::nullopt, std::nullopt);
case arrow::Type::DATE32:
return ColumnStats::CreateDateColumnStats(std::nullopt,
std::nullopt, std::nullopt);
+ case arrow::Type::TIME32:
+ return ColumnStats::CreateIntColumnStats(std::nullopt,
std::nullopt, std::nullopt);
case arrow::Type::TIMESTAMP: {
std::shared_ptr<arrow::TimestampType> timestamp_type =
checked_pointer_cast<arrow::TimestampType>(type);
diff --git a/test/inte/scan_and_read_inte_test.cpp
b/test/inte/scan_and_read_inte_test.cpp
index b7cc73b9..3bfb58c7 100644
--- a/test/inte/scan_and_read_inte_test.cpp
+++ b/test/inte/scan_and_read_inte_test.cpp
@@ -3479,6 +3479,8 @@ TEST_F(ScanAndReadInteTest, TestLanceJavaCompatibility) {
arrow::field("f_binary", arrow::binary()),
arrow::field("f_varbinary", arrow::binary()),
arrow::field("f_date", arrow::date32()),
+ arrow::field("f_time_0", arrow::time32(arrow::TimeUnit::MILLI)),
+ arrow::field("f_time_3", arrow::time32(arrow::TimeUnit::MILLI)),
arrow::field("f_ts_0", arrow::timestamp(arrow::TimeUnit::SECOND)),
arrow::field("f_ts_3", arrow::timestamp(arrow::TimeUnit::MILLI)),
arrow::field("f_ts_6", arrow::timestamp(arrow::TimeUnit::MICRO)),
@@ -3497,10 +3499,10 @@ TEST_F(ScanAndReadInteTest, TestLanceJavaCompatibility)
{
// from null parents; see the README alongside the Java compatibility
table.
auto expected = std::make_shared<arrow::ChunkedArray>(
arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([
-[0, 1, true, -5, -1000, 10000000001, 1.25, -2.5, "char0001", "value-1",
"bin1", "\u0000\u0001\u0002\u007f", -1, "1970-01-01 00:00:01", "1970-01-01
00:00:01.123", "1970-01-01 00:00:01.123456", "1970-01-01 00:00:01.123456789",
"-3", "1.25", "12345678901234567.89",
"12345678901234567890.123456789012345678", [1, null, -1], [[1, null], null,
[]], [1, "required"], [1, "nullable"], [1.0, -1.5, 0.25]],
-[0, 2, null, null, null, null, null, null, null, null, null, null, null, null,
null, null, null, null, null, null, null, null, null, [null, null], [null,
null], null],
-[0, 10, false, -5, -1000, 10000000010, 1.25, -2.5, "char0001", "", "bin1",
"\u0000\u0001\u0002\u007f", 20000, "1970-01-01 00:00:01", "1970-01-01
00:00:01.123", "1970-01-01 00:00:01.123456", "1970-01-01 00:00:01.123456789",
"-3", "1.25", "12345678901234567.89",
"12345678901234567890.123456789012345678", [], [[10, null], null, []], [10,
"required"], [10, "nullable"], [10.0, -1.5, 0.25]],
-[0, 11, true, -5, -1000, 10000000011, 1.25, -2.5, "char0001", "value-11",
"bin1", "\u0000\u0001\u0002\u007f", 20000, "1970-01-01 00:00:01", "1970-01-01
00:00:01.123", "1970-01-01 00:00:01.123456", "1970-01-01 00:00:01.123456789",
"-3", "1.25", "12345678901234567.89",
"12345678901234567890.123456789012345678", [11, null, -11], [[11, null], null,
[]], [11, "required"], [11, "nullable"], [11.0, -1.5, 0.25]]
+[0, 1, true, -5, -1000, 10000000001, 1.25, -2.5, "char0001", "value-1",
"bin1", "\u0000\u0001\u0002\u007f", -1, 0, 86399999, "1970-01-01 00:00:01",
"1970-01-01 00:00:01.123", "1970-01-01 00:00:01.123456", "1970-01-01
00:00:01.123456789", "-3", "1.25", "12345678901234567.89",
"12345678901234567890.123456789012345678", [1, null, -1], [[1, null], null,
[]], [1, "required"], [1, "nullable"], [1.0, -1.5, 0.25]],
+[0, 2, null, null, null, null, null, null, null, null, null, null, null, null,
null, null, null, null, null, null, null, null, null, null, null, [null, null],
[null, null], null],
+[0, 10, false, -5, -1000, 10000000010, 1.25, -2.5, "char0001", "", "bin1",
"\u0000\u0001\u0002\u007f", 20000, 45296000, 1123, "1970-01-01 00:00:01",
"1970-01-01 00:00:01.123", "1970-01-01 00:00:01.123456", "1970-01-01
00:00:01.123456789", "-3", "1.25", "12345678901234567.89",
"12345678901234567890.123456789012345678", [], [[10, null], null, []], [10,
"required"], [10, "nullable"], [10.0, -1.5, 0.25]],
+[0, 11, true, -5, -1000, 10000000011, 1.25, -2.5, "char0001", "value-11",
"bin1", "\u0000\u0001\u0002\u007f", 20000, 86399000, 45296789, "1970-01-01
00:00:01", "1970-01-01 00:00:01.123", "1970-01-01 00:00:01.123456", "1970-01-01
00:00:01.123456789", "-3", "1.25", "12345678901234567.89",
"12345678901234567890.123456789012345678", [11, null, -11], [[11, null], null,
[]], [11, "required"], [11, "nullable"], [11.0, -1.5, 0.25]]
])")
.ValueOrDie());
const std::string table_path = GetDataDir() +
"/lance/append_java_compat.db/append_java_compat";
@@ -3546,7 +3548,7 @@ TEST_F(ScanAndReadInteTest, TestLanceJavaCompatibility) {
auto projected = std::make_shared<arrow::ChunkedArray>(
arrow::ipc::internal::json::ArrayFromJSON(
- arrow::struct_({fields[0], fields[24], fields[1], fields[25]}),
R"([
+ arrow::struct_({fields[0], fields[26], fields[1], fields[27]}),
R"([
[0, [1, "nullable"], 1, [1.0, -1.5, 0.25]],
[0, [null, null], 2, null],
[0, [10, "nullable"], 10, [10.0, -1.5, 0.25]],
@@ -3570,6 +3572,8 @@ TEST_F(ScanAndReadInteTest, TestLancePythonCompatibility)
{
arrow::field("f_string", arrow::utf8()),
arrow::field("f_binary", arrow::binary()),
arrow::field("f_date", arrow::date32()),
+ arrow::field("f_time_0", arrow::time32(arrow::TimeUnit::MILLI)),
+ arrow::field("f_time_3", arrow::time32(arrow::TimeUnit::MILLI)),
arrow::field("f_ts_0", arrow::timestamp(arrow::TimeUnit::SECOND)),
arrow::field("f_ts_3", arrow::timestamp(arrow::TimeUnit::MILLI)),
arrow::field("f_ts_6", arrow::timestamp(arrow::TimeUnit::MICRO)),
@@ -3586,10 +3590,10 @@ TEST_F(ScanAndReadInteTest,
TestLancePythonCompatibility) {
};
auto expected = std::make_shared<arrow::ChunkedArray>(
arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([
-[0, 1, true, -5, -1000, 10000000001, 1.25, -2.5, "value-1",
"\u0000\u0001\u0002\u007f", -1, "1970-01-01 00:00:01", "1970-01-01
00:00:01.123", "1970-01-01 00:00:01.123456", "1970-01-01 00:00:01.123456789",
"-3", "1.25", "12345678901234567.89",
"12345678901234567890.123456789012345678", [1, null, -1], [[1, null], null,
[]], [1, "row-1"], [1.0, -1.5, 0.25]],
-[0, 2, null, null, null, null, null, null, null, null, null, null, null, null,
null, null, null, null, null, null, null, [null, null], null],
-[0, 10, false, -5, -1000, 10000000010, 1.25, -2.5, "",
"\u0000\u0001\u0002\u007f", 20000, "1970-01-01 00:00:01", "1970-01-01
00:00:01.123", "1970-01-01 00:00:01.123456", "1970-01-01 00:00:01.123456789",
"-3", "1.25", "12345678901234567.89",
"12345678901234567890.123456789012345678", [], [[10, null], null, []], [10,
"row-10"], [10.0, -1.5, 0.25]],
-[0, 11, true, -5, -1000, 10000000011, 1.25, -2.5, "value-11",
"\u0000\u0001\u0002\u007f", 20000, "1970-01-01 00:00:01", "1970-01-01
00:00:01.123", "1970-01-01 00:00:01.123456", "1970-01-01 00:00:01.123456789",
"-3", "1.25", "12345678901234567.89",
"12345678901234567890.123456789012345678", [11, null, -11], [[11, null], null,
[]], [11, "row-11"], [11.0, -1.5, 0.25]]
+[0, 1, true, -5, -1000, 10000000001, 1.25, -2.5, "value-1",
"\u0000\u0001\u0002\u007f", -1, 0, 86399999, "1970-01-01 00:00:01", "1970-01-01
00:00:01.123", "1970-01-01 00:00:01.123456", "1970-01-01 00:00:01.123456789",
"-3", "1.25", "12345678901234567.89",
"12345678901234567890.123456789012345678", [1, null, -1], [[1, null], null,
[]], [1, "row-1"], [1.0, -1.5, 0.25]],
+[0, 2, null, null, null, null, null, null, null, null, null, null, null, null,
null, null, null, null, null, null, null, null, null, [null, null], null],
+[0, 10, false, -5, -1000, 10000000010, 1.25, -2.5, "",
"\u0000\u0001\u0002\u007f", 20000, 45296000, 1123, "1970-01-01 00:00:01",
"1970-01-01 00:00:01.123", "1970-01-01 00:00:01.123456", "1970-01-01
00:00:01.123456789", "-3", "1.25", "12345678901234567.89",
"12345678901234567890.123456789012345678", [], [[10, null], null, []], [10,
"row-10"], [10.0, -1.5, 0.25]],
+[0, 11, true, -5, -1000, 10000000011, 1.25, -2.5, "value-11",
"\u0000\u0001\u0002\u007f", 20000, 86399000, 45296789, "1970-01-01 00:00:01",
"1970-01-01 00:00:01.123", "1970-01-01 00:00:01.123456", "1970-01-01
00:00:01.123456789", "-3", "1.25", "12345678901234567.89",
"12345678901234567890.123456789012345678", [11, null, -11], [[11, null], null,
[]], [11, "row-11"], [11.0, -1.5, 0.25]]
])")
.ValueOrDie());
const std::string table_path =
@@ -3623,7 +3627,7 @@ TEST_F(ScanAndReadInteTest, TestLancePythonCompatibility)
{
check(2, /*projection=*/{}, expected);
auto projected = std::make_shared<arrow::ChunkedArray>(
arrow::ipc::internal::json::ArrayFromJSON(
- arrow::struct_({fields[0], fields[21], fields[1], fields[22],
fields[19]}), R"([
+ arrow::struct_({fields[0], fields[23], fields[1], fields[24],
fields[21]}), R"([
[0, [1, "row-1"], 1, [1.0, -1.5, 0.25], [1, null, -1]],
[0, [null, null], 2, null, null],
[0, [10, "row-10"], 10, [10.0, -1.5, 0.25], []],
diff --git a/test/inte/write_and_read_inte_test.cpp
b/test/inte/write_and_read_inte_test.cpp
index 441c6d81..8b36d3c7 100644
--- a/test/inte/write_and_read_inte_test.cpp
+++ b/test/inte/write_and_read_inte_test.cpp
@@ -5978,37 +5978,52 @@ TEST(VortexPrimaryKeyWriteInteTest,
TestPKWriteFinalize) {
R"([[0, "a", 1],[0, "b",
2]])"));
ASSERT_TRUE(success);
}
+#endif
+#if defined(PAIMON_ENABLE_LANCE) || defined(PAIMON_ENABLE_VORTEX)
// Regression: append-table finalization always extracts statistics, and TIME
has no statistics
// factory of its own, so the extractor must still yield one entry for a TIME
column.
-TEST(VortexTimeColumnWriteInteTest, TestAppendTimeColumnFinalize) {
- auto dir = UniqueTestDirectory::Create("local");
- ASSERT_NE(dir, nullptr);
- arrow::FieldVector fields = {arrow::field("f_time",
arrow::time32(arrow::TimeUnit::MILLI)),
- arrow::field("f1", arrow::int32())};
- auto schema = arrow::schema(fields);
- std::map<std::string, std::string> options = {
- {Options::FILE_FORMAT, "vortex"}, {Options::BUCKET, "-1"},
{Options::FILE_SYSTEM, "local"}};
- ASSERT_OK_AND_ASSIGN(auto helper, TestHelper::Create(dir->Str(), schema,
/*partition_keys=*/{},
- /*primary_keys=*/{},
options,
-
/*is_streaming_mode=*/false));
- int64_t commit_identifier = 0;
- ASSERT_OK_AND_ASSIGN(
- std::unique_ptr<RecordBatch> batch,
- TestHelper::MakeRecordBatch(arrow::struct_(fields), R"([[3723123,
1],[null, 2]])",
- /*partition_map=*/{}, /*bucket=*/0, {}));
- ASSERT_OK_AND_ASSIGN(auto commit_msgs,
- helper->WriteAndCommit(std::move(batch),
commit_identifier++,
-
/*expected_commit_messages=*/std::nullopt));
- arrow::FieldVector fields_with_row_kind = fields;
- fields_with_row_kind.insert(fields_with_row_kind.begin(),
- arrow::field("_VALUE_KIND", arrow::int8()));
- ASSERT_OK_AND_ASSIGN(std::vector<std::shared_ptr<Split>> splits,
- helper->NewScan(StartupMode::LatestFull(),
/*snapshot_id=*/std::nullopt));
- ASSERT_OK_AND_ASSIGN(bool success,
-
helper->ReadAndCheckResult(arrow::struct_(fields_with_row_kind), splits,
- R"([[0, 3723123, 1],[0,
null, 2]])"));
- ASSERT_TRUE(success);
+TEST(TimeColumnWriteInteTest, TestAppendTimeColumnFinalize) {
+ std::vector<std::string> file_formats;
+#ifdef PAIMON_ENABLE_LANCE
+ file_formats.push_back("lance");
+#endif
+#ifdef PAIMON_ENABLE_VORTEX
+ file_formats.push_back("vortex");
+#endif
+ for (const auto& file_format : file_formats) {
+ SCOPED_TRACE(file_format);
+ auto dir = UniqueTestDirectory::Create("local");
+ ASSERT_NE(dir, nullptr);
+ arrow::FieldVector fields = {arrow::field("f_time",
arrow::time32(arrow::TimeUnit::MILLI)),
+ arrow::field("f1", arrow::int32())};
+ auto schema = arrow::schema(fields);
+ std::map<std::string, std::string> options = {{Options::FILE_FORMAT,
file_format},
+ {Options::BUCKET, "-1"},
+ {Options::FILE_SYSTEM,
"local"}};
+ ASSERT_OK_AND_ASSIGN(auto helper,
+ TestHelper::Create(dir->Str(), schema,
/*partition_keys=*/{},
+ /*primary_keys=*/{}, options,
+ /*is_streaming_mode=*/false));
+ int64_t commit_identifier = 0;
+ ASSERT_OK_AND_ASSIGN(
+ std::unique_ptr<RecordBatch> batch,
+ TestHelper::MakeRecordBatch(arrow::struct_(fields), R"([[3723123,
1],[null, 2]])",
+ /*partition_map=*/{}, /*bucket=*/0,
{}));
+ ASSERT_OK_AND_ASSIGN(auto commit_msgs,
+ helper->WriteAndCommit(std::move(batch),
commit_identifier++,
+
/*expected_commit_messages=*/std::nullopt));
+ arrow::FieldVector fields_with_row_kind = fields;
+ fields_with_row_kind.insert(fields_with_row_kind.begin(),
+ arrow::field("_VALUE_KIND",
arrow::int8()));
+ ASSERT_OK_AND_ASSIGN(
+ std::vector<std::shared_ptr<Split>> splits,
+ helper->NewScan(StartupMode::LatestFull(),
/*snapshot_id=*/std::nullopt));
+ ASSERT_OK_AND_ASSIGN(
+ bool success,
helper->ReadAndCheckResult(arrow::struct_(fields_with_row_kind), splits,
+ R"([[0, 3723123, 1],[0,
null, 2]])"));
+ ASSERT_TRUE(success);
+ }
}
#endif
diff --git
a/test/test_data/lance/append_java_compat.db/append_java_compat/README.md
b/test/test_data/lance/append_java_compat.db/append_java_compat/README.md
index b80f6e9a..71ae1ef9 100644
--- a/test/test_data/lance/append_java_compat.db/append_java_compat/README.md
+++ b/test/test_data/lance/append_java_compat.db/append_java_compat/README.md
@@ -16,6 +16,8 @@ f_string STRING
f_binary BINARY(4)
f_varbinary BYTES
f_date DATE
+f_time_0 TIME(0)
+f_time_3 TIME(3)
f_ts_0 TIMESTAMP(0)
f_ts_3 TIMESTAMP(3)
f_ts_6 TIMESTAMP(6)
@@ -41,18 +43,20 @@ BatchTableWrite/BatchTableCommit APIs wrote two rows per
commit into two Lance d
with schema, Avro manifests and two snapshots. All four rows were read back
and verified in Java.
Values below follow schema order; timestamps are UTC and binary values are
hexadecimal.
-Add: (1, true, -5, -1000, 10000000001, 1.25, -2.5, "char0001", "value-1",
0x62696e31, 0x0001027f, 1969-12-31, 1970-01-01 00:00:01, 1970-01-01
00:00:01.123, 1970-01-01 00:00:01.123456, 1970-01-01 00:00:01.123456789, -3,
1.25, 12345678901234567.89, 12345678901234567890.123456789012345678, [1, NULL,
-1], [[1, NULL], NULL, []], (1, "required"), (1, "nullable"), [1.0, -1.5, 0.25])
-Add: (2, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, (NULL, NULL),
(NULL, NULL), NULL)
+Add: (1, true, -5, -1000, 10000000001, 1.25, -2.5, "char0001", "value-1",
0x62696e31, 0x0001027f, 1969-12-31, 00:00:00, 23:59:59.999, 1970-01-01
00:00:01, 1970-01-01 00:00:01.123, 1970-01-01 00:00:01.123456, 1970-01-01
00:00:01.123456789, -3, 1.25, 12345678901234567.89,
12345678901234567890.123456789012345678, [1, NULL, -1], [[1, NULL], NULL, []],
(1, "required"), (1, "nullable"), [1.0, -1.5, 0.25])
+Add: (2, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, (NULL,
NULL), (NULL, NULL), NULL)
Commit - snapshot 1 (2 rows)
-Add: (10, false, -5, -1000, 10000000010, 1.25, -2.5, "char0001", "",
0x62696e31, 0x0001027f, 2024-10-04, 1970-01-01 00:00:01, 1970-01-01
00:00:01.123, 1970-01-01 00:00:01.123456, 1970-01-01 00:00:01.123456789, -3,
1.25, 12345678901234567.89, 12345678901234567890.123456789012345678, [], [[10,
NULL], NULL, []], (10, "required"), (10, "nullable"), [10.0, -1.5, 0.25])
-Add: (11, true, -5, -1000, 10000000011, 1.25, -2.5, "char0001", "value-11",
0x62696e31, 0x0001027f, 2024-10-04, 1970-01-01 00:00:01, 1970-01-01
00:00:01.123, 1970-01-01 00:00:01.123456, 1970-01-01 00:00:01.123456789, -3,
1.25, 12345678901234567.89, 12345678901234567890.123456789012345678, [11, NULL,
-11], [[11, NULL], NULL, []], (11, "required"), (11, "nullable"), [11.0, -1.5,
0.25])
+Add: (10, false, -5, -1000, 10000000010, 1.25, -2.5, "char0001", "",
0x62696e31, 0x0001027f, 2024-10-04, 12:34:56, 00:00:01.123, 1970-01-01
00:00:01, 1970-01-01 00:00:01.123, 1970-01-01 00:00:01.123456, 1970-01-01
00:00:01.123456789, -3, 1.25, 12345678901234567.89,
12345678901234567890.123456789012345678, [], [[10, NULL], NULL, []], (10,
"required"), (10, "nullable"), [10.0, -1.5, 0.25])
+Add: (11, true, -5, -1000, 10000000011, 1.25, -2.5, "char0001", "value-11",
0x62696e31, 0x0001027f, 2024-10-04, 23:59:59, 12:34:56.789, 1970-01-01
00:00:01, 1970-01-01 00:00:01.123, 1970-01-01 00:00:01.123456, 1970-01-01
00:00:01.123456789, -3, 1.25, 12345678901234567.89,
12345678901234567890.123456789012345678, [11, NULL, -11], [[11, NULL], NULL,
[]], (11, "required"), (11, "nullable"), [11.0, -1.5, 0.25])
Commit - snapshot 2 (4 rows total)
-At this Java revision, nullable ROW declarations are accepted but actual null
ROW parents
+At the writer commit above, nullable ROW declarations are accepted but actual
null ROW parents
do not survive the Lance round trip (a separate probe read id=2's null parent
as non-null).
Both struct parents here are valid, including id=2 where their children are
NULL.
-TIME32 is omitted because the C++ table path does not support it; Java's Lance
validator
-rejects local-zoned timestamps, MAP, MULTISET, VARIANT and BLOB.
+Java writes both TIME columns to Lance as time32[ms] holding milliseconds of
the day, with
+no precision in the field metadata; the declared precision comes from the
table schema.
+At the writer commit above, Java's Lance validator rejects local-zoned
timestamps, MAP,
+MULTISET, VARIANT and BLOB, so this fixture omits them.
diff --git
a/test/test_data/lance/append_java_compat.db/append_java_compat/bucket-0/data-43b44278-48ac-4cb0-aac9-6ebe5bd79117-0.lance
b/test/test_data/lance/append_java_compat.db/append_java_compat/bucket-0/data-26e3a06f-8bc4-4bea-99c2-1146b8c0bccf-0.lance
similarity index 75%
rename from
test/test_data/lance/append_java_compat.db/append_java_compat/bucket-0/data-43b44278-48ac-4cb0-aac9-6ebe5bd79117-0.lance
rename to
test/test_data/lance/append_java_compat.db/append_java_compat/bucket-0/data-26e3a06f-8bc4-4bea-99c2-1146b8c0bccf-0.lance
index d95d1d23..baebf061 100644
Binary files
a/test/test_data/lance/append_java_compat.db/append_java_compat/bucket-0/data-43b44278-48ac-4cb0-aac9-6ebe5bd79117-0.lance
and
b/test/test_data/lance/append_java_compat.db/append_java_compat/bucket-0/data-26e3a06f-8bc4-4bea-99c2-1146b8c0bccf-0.lance
differ
diff --git
a/test/test_data/lance/append_java_compat.db/append_java_compat/bucket-0/data-d5304b98-0401-4a8f-88df-3af803c91e31-0.lance
b/test/test_data/lance/append_java_compat.db/append_java_compat/bucket-0/data-e2d161b4-2446-450b-9b59-2e880ced6aaa-0.lance
similarity index 78%
rename from
test/test_data/lance/append_java_compat.db/append_java_compat/bucket-0/data-d5304b98-0401-4a8f-88df-3af803c91e31-0.lance
rename to
test/test_data/lance/append_java_compat.db/append_java_compat/bucket-0/data-e2d161b4-2446-450b-9b59-2e880ced6aaa-0.lance
index d65dffbe..22c86f74 100644
Binary files
a/test/test_data/lance/append_java_compat.db/append_java_compat/bucket-0/data-d5304b98-0401-4a8f-88df-3af803c91e31-0.lance
and
b/test/test_data/lance/append_java_compat.db/append_java_compat/bucket-0/data-e2d161b4-2446-450b-9b59-2e880ced6aaa-0.lance
differ
diff --git
a/test/test_data/lance/append_java_compat.db/append_java_compat/manifest/manifest-f59efaf6-4cbd-458b-bd4d-c2eb80905d0a-0
b/test/test_data/lance/append_java_compat.db/append_java_compat/manifest/manifest-0dcbe4b0-b6a6-4cba-9c74-b6fa5db0ad61-0
similarity index 91%
rename from
test/test_data/lance/append_java_compat.db/append_java_compat/manifest/manifest-f59efaf6-4cbd-458b-bd4d-c2eb80905d0a-0
rename to
test/test_data/lance/append_java_compat.db/append_java_compat/manifest/manifest-0dcbe4b0-b6a6-4cba-9c74-b6fa5db0ad61-0
index 92427a91..4d08515b 100644
Binary files
a/test/test_data/lance/append_java_compat.db/append_java_compat/manifest/manifest-f59efaf6-4cbd-458b-bd4d-c2eb80905d0a-0
and
b/test/test_data/lance/append_java_compat.db/append_java_compat/manifest/manifest-0dcbe4b0-b6a6-4cba-9c74-b6fa5db0ad61-0
differ
diff --git
a/test/test_data/lance/append_java_compat.db/append_java_compat/manifest/manifest-ce7a697c-635e-420f-8014-af0166fa5eea-0
b/test/test_data/lance/append_java_compat.db/append_java_compat/manifest/manifest-428dd1a6-0756-44d3-939c-f39159964fbc-0
similarity index 91%
rename from
test/test_data/lance/append_java_compat.db/append_java_compat/manifest/manifest-ce7a697c-635e-420f-8014-af0166fa5eea-0
rename to
test/test_data/lance/append_java_compat.db/append_java_compat/manifest/manifest-428dd1a6-0756-44d3-939c-f39159964fbc-0
index f208ef70..c5702815 100644
Binary files
a/test/test_data/lance/append_java_compat.db/append_java_compat/manifest/manifest-ce7a697c-635e-420f-8014-af0166fa5eea-0
and
b/test/test_data/lance/append_java_compat.db/append_java_compat/manifest/manifest-428dd1a6-0756-44d3-939c-f39159964fbc-0
differ
diff --git
a/test/test_data/lance/append_java_compat.db/append_java_compat/manifest/manifest-list-c02b5f45-8d05-46ce-851b-07907154a8e6-1
b/test/test_data/lance/append_java_compat.db/append_java_compat/manifest/manifest-list-1a7f860c-eed7-4f3b-9134-dd5f12b8a7d3-0
similarity index 86%
rename from
test/test_data/lance/append_java_compat.db/append_java_compat/manifest/manifest-list-c02b5f45-8d05-46ce-851b-07907154a8e6-1
rename to
test/test_data/lance/append_java_compat.db/append_java_compat/manifest/manifest-list-1a7f860c-eed7-4f3b-9134-dd5f12b8a7d3-0
index 4f1760c6..b20eaa6d 100644
Binary files
a/test/test_data/lance/append_java_compat.db/append_java_compat/manifest/manifest-list-c02b5f45-8d05-46ce-851b-07907154a8e6-1
and
b/test/test_data/lance/append_java_compat.db/append_java_compat/manifest/manifest-list-1a7f860c-eed7-4f3b-9134-dd5f12b8a7d3-0
differ
diff --git
a/test/test_data/lance/append_java_compat.db/append_java_compat/manifest/manifest-list-486973a3-9e8b-4084-8f64-651660b658e0-1
b/test/test_data/lance/append_java_compat.db/append_java_compat/manifest/manifest-list-1a7f860c-eed7-4f3b-9134-dd5f12b8a7d3-1
similarity index 86%
rename from
test/test_data/lance/append_java_compat.db/append_java_compat/manifest/manifest-list-486973a3-9e8b-4084-8f64-651660b658e0-1
rename to
test/test_data/lance/append_java_compat.db/append_java_compat/manifest/manifest-list-1a7f860c-eed7-4f3b-9134-dd5f12b8a7d3-1
index 13766e16..77e608f6 100644
Binary files
a/test/test_data/lance/append_java_compat.db/append_java_compat/manifest/manifest-list-486973a3-9e8b-4084-8f64-651660b658e0-1
and
b/test/test_data/lance/append_java_compat.db/append_java_compat/manifest/manifest-list-1a7f860c-eed7-4f3b-9134-dd5f12b8a7d3-1
differ
diff --git
a/test/test_data/lance/append_java_compat.db/append_java_compat/manifest/manifest-list-c02b5f45-8d05-46ce-851b-07907154a8e6-0
b/test/test_data/lance/append_java_compat.db/append_java_compat/manifest/manifest-list-24709406-a874-4961-b200-879ceba4452c-0
similarity index 95%
rename from
test/test_data/lance/append_java_compat.db/append_java_compat/manifest/manifest-list-c02b5f45-8d05-46ce-851b-07907154a8e6-0
rename to
test/test_data/lance/append_java_compat.db/append_java_compat/manifest/manifest-list-24709406-a874-4961-b200-879ceba4452c-0
index 96b43273..17888a46 100644
Binary files
a/test/test_data/lance/append_java_compat.db/append_java_compat/manifest/manifest-list-c02b5f45-8d05-46ce-851b-07907154a8e6-0
and
b/test/test_data/lance/append_java_compat.db/append_java_compat/manifest/manifest-list-24709406-a874-4961-b200-879ceba4452c-0
differ
diff --git
a/test/test_data/lance/append_java_compat.db/append_java_compat/manifest/manifest-list-486973a3-9e8b-4084-8f64-651660b658e0-0
b/test/test_data/lance/append_java_compat.db/append_java_compat/manifest/manifest-list-24709406-a874-4961-b200-879ceba4452c-1
similarity index 86%
rename from
test/test_data/lance/append_java_compat.db/append_java_compat/manifest/manifest-list-486973a3-9e8b-4084-8f64-651660b658e0-0
rename to
test/test_data/lance/append_java_compat.db/append_java_compat/manifest/manifest-list-24709406-a874-4961-b200-879ceba4452c-1
index 961f1f87..a338618f 100644
Binary files
a/test/test_data/lance/append_java_compat.db/append_java_compat/manifest/manifest-list-486973a3-9e8b-4084-8f64-651660b658e0-0
and
b/test/test_data/lance/append_java_compat.db/append_java_compat/manifest/manifest-list-24709406-a874-4961-b200-879ceba4452c-1
differ
diff --git
a/test/test_data/lance/append_java_compat.db/append_java_compat/schema/schema-0
b/test/test_data/lance/append_java_compat.db/append_java_compat/schema/schema-0
index 2d987449..2be1bf70 100644
---
a/test/test_data/lance/append_java_compat.db/append_java_compat/schema/schema-0
+++
b/test/test_data/lance/append_java_compat.db/append_java_compat/schema/schema-0
@@ -51,45 +51,53 @@
"type" : "DATE"
}, {
"id" : 12,
+ "name" : "f_time_0",
+ "type" : "TIME(0)"
+ }, {
+ "id" : 13,
+ "name" : "f_time_3",
+ "type" : "TIME(3)"
+ }, {
+ "id" : 14,
"name" : "f_ts_0",
"type" : "TIMESTAMP(0)"
}, {
- "id" : 13,
+ "id" : 15,
"name" : "f_ts_3",
"type" : "TIMESTAMP(3)"
}, {
- "id" : 14,
+ "id" : 16,
"name" : "f_ts_6",
"type" : "TIMESTAMP(6)"
}, {
- "id" : 15,
+ "id" : 17,
"name" : "f_ts_9",
"type" : "TIMESTAMP(9)"
}, {
- "id" : 16,
+ "id" : 18,
"name" : "f_decimal_1_0",
"type" : "DECIMAL(1, 0)"
}, {
- "id" : 17,
+ "id" : 19,
"name" : "f_decimal_18_2",
"type" : "DECIMAL(18, 2)"
}, {
- "id" : 18,
+ "id" : 20,
"name" : "f_decimal_19_2",
"type" : "DECIMAL(19, 2)"
}, {
- "id" : 19,
+ "id" : 21,
"name" : "f_decimal_38_18",
"type" : "DECIMAL(38, 18)"
}, {
- "id" : 20,
+ "id" : 22,
"name" : "f_array_int",
"type" : {
"type" : "ARRAY",
"element" : "INT"
}
}, {
- "id" : 21,
+ "id" : 23,
"name" : "f_array_array_int",
"type" : {
"type" : "ARRAY",
@@ -99,37 +107,37 @@
}
}
}, {
- "id" : 22,
+ "id" : 24,
"name" : "f_struct",
"type" : {
"type" : "ROW NOT NULL",
"fields" : [ {
- "id" : 23,
+ "id" : 25,
"name" : "number",
"type" : "INT"
}, {
- "id" : 24,
+ "id" : 26,
"name" : "label",
"type" : "STRING"
} ]
}
}, {
- "id" : 25,
+ "id" : 27,
"name" : "f_nullable_struct",
"type" : {
"type" : "ROW",
"fields" : [ {
- "id" : 26,
+ "id" : 28,
"name" : "number",
"type" : "INT"
}, {
- "id" : 27,
+ "id" : 29,
"name" : "label",
"type" : "STRING"
} ]
}
}, {
- "id" : 28,
+ "id" : 30,
"name" : "f_vector",
"type" : {
"type" : "VECTOR",
@@ -137,7 +145,7 @@
"length" : 3
}
} ],
- "highestFieldId" : 28,
+ "highestFieldId" : 30,
"partitionKeys" : [ ],
"primaryKeys" : [ ],
"options" : {
@@ -146,5 +154,5 @@
"file.format" : "lance",
"metadata.stats-mode" : "none"
},
- "timeMillis" : 1789217950502
+ "timeMillis" : 1793168650637
}
\ No newline at end of file
diff --git
a/test/test_data/lance/append_java_compat.db/append_java_compat/snapshot/snapshot-1
b/test/test_data/lance/append_java_compat.db/append_java_compat/snapshot/snapshot-1
index 2c51d03d..397bf481 100644
---
a/test/test_data/lance/append_java_compat.db/append_java_compat/snapshot/snapshot-1
+++
b/test/test_data/lance/append_java_compat.db/append_java_compat/snapshot/snapshot-1
@@ -1,17 +1,17 @@
{
"version" : 3,
- "uuid" : "fb7e0704-7508-46bf-8a23-f4774481757c",
+ "uuid" : "a5063bd4-985c-4cde-976f-4379dbbc7ceb",
"id" : 1,
"schemaId" : 0,
- "baseManifestList" : "manifest-list-c02b5f45-8d05-46ce-851b-07907154a8e6-0",
+ "baseManifestList" : "manifest-list-24709406-a874-4961-b200-879ceba4452c-0",
"baseManifestListSize" : 1006,
- "deltaManifestList" : "manifest-list-c02b5f45-8d05-46ce-851b-07907154a8e6-1",
+ "deltaManifestList" : "manifest-list-24709406-a874-4961-b200-879ceba4452c-1",
"deltaManifestListSize" : 1110,
- "commitUser" : "ec878fff-cb67-41b2-a424-5b224e88b514",
+ "commitUser" : "c29d2747-b5d7-44ef-9b72-8b2fe1ca1d31",
"writerVersion" :
"java-2.1-SNAPSHOT-63ffd403648e1cf9848e32bb29a398c1b3b16ab1",
"commitIdentifier" : 9223372036854775807,
"commitKind" : "APPEND",
- "timeMillis" : 1789217951996,
+ "timeMillis" : 1793168652996,
"totalRecordCount" : 2,
"deltaRecordCount" : 2,
"nextRowId" : 0
diff --git
a/test/test_data/lance/append_java_compat.db/append_java_compat/snapshot/snapshot-2
b/test/test_data/lance/append_java_compat.db/append_java_compat/snapshot/snapshot-2
index 3ef365f2..ab9191eb 100644
---
a/test/test_data/lance/append_java_compat.db/append_java_compat/snapshot/snapshot-2
+++
b/test/test_data/lance/append_java_compat.db/append_java_compat/snapshot/snapshot-2
@@ -1,17 +1,17 @@
{
"version" : 3,
- "uuid" : "9260373b-900c-43b2-b45b-f741cff6fe35",
+ "uuid" : "e069c81e-0707-43f5-8e9d-027e487bb3b4",
"id" : 2,
"schemaId" : 0,
- "baseManifestList" : "manifest-list-486973a3-9e8b-4084-8f64-651660b658e0-0",
+ "baseManifestList" : "manifest-list-1a7f860c-eed7-4f3b-9134-dd5f12b8a7d3-0",
"baseManifestListSize" : 1110,
- "deltaManifestList" : "manifest-list-486973a3-9e8b-4084-8f64-651660b658e0-1",
+ "deltaManifestList" : "manifest-list-1a7f860c-eed7-4f3b-9134-dd5f12b8a7d3-1",
"deltaManifestListSize" : 1113,
- "commitUser" : "ede562c1-31df-47bd-b685-36c36f49ef2e",
+ "commitUser" : "25bacb07-4a95-4cd5-94a3-36a045c65466",
"writerVersion" :
"java-2.1-SNAPSHOT-63ffd403648e1cf9848e32bb29a398c1b3b16ab1",
"commitIdentifier" : 9223372036854775807,
"commitKind" : "APPEND",
- "timeMillis" : 1789217952098,
+ "timeMillis" : 1793168653159,
"totalRecordCount" : 4,
"deltaRecordCount" : 2,
"nextRowId" : 0
diff --git
a/test/test_data/lance/append_python_compat.db/append_python_compat/README.md
b/test/test_data/lance/append_python_compat.db/append_python_compat/README.md
index bf2f870c..bddeaee7 100644
---
a/test/test_data/lance/append_python_compat.db/append_python_compat/README.md
+++
b/test/test_data/lance/append_python_compat.db/append_python_compat/README.md
@@ -21,6 +21,8 @@ f_double DOUBLE
f_string STRING
f_binary BYTES
f_date DATE
+f_time_0 TIME(0)
+f_time_3 TIME(3)
f_ts_0 TIMESTAMP(0)
f_ts_3 TIMESTAMP(3)
f_ts_6 TIMESTAMP(6)
@@ -43,16 +45,20 @@ metadata.stats-mode = none
Values below follow schema order; timestamps are UTC and binary values are
hexadecimal.
The second row has NULL in every field except id and the valid ROW parent with
two NULL children.
-Add: (1, true, -5, -1000, 10000000001, 1.25, -2.5, "value-1", 0x0001027f,
1969-12-31, 1970-01-01 00:00:01, 1970-01-01 00:00:01.123, 1970-01-01
00:00:01.123456, 1970-01-01 00:00:01.123456789, -3, 1.25, 12345678901234567.89,
12345678901234567890.123456789012345678, [1, NULL, -1], [[1, NULL], NULL, []],
(1, "row-1"), [1.0, -1.5, 0.25])
-Add: (2, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, (NULL, NULL), NULL)
+Add: (1, true, -5, -1000, 10000000001, 1.25, -2.5, "value-1", 0x0001027f,
1969-12-31, 00:00:00, 23:59:59.999, 1970-01-01 00:00:01, 1970-01-01
00:00:01.123, 1970-01-01 00:00:01.123456, 1970-01-01 00:00:01.123456789, -3,
1.25, 12345678901234567.89, 12345678901234567890.123456789012345678, [1, NULL,
-1], [[1, NULL], NULL, []], (1, "row-1"), [1.0, -1.5, 0.25])
+Add: (2, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, (NULL, NULL), NULL)
Commit - snapshot 1 (2 rows)
-Add: (10, false, -5, -1000, 10000000010, 1.25, -2.5, "", 0x0001027f,
2024-10-04, 1970-01-01 00:00:01, 1970-01-01 00:00:01.123, 1970-01-01
00:00:01.123456, 1970-01-01 00:00:01.123456789, -3, 1.25, 12345678901234567.89,
12345678901234567890.123456789012345678, [], [[10, NULL], NULL, []], (10,
"row-10"), [10.0, -1.5, 0.25])
-Add: (11, true, -5, -1000, 10000000011, 1.25, -2.5, "value-11", 0x0001027f,
2024-10-04, 1970-01-01 00:00:01, 1970-01-01 00:00:01.123, 1970-01-01
00:00:01.123456, 1970-01-01 00:00:01.123456789, -3, 1.25, 12345678901234567.89,
12345678901234567890.123456789012345678, [11, NULL, -11], [[11, NULL], NULL,
[]], (11, "row-11"), [11.0, -1.5, 0.25])
+Add: (10, false, -5, -1000, 10000000010, 1.25, -2.5, "", 0x0001027f,
2024-10-04, 12:34:56, 00:00:01.123, 1970-01-01 00:00:01, 1970-01-01
00:00:01.123, 1970-01-01 00:00:01.123456, 1970-01-01 00:00:01.123456789, -3,
1.25, 12345678901234567.89, 12345678901234567890.123456789012345678, [], [[10,
NULL], NULL, []], (10, "row-10"), [10.0, -1.5, 0.25])
+Add: (11, true, -5, -1000, 10000000011, 1.25, -2.5, "value-11", 0x0001027f,
2024-10-04, 23:59:59, 12:34:56.789, 1970-01-01 00:00:01, 1970-01-01
00:00:01.123, 1970-01-01 00:00:01.123456, 1970-01-01 00:00:01.123456789, -3,
1.25, 12345678901234567.89, 12345678901234567890.123456789012345678, [11, NULL,
-11], [[11, NULL], NULL, []], (11, "row-11"), [11.0, -1.5, 0.25])
Commit - snapshot 2 (4 rows total)
+Paimon Python maps every pyarrow time type to TIME(0), so f_time_3 was
declared TIME(3)
+directly in the Paimon schema. Paimon Python writes both TIME columns to Lance
as time32[ms]
+holding milliseconds of the day, with no field metadata; the declared
precision comes from
+the table schema.
Null and empty lists, null list elements, an empty string,
binary zero bytes, null scalars, null ROW children and a null vector are
included.
ROW parents are always valid: this fixture does not claim null-parent ROW
compatibility.
diff --git
a/test/test_data/lance/append_python_compat.db/append_python_compat/bucket-0/data-85a6f7a6-c63f-451f-a4d7-3c947244ff8e-0.lance
b/test/test_data/lance/append_python_compat.db/append_python_compat/bucket-0/data-24cc7753-16ed-451d-8899-984adb2de133-0.lance
similarity index 71%
rename from
test/test_data/lance/append_python_compat.db/append_python_compat/bucket-0/data-85a6f7a6-c63f-451f-a4d7-3c947244ff8e-0.lance
rename to
test/test_data/lance/append_python_compat.db/append_python_compat/bucket-0/data-24cc7753-16ed-451d-8899-984adb2de133-0.lance
index 37aaea1d..693e37e9 100644
Binary files
a/test/test_data/lance/append_python_compat.db/append_python_compat/bucket-0/data-85a6f7a6-c63f-451f-a4d7-3c947244ff8e-0.lance
and
b/test/test_data/lance/append_python_compat.db/append_python_compat/bucket-0/data-24cc7753-16ed-451d-8899-984adb2de133-0.lance
differ
diff --git
a/test/test_data/lance/append_python_compat.db/append_python_compat/bucket-0/data-6f71db8f-5863-47f9-847a-acfdac671bb2-0.lance
b/test/test_data/lance/append_python_compat.db/append_python_compat/bucket-0/data-27b4eee6-062d-4781-b762-d18f2f1a90c8-0.lance
similarity index 75%
rename from
test/test_data/lance/append_python_compat.db/append_python_compat/bucket-0/data-6f71db8f-5863-47f9-847a-acfdac671bb2-0.lance
rename to
test/test_data/lance/append_python_compat.db/append_python_compat/bucket-0/data-27b4eee6-062d-4781-b762-d18f2f1a90c8-0.lance
index a1cc70ab..0891df17 100644
Binary files
a/test/test_data/lance/append_python_compat.db/append_python_compat/bucket-0/data-6f71db8f-5863-47f9-847a-acfdac671bb2-0.lance
and
b/test/test_data/lance/append_python_compat.db/append_python_compat/bucket-0/data-27b4eee6-062d-4781-b762-d18f2f1a90c8-0.lance
differ
diff --git
a/test/test_data/lance/append_python_compat.db/append_python_compat/manifest/manifest-ea38ae03-9503-4c9f-95ee-0ccc0fe72767-0
b/test/test_data/lance/append_python_compat.db/append_python_compat/manifest/manifest-63284fb6-53d2-4c5d-8719-a411c920e155-0
similarity index 92%
rename from
test/test_data/lance/append_python_compat.db/append_python_compat/manifest/manifest-ea38ae03-9503-4c9f-95ee-0ccc0fe72767-0
rename to
test/test_data/lance/append_python_compat.db/append_python_compat/manifest/manifest-63284fb6-53d2-4c5d-8719-a411c920e155-0
index 27f94dab..bb466bdd 100644
Binary files
a/test/test_data/lance/append_python_compat.db/append_python_compat/manifest/manifest-ea38ae03-9503-4c9f-95ee-0ccc0fe72767-0
and
b/test/test_data/lance/append_python_compat.db/append_python_compat/manifest/manifest-63284fb6-53d2-4c5d-8719-a411c920e155-0
differ
diff --git
a/test/test_data/lance/append_python_compat.db/append_python_compat/manifest/manifest-6b2e90c5-eabd-41e9-9a02-099ecdb7fede-0
b/test/test_data/lance/append_python_compat.db/append_python_compat/manifest/manifest-a10d7552-9602-4543-bc69-152765c1c59c-0
similarity index 92%
rename from
test/test_data/lance/append_python_compat.db/append_python_compat/manifest/manifest-6b2e90c5-eabd-41e9-9a02-099ecdb7fede-0
rename to
test/test_data/lance/append_python_compat.db/append_python_compat/manifest/manifest-a10d7552-9602-4543-bc69-152765c1c59c-0
index 28895ae5..708ee625 100644
Binary files
a/test/test_data/lance/append_python_compat.db/append_python_compat/manifest/manifest-6b2e90c5-eabd-41e9-9a02-099ecdb7fede-0
and
b/test/test_data/lance/append_python_compat.db/append_python_compat/manifest/manifest-a10d7552-9602-4543-bc69-152765c1c59c-0
differ
diff --git
a/test/test_data/lance/append_python_compat.db/append_python_compat/manifest/manifest-list-4fbd11a3-f7c7-45e5-8613-66ffd257ebdd-0
b/test/test_data/lance/append_python_compat.db/append_python_compat/manifest/manifest-list-cb00c974-4152-4c3c-aa64-7910d413b4d6-0
similarity index 96%
rename from
test/test_data/lance/append_python_compat.db/append_python_compat/manifest/manifest-list-4fbd11a3-f7c7-45e5-8613-66ffd257ebdd-0
rename to
test/test_data/lance/append_python_compat.db/append_python_compat/manifest/manifest-list-cb00c974-4152-4c3c-aa64-7910d413b4d6-0
index 87d718a2..d33af15b 100644
Binary files
a/test/test_data/lance/append_python_compat.db/append_python_compat/manifest/manifest-list-4fbd11a3-f7c7-45e5-8613-66ffd257ebdd-0
and
b/test/test_data/lance/append_python_compat.db/append_python_compat/manifest/manifest-list-cb00c974-4152-4c3c-aa64-7910d413b4d6-0
differ
diff --git
a/test/test_data/lance/append_python_compat.db/append_python_compat/manifest/manifest-list-4fbd11a3-f7c7-45e5-8613-66ffd257ebdd-1
b/test/test_data/lance/append_python_compat.db/append_python_compat/manifest/manifest-list-cb00c974-4152-4c3c-aa64-7910d413b4d6-1
similarity index 85%
rename from
test/test_data/lance/append_python_compat.db/append_python_compat/manifest/manifest-list-4fbd11a3-f7c7-45e5-8613-66ffd257ebdd-1
rename to
test/test_data/lance/append_python_compat.db/append_python_compat/manifest/manifest-list-cb00c974-4152-4c3c-aa64-7910d413b4d6-1
index 9b5cf56f..4e3ebb9d 100644
Binary files
a/test/test_data/lance/append_python_compat.db/append_python_compat/manifest/manifest-list-4fbd11a3-f7c7-45e5-8613-66ffd257ebdd-1
and
b/test/test_data/lance/append_python_compat.db/append_python_compat/manifest/manifest-list-cb00c974-4152-4c3c-aa64-7910d413b4d6-1
differ
diff --git
a/test/test_data/lance/append_python_compat.db/append_python_compat/manifest/manifest-list-9e90d5ee-a3d2-4d64-ad27-2e15436b5391-0
b/test/test_data/lance/append_python_compat.db/append_python_compat/manifest/manifest-list-ce87ee4f-fee3-4059-b37f-a1f295a72a7c-0
similarity index 85%
rename from
test/test_data/lance/append_python_compat.db/append_python_compat/manifest/manifest-list-9e90d5ee-a3d2-4d64-ad27-2e15436b5391-0
rename to
test/test_data/lance/append_python_compat.db/append_python_compat/manifest/manifest-list-ce87ee4f-fee3-4059-b37f-a1f295a72a7c-0
index 98c1af25..0b356840 100644
Binary files
a/test/test_data/lance/append_python_compat.db/append_python_compat/manifest/manifest-list-9e90d5ee-a3d2-4d64-ad27-2e15436b5391-0
and
b/test/test_data/lance/append_python_compat.db/append_python_compat/manifest/manifest-list-ce87ee4f-fee3-4059-b37f-a1f295a72a7c-0
differ
diff --git
a/test/test_data/lance/append_python_compat.db/append_python_compat/manifest/manifest-list-9e90d5ee-a3d2-4d64-ad27-2e15436b5391-1
b/test/test_data/lance/append_python_compat.db/append_python_compat/manifest/manifest-list-ce87ee4f-fee3-4059-b37f-a1f295a72a7c-1
similarity index 85%
rename from
test/test_data/lance/append_python_compat.db/append_python_compat/manifest/manifest-list-9e90d5ee-a3d2-4d64-ad27-2e15436b5391-1
rename to
test/test_data/lance/append_python_compat.db/append_python_compat/manifest/manifest-list-ce87ee4f-fee3-4059-b37f-a1f295a72a7c-1
index 4ead24d6..28c5a857 100644
Binary files
a/test/test_data/lance/append_python_compat.db/append_python_compat/manifest/manifest-list-9e90d5ee-a3d2-4d64-ad27-2e15436b5391-1
and
b/test/test_data/lance/append_python_compat.db/append_python_compat/manifest/manifest-list-ce87ee4f-fee3-4059-b37f-a1f295a72a7c-1
differ
diff --git
a/test/test_data/lance/append_python_compat.db/append_python_compat/schema/schema-0
b/test/test_data/lance/append_python_compat.db/append_python_compat/schema/schema-0
index 05be14c1..7243667d 100644
---
a/test/test_data/lance/append_python_compat.db/append_python_compat/schema/schema-0
+++
b/test/test_data/lance/append_python_compat.db/append_python_compat/schema/schema-0
@@ -54,46 +54,56 @@
},
{
"id": 10,
+ "name": "f_time_0",
+ "type": "TIME(0)"
+ },
+ {
+ "id": 11,
+ "name": "f_time_3",
+ "type": "TIME(3)"
+ },
+ {
+ "id": 12,
"name": "f_ts_0",
"type": "TIMESTAMP(0)"
},
{
- "id": 11,
+ "id": 13,
"name": "f_ts_3",
"type": "TIMESTAMP(3)"
},
{
- "id": 12,
+ "id": 14,
"name": "f_ts_6",
"type": "TIMESTAMP(6)"
},
{
- "id": 13,
+ "id": 15,
"name": "f_ts_9",
"type": "TIMESTAMP(9)"
},
{
- "id": 14,
+ "id": 16,
"name": "f_decimal_1_0",
"type": "DECIMAL(1, 0)"
},
{
- "id": 15,
+ "id": 17,
"name": "f_decimal_18_2",
"type": "DECIMAL(18, 2)"
},
{
- "id": 16,
+ "id": 18,
"name": "f_decimal_19_2",
"type": "DECIMAL(19, 2)"
},
{
- "id": 17,
+ "id": 19,
"name": "f_decimal_38_18",
"type": "DECIMAL(38, 18)"
},
{
- "id": 18,
+ "id": 20,
"name": "f_array_int",
"type": {
"type": "ARRAY",
@@ -102,7 +112,7 @@
}
},
{
- "id": 19,
+ "id": 21,
"name": "f_array_array_int",
"type": {
"type": "ARRAY",
@@ -115,18 +125,18 @@
}
},
{
- "id": 20,
+ "id": 22,
"name": "f_struct",
"type": {
"type": "ROW",
"fields": [
{
- "id": 21,
+ "id": 23,
"name": "number",
"type": "INT"
},
{
- "id": 22,
+ "id": 24,
"name": "label",
"type": "STRING"
}
@@ -135,7 +145,7 @@
}
},
{
- "id": 23,
+ "id": 25,
"name": "f_vector",
"type": {
"type": "VECTOR",
@@ -145,7 +155,7 @@
}
}
],
- "highestFieldId": 23,
+ "highestFieldId": 25,
"partitionKeys": [],
"primaryKeys": [],
"options": {
@@ -155,5 +165,5 @@
"metadata.stats-mode": "none"
},
"comment": null,
- "timeMillis": 1789410866915
+ "timeMillis": 1793168823492
}
\ No newline at end of file
diff --git
a/test/test_data/lance/append_python_compat.db/append_python_compat/snapshot/snapshot-1
b/test/test_data/lance/append_python_compat.db/append_python_compat/snapshot/snapshot-1
index 105c967e..ff267aba 100644
---
a/test/test_data/lance/append_python_compat.db/append_python_compat/snapshot/snapshot-1
+++
b/test/test_data/lance/append_python_compat.db/append_python_compat/snapshot/snapshot-1
@@ -2,14 +2,14 @@
"version": 3,
"id": 1,
"schemaId": 0,
- "baseManifestList": "manifest-list-4fbd11a3-f7c7-45e5-8613-66ffd257ebdd-0",
- "deltaManifestList": "manifest-list-4fbd11a3-f7c7-45e5-8613-66ffd257ebdd-1",
+ "baseManifestList": "manifest-list-cb00c974-4152-4c3c-aa64-7910d413b4d6-0",
+ "deltaManifestList": "manifest-list-cb00c974-4152-4c3c-aa64-7910d413b4d6-1",
"totalRecordCount": 2,
"deltaRecordCount": 2,
- "commitUser": "020804d7-bcfa-4b8d-9c9c-176b700109e2",
+ "commitUser": "69ff4ed6-dff4-4e58-acb0-da04946ff985",
"commitIdentifier": 9223372036854775807,
"commitKind": "APPEND",
- "timeMillis": 1789410866939,
- "uuid": "5d3163c5-e36c-4e33-b690-fc8fb5f6438d",
+ "timeMillis": 1793168823524,
+ "uuid": "7a678363-830b-45f3-913a-8666e6c0ac02",
"writerVersion": "python-2.1.dev-63ffd403648e1cf9848e32bb29a398c1b3b16ab1"
}
\ No newline at end of file
diff --git
a/test/test_data/lance/append_python_compat.db/append_python_compat/snapshot/snapshot-2
b/test/test_data/lance/append_python_compat.db/append_python_compat/snapshot/snapshot-2
index a641eacf..9ea45567 100644
---
a/test/test_data/lance/append_python_compat.db/append_python_compat/snapshot/snapshot-2
+++
b/test/test_data/lance/append_python_compat.db/append_python_compat/snapshot/snapshot-2
@@ -2,14 +2,14 @@
"version": 3,
"id": 2,
"schemaId": 0,
- "baseManifestList": "manifest-list-9e90d5ee-a3d2-4d64-ad27-2e15436b5391-0",
- "deltaManifestList": "manifest-list-9e90d5ee-a3d2-4d64-ad27-2e15436b5391-1",
+ "baseManifestList": "manifest-list-ce87ee4f-fee3-4059-b37f-a1f295a72a7c-0",
+ "deltaManifestList": "manifest-list-ce87ee4f-fee3-4059-b37f-a1f295a72a7c-1",
"totalRecordCount": 4,
"deltaRecordCount": 2,
- "commitUser": "c06f85ce-2b7e-449c-af6c-b7b5b0e50dce",
+ "commitUser": "e4715de3-7139-413b-91fe-ca187152a9dd",
"commitIdentifier": 9223372036854775807,
"commitKind": "APPEND",
- "timeMillis": 1789410866974,
- "uuid": "f7a0e42b-72a4-4785-8ec3-bbd59f1ea435",
+ "timeMillis": 1793168823541,
+ "uuid": "35aa33df-3390-4d54-95e3-8ef97e38f6ea",
"writerVersion": "python-2.1.dev-63ffd403648e1cf9848e32bb29a398c1b3b16ab1"
}
\ No newline at end of file
