This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/paimon-rust.git
The following commit(s) were added to refs/heads/main by this push:
new 54a4971d fix(core): preserve local timezone semantics for
TIMESTAMP_LTZ schemas (#834)
54a4971d is described below
commit 54a4971dd339cf9141d9251e35792da490e4dd05
Author: Jingsong Lee <[email protected]>
AuthorDate: Tue Sep 15 12:55:33 2026 +0800
fix(core): preserve local timezone semantics for TIMESTAMP_LTZ schemas
(#834)
---
bindings/python/tests/test_write.py | 34 +++++++++++++++++++++++++++++++++-
crates/paimon/src/spec/types.rs | 37 +++++++++++++++++++++++++++++++++++--
2 files changed, 68 insertions(+), 3 deletions(-)
diff --git a/bindings/python/tests/test_write.py
b/bindings/python/tests/test_write.py
index 8d524354..ed5be8b3 100644
--- a/bindings/python/tests/test_write.py
+++ b/bindings/python/tests/test_write.py
@@ -15,8 +15,9 @@
# specific language governing permissions and limitations
# under the License.
+import json
import tempfile
-from datetime import datetime
+from datetime import datetime, timezone
import pyarrow as pa
import pytest
@@ -281,3 +282,34 @@ def
test_timestamp_partition_writes_and_reads_use_java_paths(tmp_path, legacy, p
assert sum(split.row_count() for split in plan.splits()) == 2
rows = pa.Table.from_batches(ctx.sql("SELECT id, ts FROM
paimon.wdb.t")).sort_by("id")
assert rows.to_pydict() == {"id": [1, 2], "ts": [value, value]}
+
+
[email protected]("precision,unit,micros,fraction", [(3, "ms", 120000,
"120"), (6, "us", 120100, "120100")])
+def test_ltz_schema_alias_write_roundtrip(tmp_path, precision, unit, micros,
fraction):
+ ctx = SQLContext()
+ ctx.register_catalog("paimon", {"warehouse": str(tmp_path)})
+ ctx.sql("CREATE SCHEMA paimon.wdb")
+ ctx.sql(
+ "CREATE TABLE paimon.wdb.t (id INT, ts TIMESTAMP({}) WITH TIME ZONE)
PARTITIONED BY (ts) "
+ "WITH ('partition.legacy-name' = 'true')".format(precision)
+ )
+ # PyPaimon persists this alias; reloading must retain its timezone
semantics.
+ schema_path = next(tmp_path.rglob("schema-0"))
+ schema = json.loads(schema_path.read_text())
+ schema["fields"][1]["type"] = "TIMESTAMP_LTZ({})".format(precision)
+ schema_path.write_text(json.dumps(schema))
+ table = _get_table(str(tmp_path))
+ assert "LocalZonedTimestamp" in table.schema().fields()[1].field_type()
+ value = datetime(2026, 9, 15, 20, 0, 0, micros, tzinfo=timezone.utc)
+ arrow_schema = pa.schema([("id", pa.int32()), ("ts", pa.timestamp(unit,
tz="UTC"))])
+ builder = table.new_write_builder()
+ write = builder.new_write()
+ write.write_arrow(pa.record_batch([[1], [value]], schema=arrow_schema))
+ builder.new_commit().commit(write.prepare_commit())
+ files = list(tmp_path.rglob("data-*.parquet"))
+ assert len(files) == 1
+ assert files[0].parent.parent.name == "ts=2026-09-15T20%3A00%3A00." +
fraction
+ table = _get_table(str(tmp_path))
+ assert sum(split.row_count() for split in
table.new_read_builder().new_scan().plan().splits()) == 1
+ rows = pa.Table.from_batches(ctx.sql("SELECT id, ts FROM paimon.wdb.t"))
+ assert rows.to_pydict() == {"id": [1], "ts": [value]}
diff --git a/crates/paimon/src/spec/types.rs b/crates/paimon/src/spec/types.rs
index 20f71dd4..e159d6e8 100644
--- a/crates/paimon/src/spec/types.rs
+++ b/crates/paimon/src/spec/types.rs
@@ -1105,7 +1105,10 @@ impl FromStr for LocalZonedTimestampType {
type Err = Error;
fn from_str(s: &str) -> Result<Self, Self::Err> {
- if !s.starts_with(serde_utils::LocalZonedTimestamp::NAME) ||
!s.contains("WITH") {
+ // PyPaimon persists the TIMESTAMP_LTZ(p) alias for this type.
+ let alias = s.starts_with("TIMESTAMP_LTZ(");
+ if !alias && (!s.starts_with(serde_utils::LocalZonedTimestamp::NAME)
|| !s.contains("WITH"))
+ {
return DataTypeInvalidSnafu {
message:
"Invalid LocalZonedTimestamp type. Expected string to
start with 'TIMESTAMP'.",
@@ -1342,7 +1345,10 @@ impl FromStr for TimestampType {
type Err = Error;
fn from_str(s: &str) -> Result<Self, Self::Err> {
- if !s.starts_with(serde_utils::TIMESTAMP::NAME) {
+ if !s.starts_with(serde_utils::TIMESTAMP::NAME)
+ || s.starts_with("TIMESTAMP_LTZ")
+ || s.contains("WITH LOCAL TIME ZONE")
+ {
return DataTypeInvalidSnafu {
message: "Invalid TIMESTAMP type. Expected string to start
with 'TIMESTAMP'.",
}
@@ -2556,6 +2562,33 @@ mod tests {
}
}
+ #[test]
+ fn test_timestamp_ltz_alias_preserves_type_precision_and_nullability() {
+ for precision in [0, 3, 6, 9] {
+ for nullable in [true, false] {
+ let suffix = if nullable { "" } else { " NOT NULL" };
+ let canonical = format!("TIMESTAMP({precision}) WITH LOCAL
TIME ZONE{suffix}");
+ let expected =
LocalZonedTimestampType::with_nullable(nullable, precision).unwrap();
+ for name in [
+ format!("TIMESTAMP_LTZ({precision}){suffix}"),
+ canonical.clone(),
+ ] {
+ let json = serde_json::to_string(&name).unwrap();
+ let data_type: DataType =
serde_json::from_str(&json).unwrap();
+ assert_eq!(data_type,
DataType::LocalZonedTimestamp(expected.clone()));
+
assert_eq!(LocalZonedTimestampType::from_str(&name).unwrap(), expected);
+ assert_eq!(serde_json::to_value(&data_type).unwrap(),
canonical);
+ assert!(TimestampType::from_str(&name).is_err());
+ }
+ let ordinary = format!("TIMESTAMP({precision}){suffix}");
+ assert_eq!(
+
serde_json::from_value::<DataType>(serde_json::json!(ordinary)).unwrap(),
+ DataType::Timestamp(TimestampType::with_nullable(nullable,
precision).unwrap())
+ );
+ }
+ }
+ }
+
#[test]
fn test_string_type_serializes_like_java() {
let nullable = DataType::VarChar(VarCharType::string_type());
