924060929 commented on code in PR #68161:
URL: https://github.com/apache/doris/pull/68161#discussion_r4256366486
##########
fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonScanPlanProvider.java:
##########
@@ -1289,6 +1306,135 @@ public Map<String, String> getScanNodeProperties(
return props;
}
+ static boolean containsTimestampType(DataType type) {
+ if (type instanceof org.apache.paimon.types.TimestampType
+ || type instanceof
org.apache.paimon.types.LocalZonedTimestampType) {
+ return true;
+ }
+ if (type instanceof org.apache.paimon.types.RowType) {
+ return ((org.apache.paimon.types.RowType)
type).getFields().stream()
+ .anyMatch(field -> containsTimestampType(field.type()));
+ }
+ if (type instanceof org.apache.paimon.types.ArrayType) {
+ return containsTimestampType(((org.apache.paimon.types.ArrayType)
type).getElementType());
+ }
+ if (type instanceof org.apache.paimon.types.MapType) {
+ org.apache.paimon.types.MapType mapType =
(org.apache.paimon.types.MapType) type;
+ return containsTimestampType(mapType.getKeyType()) ||
containsTimestampType(mapType.getValueType());
+ }
+ return false;
+ }
+
+ static boolean filterReadsTimestamp(RowType rowType, ConnectorExpression
filter) {
+ Set<String> names = new HashSet<>();
+ collectFilterColumnNames(filter, names);
+ return rowType.getFields().stream().anyMatch(field ->
+ names.contains(field.name().toLowerCase(Locale.ROOT)) &&
containsTimestampType(field.type()));
+ }
+
+ boolean requiresTimestampPrecisionRepair(ConnectorSession session, Table
table, PaimonTableHandle handle) {
+ if (!containsTimestampType(table.rowType())) {
+ return false;
+ }
+ Supplier<Boolean> lookup = () -> withSchemaHistoryAuthentication(
+ () ->
hasTimestampPrecisionHistory(resolveSchemaDictTable(table, handle)));
+ if (session == null || session.getStatementScope() == null
+ || session.getStatementScope() ==
ConnectorStatementScope.NONE) {
+ return lookup.get();
+ }
+ String memoKey = SCAN_REUSE_NAMESPACE + ":timestamp-history:"
+ + session.getCatalogId() + ":" + session.getQueryId();
+ Map<PaimonTableResolutionKey, Boolean> history =
session.getStatementScope().computeIfAbsent(
+ memoKey, () -> new ConcurrentHashMap<>());
+ return history.computeIfAbsent(new PaimonTableResolutionKey(handle,
effectiveScanOptions(handle)),
+ ignored -> lookup.get());
+ }
+
+ private <T> T withSchemaHistoryAuthentication(Supplier<T> action) {
+ if (context == null) {
+ return action.get();
+ }
+ try {
+ return context.executeAuthenticated(action::get);
+ } catch (RuntimeException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new RuntimeException("Failed to read Paimon timestamp schema
history", e);
+ }
+ }
+
+ private static boolean hasTimestampPrecisionHistory(Table schemaTable) {
+ if (!(schemaTable instanceof FileStoreTable)) {
+ return false;
+ }
+ FileStoreTable fileStoreTable =
PaimonTableDecorators.unwrapToFallbackOrBase((FileStoreTable) schemaTable);
+ if (fileStoreTable instanceof FallbackReadFileStoreTable) {
+ FallbackReadFileStoreTable pair = (FallbackReadFileStoreTable)
fileStoreTable;
+ return hasTimestampPrecisionHistory(pair.wrapped()) ||
hasTimestampPrecisionHistory(pair.other());
+ }
+ RowType currentType = schemaTable.rowType();
+ SchemaManager schemaManager = fileStoreTable.schemaManager();
+ for (Long schemaId : schemaManager.listAllIds()) {
+ TableSchema historicalSchema = schemaManager.schema(schemaId);
+ if (historicalSchema != null
+ &&
hasTimestampPrecisionNarrowing(historicalSchema.logicalRowType(), currentType))
{
+ return true;
+ }
+ }
+ return false;
+ }
+
+ static boolean hasTimestampPrecisionNarrowing(DataType historicalType,
DataType currentType) {
+ return hasTimestampPrecisionNarrowing(historicalType, currentType, 0);
+ }
+
+ private static boolean hasTimestampPrecisionNarrowing(
+ DataType historicalType, DataType currentType, int
minimumHistoricalPrecision) {
+ if (historicalType instanceof org.apache.paimon.types.TimestampType
+ && currentType instanceof
org.apache.paimon.types.TimestampType) {
Review Comment:
[P1] Include cross-family timestamp narrowing in the history check
This only compares precision when both leaves have the same concrete
timestamp type. Paimon 1.4.2 also permits `TIMESTAMP(6) -> TIMESTAMP_LTZ(0)`
and the reverse: `DataTypeCasts` accepts both roots, and
`TimestampToTimestampCastRule` performs the zone conversion followed by
precision truncation.
For an append-only Parquet table containing historical `TIMESTAMP(6)` value
`1969-12-31 23:59:59.600000`, change the column to `TIMESTAMP_LTZ(0)` and force
JNI, with both JVM and session zones set to UTC. This helper returns false, so
FE sends `doris.timestamp-precision-repair=false` and JNI skips the widened
read type. The SDK then truncates -400ms toward zero to the epoch before
`PaimonColumnValue` receives it. The getter sees an already aligned value and
returns `1970-01-01 00:00:00`, rather than the correct `1969-12-31 23:59:59`.
Please recognize either timestamp class on each side and compare their
precisions uniformly, while preserving the SDK's normal zone conversion. Add
historical-file JNI coverage for both directions, including a negative-epoch
value. This is a supported case omitted by the repair, rather than a newly
introduced SDK defect; the distinction from the earlier same-family/getter
threads is this FE guard's cross-family false negative.
Evidence:
[DataTypeCasts](https://github.com/apache/paimon/blob/release-1.4.2/paimon-api/src/main/java/org/apache/paimon/types/DataTypeCasts.java#L167-L176),
[TimestampToTimestampCastRule](https://github.com/apache/paimon/blob/release-1.4.2/paimon-common/src/main/java/org/apache/paimon/casting/TimestampToTimestampCastRule.java#L45-L69),
and
[DateTimeUtils.truncate](https://github.com/apache/paimon/blob/release-1.4.2/paimon-common/src/main/java/org/apache/paimon/utils/DateTimeUtils.java#L651-L675).
This review established the path statically; I did not run a reproduction.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
