This is an automated email from the ASF dual-hosted git repository. yiguolei pushed a commit to branch branch-4.2 in repository https://gitbox.apache.org/repos/asf/doris.git
commit ed5ada37e6addd5c043a9ee02c9d202f5dfbdbf5 Author: Gabriel <[email protected]> AuthorDate: Mon Oct 12 14:46:28 2026 +0800 [branch-4.1](fix) Correct external binary partitions and timestamp handling (#68873) ### What problem does this PR solve? Related PR: #68786, #68532 `branch-4.1` already contains #68532. This PR adapts the applicable correctness fixes found during #68786 review to the 4.1 executors and scan planner. | Area | Fix | | --- | --- | | Iceberg binary partitions | Distinguish SQL NULL from binary bytes whose Base64 path is `null` when reusing writers, preserving separate commit metadata. | | VARBINARY literals | Compare unsigned bytes with significant trailing zeros in both FE literal implementations. | | MySQL / OceanBase MySQL | Project instant columns as UTC text before driver decoding and preserve `zeroDateTimeBehavior`. | | Oracle | Initialize each borrowed driver's session time zone and use explicit-offset text binds with server-side conversion. | | Presto | Support the official driver's scalar/array timestamp representation and VARCHAR timestamp binds with explicit casts. | | JDBC query TVFs | Preserve trailing comments, terminal delimiters, MySQL/MariaDB quoting modes and Trino `WITH SESSION` when wrapping timestamp projections. Capture MySQL SQL mode from the scan connection configuration before serialization. | | JNI | Reject UTC years outside 0–9999 before packing TIMESTAMPTZ, including nested values, while preserving NULL and year zero. | | Paimon | Serialize binary partition values losslessly for native readers. | | MaxCompute | Write nested timestamp vectors using UTC microseconds. | Existing 4.1 type mappings are retained. The master connector framework, native external UUID migration and unconditional file-TVF mapping changes are outside this backport. ### Release note Fix external binary partition correctness, VARBINARY literal comparisons, JDBC timestamp handling and nested timestamp writes in 4.1. ### Check List (For Author) - Test: - [x] Unit Test: 76 focused FE tests, 44 Java extension tests and 31 BE writer/partition-transform tests passed locally. BE tests rebuilt the affected source and test with existing 4.1 build dependencies. The latest FE run includes the query-wrapper lifecycle and MariaDB bracket-identifier regressions. - [x] Manual test: MySQL Connector/J 5.1.49 and 8.0.28 each passed all four integration methods, covering JVM/session time zones, client/server prepared statements, cached driver time zones and zero-date policies. MariaDB 10.4 also passed three wrapped-query timestamp readbacks with MSSQL bracket identifiers and scan-local mode capture. - [x] Regression test added: Iceberg binary/NULL path collision, including independent Spark partition-filter checks for Parquet and ORC. The full external regression suite was not run locally. Updated the MySQL projection expectations and adapted the Doris binary-filter assertion to 4.1 while retaining Spark partition-pruning checks. - [x] FE Checkstyle and clang-format 16 checks. - Behavior changed: - [x] Yes: correct the existing read/write and comparison behavior described above. - Does this need documentation? - [x] No. --- .../sink/writer/iceberg/viceberg_table_writer.cpp | 17 +- .../sink/writer/iceberg/viceberg_table_writer.h | 1 + .../writer/iceberg/viceberg_table_writer_test.cpp | 8 +- .../apache/doris/common/jni/vec/VectorColumn.java | 9 + .../jni/vec/VectorColumnTimestampTzTest.java | 86 ++++++ fe/be-java-extensions/jdbc-scanner/pom.xml | 6 + .../org/apache/doris/jdbc/MySQLJdbcExecutor.java | 9 +- .../org/apache/doris/jdbc/OracleJdbcExecutor.java | 27 +- .../org/apache/doris/jdbc/TrinoJdbcExecutor.java | 32 ++- .../doris/jdbc/JdbcTimestampBackportTest.java | 70 +++++ .../apache/doris/jdbc/JdbcTimestampBindTest.java | 6 +- .../doris/jdbc/JdbcTimestampSemanticsTest.java | 4 +- .../jdbc/JdbcZonedTimestampIntegrationTest.java | 52 +++- .../doris/jdbc/MySqlTimestampIntegrationTest.java | 124 +++++++-- .../doris/jdbc/OracleSessionTimeZoneTest.java | 73 +++++ .../doris/maxcompute/MaxComputeJniWriter.java | 9 + .../doris/maxcompute/MaxComputeJniWriterTest.java | 40 +++ .../apache/doris/analysis/VarBinaryLiteral.java | 30 +- .../java/org/apache/doris/catalog/JdbcTable.java | 13 +- .../doris/datasource/jdbc/client/JdbcClient.java | 26 ++ .../doris/datasource/jdbc/source/JdbcScanNode.java | 195 ++++++++++++- .../apache/doris/datasource/paimon/PaimonUtil.java | 7 +- .../expressions/literal/VarBinaryLiteral.java | 30 +- .../analysis/VarBinaryLiteralOrderingTest.java | 34 +++ .../jdbc/source/JdbcQueryWrappingTest.java | 306 +++++++++++++++++++++ .../jdbc/source/JdbcTimestampProjectionTest.java | 7 +- .../paimon/PaimonBinaryPartitionTest.java | 36 +++ .../nereids/parser/VarBinaryLiteralParserTest.java | 4 +- .../trees/expressions/VarBinaryLiteralTest.java | 5 +- .../test_iceberg_write_binary_partitions.groovy | 21 ++ .../jdbc/test_mysql_jdbc_catalog.groovy | 5 +- 31 files changed, 1188 insertions(+), 104 deletions(-) diff --git a/be/src/exec/sink/writer/iceberg/viceberg_table_writer.cpp b/be/src/exec/sink/writer/iceberg/viceberg_table_writer.cpp index d2697f61e10..6b1f8318c33 100644 --- a/be/src/exec/sink/writer/iceberg/viceberg_table_writer.cpp +++ b/be/src/exec/sink/writer/iceberg/viceberg_table_writer.cpp @@ -481,7 +481,7 @@ Status VIcebergTableWriter::_write_prepared_block(Block& output_block) { } std::string partition_name; try { - partition_name = _partition_to_path(partition_data.value()); + partition_name = _partition_to_writer_key(partition_data.value()); } catch (doris::Exception& e) { return e.to_status(); } @@ -656,6 +656,21 @@ std::string VIcebergTableWriter::_partition_value_to_human_string(size_t index, return encoded; } +std::string VIcebergTableWriter::_partition_to_writer_key(const doris::iceberg::StructLike& data) { + std::string key = _partition_to_path(data); + // Iceberg paths encode both NULL and binary 0x9ee965 as "null". Keep their writers and + // commit metadata separate without changing the standard, externally visible path. + key.push_back('\0'); + for (size_t i = 0; i < _iceberg_partition_columns.size(); ++i) { + // Static fields are constant for this writer; only dynamic fields can collide. + key.push_back((_has_static_partition && _partition_column_is_static[i]) || + data.get(i).has_value() + ? '1' + : '0'); + } + return key; +} + std::string VIcebergTableWriter::_partition_to_path(const doris::iceberg::StructLike& data) { std::stringstream ss; for (size_t i = 0; i < _iceberg_partition_columns.size(); i++) { diff --git a/be/src/exec/sink/writer/iceberg/viceberg_table_writer.h b/be/src/exec/sink/writer/iceberg/viceberg_table_writer.h index 1b98f26c86a..c66cc1a74da 100644 --- a/be/src/exec/sink/writer/iceberg/viceberg_table_writer.h +++ b/be/src/exec/sink/writer/iceberg/viceberg_table_writer.h @@ -131,6 +131,7 @@ private: std::vector<IcebergPartitionColumn> _to_iceberg_partition_columns(); + std::string _partition_to_writer_key(const doris::iceberg::StructLike& data); std::string _partition_to_path(const doris::iceberg::StructLike& data); std::string _escape(const std::string& path); std::vector<std::string> _partition_values(const doris::iceberg::StructLike& data); diff --git a/be/test/exec/sink/writer/iceberg/viceberg_table_writer_test.cpp b/be/test/exec/sink/writer/iceberg/viceberg_table_writer_test.cpp index 852adb276fd..849b55b2437 100644 --- a/be/test/exec/sink/writer/iceberg/viceberg_table_writer_test.cpp +++ b/be/test/exec/sink/writer/iceberg/viceberg_table_writer_test.cpp @@ -180,7 +180,7 @@ TEST(VIcebergTableWriterTest, StaticNullBinaryPartitionsKeepNullPathsAndCommitVa IcebergPartitionData hybrid({std::any(), Int32(7)}); ASSERT_EQ("key=null/id=7", writer._partition_to_path(hybrid)); EXPECT_EQ(std::vector<std::string>({"null", "7"}), writer._partition_values(hybrid)); - writer._partitions_to_writers["key=null/id=7"] = recording_writer; + writer._partitions_to_writers[writer._partition_to_writer_key(hybrid)] = recording_writer; Block hybrid_block({ColumnWithTypeAndName(null_column->get_ptr(), type, "key"), ColumnWithTypeAndName(id_column->get_ptr(), id_type, "id")}); ASSERT_TRUE(writer._write_prepared_block(hybrid_block).ok()); @@ -226,6 +226,12 @@ TEST(VIcebergTableWriterTest, BinaryIdentityStaticAndDynamicRouting) { IcebergPartitionData data({value}); EXPECT_EQ("key=3q0%3D", writer._partition_to_path(data)); EXPECT_EQ(std::vector<std::string>({"0xdead"}), writer._partition_values(data)); + IcebergPartitionData null_value({std::any()}); + IcebergPartitionData null_text({std::string("\x9e\xe9\x65", 3)}); + // Distinct values can share the standard Iceberg directory, but must never share a writer. + EXPECT_EQ(writer._partition_to_path(null_value), writer._partition_to_path(null_text)); + EXPECT_NE(writer._partition_to_writer_key(null_value), + writer._partition_to_writer_key(null_text)); writer._init_static_partition_values(); EXPECT_TRUE(writer._is_full_static_partition); EXPECT_EQ("key=3q0%3D", writer._static_partition_path); diff --git a/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/VectorColumn.java b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/VectorColumn.java index 010bf7eba8f..ec66746c6dc 100644 --- a/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/VectorColumn.java +++ b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/VectorColumn.java @@ -1187,6 +1187,15 @@ public class VectorColumn { } private void putTimeStampTz(int rowId, LocalDateTime v) { + // SQL NULL uses an out-of-range sentinel; only real values need range validation. + if (isNullAt(rowId)) { + OffHeap.putLong(null, data + rowId * 8L, 0L); + return; + } + // Validate UTC components before bit packing, including values from nested containers. + if (v.getYear() < 0 || v.getYear() > 9999) { + throw new IllegalArgumentException("TIMESTAMPTZ year must be between 0 and 9999: " + v); + } // TimeStampTz use the same storage format as DateTimeV2 long time = TypeNativeBytes.convertToDateTimeV2(v.getYear(), v.getMonthValue(), v.getDayOfMonth(), v.getHour(), v.getMinute(), v.getSecond(), v.getNano() / 1000); diff --git a/fe/be-java-extensions/java-common/src/test/java/org/apache/doris/common/jni/vec/VectorColumnTimestampTzTest.java b/fe/be-java-extensions/java-common/src/test/java/org/apache/doris/common/jni/vec/VectorColumnTimestampTzTest.java new file mode 100644 index 00000000000..6d0dbec503d --- /dev/null +++ b/fe/be-java-extensions/java-common/src/test/java/org/apache/doris/common/jni/vec/VectorColumnTimestampTzTest.java @@ -0,0 +1,86 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.common.jni.vec; + +import org.apache.doris.common.jni.utils.OffHeap; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.time.LocalDateTime; +import java.util.Arrays; + +class VectorColumnTimestampTzTest { + @Test + void rejectsOutOfRangeScalarAndBatchValuesButPreservesNulls() { + OffHeap.setTesting(); + VectorColumn column = VectorColumn.createWritableColumn(ColumnType.parseType("ts", "timestamptz(6)"), 1); + try { + LocalDateTime min = LocalDateTime.of(0, 1, 1, 0, 0); + LocalDateTime max = LocalDateTime.of(9999, 12, 31, 23, 59, 59, 999999000); + column.appendTimeStampTz(min); + column.appendNull(ColumnType.Type.TIMESTAMPTZ); + column.appendTimeStampTz(new LocalDateTime[] {null, min, max}, true); + Assertions.assertArrayEquals(new LocalDateTime[] {min, null, null, min, max}, + column.getTimeStampTzColumn(0, 5)); + for (LocalDateTime invalid : Arrays.asList(min.minusNanos(1000), max.plusNanos(1000))) { + Assertions.assertThrows(IllegalArgumentException.class, () -> column.appendTimeStampTz(invalid)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> column.appendTimeStampTz(new LocalDateTime[] {invalid}, false)); + } + } finally { + column.close(); + } + } + + @Test + void preservesYearZeroArrayElements() { + OffHeap.setTesting(); + VectorColumn column = VectorColumn.createWritableColumn( + ColumnType.parseType("events", "array<timestamptz(6)>"), 1); + try { + LocalDateTime min = LocalDateTime.of(0, 1, 1, 0, 0); + Object[] values = column.newObjectContainerArray(1); + values[0] = new java.util.ArrayList<>(Arrays.asList(min, null, min.plusNanos(1000))); + column.appendObjectColumn(values, true); + Assertions.assertEquals(values[0], column.getObjectColumn(0, 1)[0]); + } finally { + column.close(); + } + } + + @Test + void rejectsOutOfRangeNestedValues() { + OffHeap.setTesting(); + for (String type : new String[] {"array<timestamptz(6)>", "map<string,timestamptz(6)>", + "struct<ts:timestamptz(6)>"}) { + VectorColumn column = VectorColumn.createWritableColumn(ColumnType.parseType("nested", type), 1); + try { + LocalDateTime invalid = LocalDateTime.of(10000, 1, 1, 0, 0); + Object value = type.startsWith("array") ? new java.util.ArrayList<>(Arrays.asList(invalid)) + : new java.util.HashMap<>(java.util.Collections.singletonMap("ts", invalid)); + Object[] batch = column.newObjectContainerArray(1); + batch[0] = value; + Assertions.assertThrows(IllegalArgumentException.class, + () -> column.appendObjectColumn(batch, true)); + } finally { + column.close(); + } + } + } +} diff --git a/fe/be-java-extensions/jdbc-scanner/pom.xml b/fe/be-java-extensions/jdbc-scanner/pom.xml index 84dd37e7946..fa866ddd6b2 100644 --- a/fe/be-java-extensions/jdbc-scanner/pom.xml +++ b/fe/be-java-extensions/jdbc-scanner/pom.xml @@ -35,6 +35,12 @@ under the License. </properties> <dependencies> + <dependency> + <groupId>com.oracle.database.jdbc</groupId> + <artifactId>ojdbc11</artifactId> + <version>23.26.2.0.0</version> + <scope>test</scope> + </dependency> <dependency> <groupId>org.mockito</groupId> <artifactId>mockito-core</artifactId> diff --git a/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/MySQLJdbcExecutor.java b/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/MySQLJdbcExecutor.java index f0cb06163be..0cc7d0e2006 100644 --- a/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/MySQLJdbcExecutor.java +++ b/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/MySQLJdbcExecutor.java @@ -180,7 +180,14 @@ public class MySQLJdbcExecutor extends BaseJdbcExecutor { } case TIMESTAMPTZ: { if (usesMySqlTimestampProtocol(config)) { - return resultSet.getObject(columnIndex + 1, LocalDateTime.class); + // Read the server's UTC text projection to bypass cached driver time zones. + String text = resultSet.getString(columnIndex + 1); + if (text != null && text.startsWith("0000-00-00 ")) { + // The date decoder preserves zeroDateTimeBehavior even for VARCHAR projections. + java.sql.Date date = resultSet.getDate(columnIndex + 1); + return date == null ? null : date.toLocalDate().atStartOfDay(); + } + return text == null ? null : LocalDateTime.parse(text.replace(' ', 'T')); } Timestamp value = resultSet.getTimestamp(columnIndex + 1); return value == null ? null : LocalDateTime.ofInstant(value.toInstant(), ZoneOffset.UTC); diff --git a/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/OracleJdbcExecutor.java b/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/OracleJdbcExecutor.java index 12e8bed89ea..4332c1472f2 100644 --- a/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/OracleJdbcExecutor.java +++ b/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/OracleJdbcExecutor.java @@ -32,13 +32,15 @@ import java.nio.charset.CharacterCodingException; import java.nio.charset.CharsetDecoder; import java.nio.charset.StandardCharsets; import java.sql.Clob; +import java.sql.Connection; import java.sql.SQLException; import java.sql.Timestamp; import java.sql.Types; import java.time.LocalDateTime; -import java.time.ZoneOffset; public class OracleJdbcExecutor extends BaseJdbcExecutor { + private static final java.time.format.DateTimeFormatter TIMESTAMP_TZ_FORMATTER = + java.time.format.DateTimeFormatter.ofPattern("uuuu-MM-dd HH:mm:ss.SSSSSS"); private static final Logger LOG = Logger.getLogger(OracleJdbcExecutor.class); private final CharsetDecoder utf8Decoder = StandardCharsets.UTF_8.newDecoder(); private final boolean isNewJdbcVersion; @@ -50,8 +52,27 @@ public class OracleJdbcExecutor extends BaseJdbcExecutor { @Override protected void setTimestampTz(int parameterIndex, LocalDateTime value) throws SQLException { - // An unzoned TIMESTAMP bind is session-local for both Oracle TZ and LOCAL TIME ZONE columns. - preparedStatement.setObject(parameterIndex, value.atOffset(ZoneOffset.UTC), Types.TIMESTAMP_WITH_TIMEZONE); + // Older Oracle drivers reject JDBC type 2014; the sink SQL parses an explicit UTC offset. + preparedStatement.setString(parameterIndex, value.format(TIMESTAMP_TZ_FORMATTER) + " +00:00"); + } + + @Override + protected void setTimestampTzNull(int parameterIndex) throws SQLException { + preparedStatement.setNull(parameterIndex, Types.VARCHAR); + } + + @Override + protected void initializeStatement(Connection conn, JdbcDataSourceConfig config, String sql) throws SQLException { + // ALTER SESSION alone leaves the driver's TSLTZ zone unset; initialize every borrowed connection. + try { + Connection physical = conn.unwrap(Connection.class); + Class<?> oracleConnection = Class.forName("oracle.jdbc.OracleConnection", true, + physical.getClass().getClassLoader()); + oracleConnection.getMethod("setSessionTimeZone", String.class).invoke(conn.unwrap(oracleConnection), "UTC"); + } catch (ReflectiveOperationException e) { + throw new SQLException("Failed to initialize Oracle session time zone", e); + } + super.initializeStatement(conn, config, sql); } @Override diff --git a/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/TrinoJdbcExecutor.java b/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/TrinoJdbcExecutor.java index 49566afe350..a1304a74d88 100644 --- a/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/TrinoJdbcExecutor.java +++ b/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/TrinoJdbcExecutor.java @@ -42,6 +42,10 @@ import java.util.List; public class TrinoJdbcExecutor extends BaseJdbcExecutor { private static final DateTimeFormatter TIMESTAMP_TZ_WRITE_FORMATTER = DateTimeFormatter.ofPattern("uuuu-MM-dd HH:mm:ss.SSSSSS"); + private static final DateTimeFormatter ARRAY_TIMESTAMP_FORMATTER = new java.time.format.DateTimeFormatterBuilder() + .appendPattern("uuuu-MM-dd HH:mm:ss") + .appendFraction(java.time.temporal.ChronoField.NANO_OF_SECOND, 0, 9, true) + .appendLiteral(' ').appendZoneOrOffsetId().toFormatter(); public TrinoJdbcExecutor(byte[] thriftParams) throws Exception { super(thriftParams); @@ -49,6 +53,11 @@ public class TrinoJdbcExecutor extends BaseJdbcExecutor { @Override protected void setTimestampTz(int parameterIndex, LocalDateTime value) throws SQLException { + if (config != null && config.getTableType() == org.apache.doris.thrift.TOdbcTableType.PRESTO) { + // PrestoDB requires VARCHAR plus a server-side cast; its driver rejects JDBC type 2014. + preparedStatement.setString(parameterIndex, value.format(TIMESTAMP_TZ_WRITE_FORMATTER) + " UTC"); + return; + } // Trino/Presto require a string for typed zoned binds; Timestamp drops the zone and sub-millisecond digits. preparedStatement.setObject(parameterIndex, value.format(TIMESTAMP_TZ_WRITE_FORMATTER) + " UTC", Types.TIMESTAMP_WITH_TIMEZONE); @@ -57,7 +66,9 @@ public class TrinoJdbcExecutor extends BaseJdbcExecutor { @Override protected void setTimestampTzNull(int parameterIndex) throws SQLException { // These drivers reject TIMESTAMP_WITH_TIMEZONE in setNull; SQL NULL is coerced by the target column. - preparedStatement.setNull(parameterIndex, Types.NULL); + preparedStatement.setNull(parameterIndex, + config != null && config.getTableType() == org.apache.doris.thrift.TOdbcTableType.PRESTO + ? Types.VARCHAR : Types.NULL); } @Override @@ -79,6 +90,11 @@ public class TrinoJdbcExecutor extends BaseJdbcExecutor { protected Object getColumnValue(int columnIndex, ColumnType type, String[] replaceStringList) throws SQLException { switch (type.getType()) { case TIMESTAMPTZ: { + if (config != null && config.getTableType() == org.apache.doris.thrift.TOdbcTableType.PRESTO) { + // PrestoDB does not implement typed getObject for zoned timestamps. + Timestamp value = resultSet.getTimestamp(columnIndex + 1); + return value == null ? null : LocalDateTime.ofInstant(value.toInstant(), ZoneOffset.UTC); + } // JdbcScanNode projects to UTC before the driver can lose a named-zone overlap offset. // JNI carries instants as UTC components, not the source zone's wall clock. ZonedDateTime value = resultSet.getObject(columnIndex + 1, ZonedDateTime.class); @@ -167,7 +183,7 @@ public class TrinoJdbcExecutor extends BaseJdbcExecutor { // Trino JDBC exposes timestamp-with-zone array elements as java.sql.Timestamp. for (Object element : array) { result.add(element == null ? null - : LocalDateTime.ofInstant(((Timestamp) element).toInstant(), ZoneOffset.UTC)); + : convertTimestampTzArrayElement(element)); } return result; } @@ -195,4 +211,16 @@ public class TrinoJdbcExecutor extends BaseJdbcExecutor { return array; } } + + private static LocalDateTime convertTimestampTzArrayElement(Object element) { + // PrestoDB returns zoned text for arrays, whereas Trino returns Timestamp objects. + java.time.Instant instant; + if (element instanceof String) { + instant = ZonedDateTime.parse((String) element, ARRAY_TIMESTAMP_FORMATTER).toInstant(); + } else { + instant = ((Timestamp) element).toInstant(); + } + return LocalDateTime.ofInstant(instant, ZoneOffset.UTC); + } + } diff --git a/fe/be-java-extensions/jdbc-scanner/src/test/java/org/apache/doris/jdbc/JdbcTimestampBackportTest.java b/fe/be-java-extensions/jdbc-scanner/src/test/java/org/apache/doris/jdbc/JdbcTimestampBackportTest.java new file mode 100644 index 00000000000..8977b53fdbe --- /dev/null +++ b/fe/be-java-extensions/jdbc-scanner/src/test/java/org/apache/doris/jdbc/JdbcTimestampBackportTest.java @@ -0,0 +1,70 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.jdbc; + +import org.apache.doris.common.jni.vec.ColumnType; +import org.apache.doris.thrift.TOdbcTableType; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.time.LocalDateTime; +import java.util.Arrays; + +class JdbcTimestampBackportTest { + @Test + void mysqlReadsProjectedTextAndHonorsZeroDates() throws Exception { + MySQLJdbcExecutor executor = Mockito.mock(MySQLJdbcExecutor.class, Mockito.CALLS_REAL_METHODS); + executor.config = new JdbcDataSourceConfig().setTableType(TOdbcTableType.MYSQL); + executor.resultSet = Mockito.mock(ResultSet.class); + ColumnType type = ColumnType.parseType("ts", "timestamptz(6)"); + Mockito.when(executor.resultSet.getString(1)).thenReturn("2021-11-07 06:30:00.123456"); + Assertions.assertEquals(LocalDateTime.parse("2021-11-07T06:30:00.123456"), + executor.getColumnValue(0, type, new String[0])); + Mockito.when(executor.resultSet.getString(1)).thenReturn("0000-00-00 00:00:00"); + Assertions.assertNull(executor.getColumnValue(0, type, new String[0])); + Mockito.verify(executor.resultSet).getDate(1); + } + + @Test + void prestoReadsZonedStringsInNestedArrays() { + TrinoJdbcExecutor executor = Mockito.mock(TrinoJdbcExecutor.class, Mockito.CALLS_REAL_METHODS); + executor.config = new JdbcDataSourceConfig().setTableType(TOdbcTableType.PRESTO); + ColumnType type = ColumnType.parseType("ts", "array<array<timestamptz(6)>>"); + Object input = Arrays.asList(Arrays.asList("1969-12-31 23:59:59.999999 UTC", null)); + Object expected = Arrays.asList(Arrays.asList(LocalDateTime.parse("1969-12-31T23:59:59.999999"), null)); + Assertions.assertEquals(expected, executor.getOutputConverter(type, "").convert(new Object[] {input})[0]); + } + + @Test + void legacyDriversBindZonedTimestampsAsText() throws Exception { + LocalDateTime value = LocalDateTime.parse("2021-11-07T06:30:00.123456"); + OracleJdbcExecutor oracle = Mockito.mock(OracleJdbcExecutor.class, Mockito.CALLS_REAL_METHODS); + oracle.preparedStatement = Mockito.mock(PreparedStatement.class); + oracle.setTimestampTz(1, value); + Mockito.verify(oracle.preparedStatement).setString(1, "2021-11-07 06:30:00.123456 +00:00"); + TrinoJdbcExecutor presto = Mockito.mock(TrinoJdbcExecutor.class, Mockito.CALLS_REAL_METHODS); + presto.config = new JdbcDataSourceConfig().setTableType(TOdbcTableType.PRESTO); + presto.preparedStatement = Mockito.mock(PreparedStatement.class); + presto.setTimestampTz(1, value); + Mockito.verify(presto.preparedStatement).setString(1, "2021-11-07 06:30:00.123456 UTC"); + } +} diff --git a/fe/be-java-extensions/jdbc-scanner/src/test/java/org/apache/doris/jdbc/JdbcTimestampBindTest.java b/fe/be-java-extensions/jdbc-scanner/src/test/java/org/apache/doris/jdbc/JdbcTimestampBindTest.java index 96f088fae52..ac90f59594f 100644 --- a/fe/be-java-extensions/jdbc-scanner/src/test/java/org/apache/doris/jdbc/JdbcTimestampBindTest.java +++ b/fe/be-java-extensions/jdbc-scanner/src/test/java/org/apache/doris/jdbc/JdbcTimestampBindTest.java @@ -87,6 +87,9 @@ class JdbcTimestampBindTest { Mockito.verify(executor.preparedStatement).setObject(1, utc.format(DateTimeFormatter.ofPattern("uuuu-MM-dd HH:mm:ss.SSSSSS")) + " UTC", Types.TIMESTAMP_WITH_TIMEZONE); + } else if (dialect == OracleJdbcExecutor.class) { + Mockito.verify(executor.preparedStatement).setString(1, + utc.format(DateTimeFormatter.ofPattern("uuuu-MM-dd HH:mm:ss.SSSSSS")) + " +00:00"); } else if (dialect == ClickHouseJdbcExecutor.class) { Mockito.verify(executor.preparedStatement).setObject(1, utc.atOffset(ZoneOffset.UTC)); } else { @@ -99,7 +102,8 @@ class JdbcTimestampBindTest { column.appendTimeStampTz(new LocalDateTime[] {null}, true); insert.invoke(executor, 0, 0, column); Mockito.verify(executor.preparedStatement).setNull(1, - dialect == TrinoJdbcExecutor.class ? Types.NULL : Types.TIMESTAMP_WITH_TIMEZONE); + dialect == TrinoJdbcExecutor.class ? Types.NULL + : dialect == OracleJdbcExecutor.class ? Types.VARCHAR : Types.TIMESTAMP_WITH_TIMEZONE); Mockito.clearInvocations(executor.preparedStatement); } } finally { diff --git a/fe/be-java-extensions/jdbc-scanner/src/test/java/org/apache/doris/jdbc/JdbcTimestampSemanticsTest.java b/fe/be-java-extensions/jdbc-scanner/src/test/java/org/apache/doris/jdbc/JdbcTimestampSemanticsTest.java index 9fd92797008..0dacaf3ce3d 100644 --- a/fe/be-java-extensions/jdbc-scanner/src/test/java/org/apache/doris/jdbc/JdbcTimestampSemanticsTest.java +++ b/fe/be-java-extensions/jdbc-scanner/src/test/java/org/apache/doris/jdbc/JdbcTimestampSemanticsTest.java @@ -167,7 +167,7 @@ class JdbcTimestampSemanticsTest { MySQLJdbcExecutor executor = Mockito.mock(MySQLJdbcExecutor.class, Mockito.CALLS_REAL_METHODS); executor.config = new JdbcDataSourceConfig().setTableType(org.apache.doris.thrift.TOdbcTableType.OCEANBASE); executor.resultSet = Mockito.mock(ResultSet.class); - Mockito.when(executor.resultSet.getObject(1, LocalDateTime.class)).thenReturn(UTC_VALUE); + Mockito.when(executor.resultSet.getString(1)).thenReturn(UTC_VALUE.toString().replace('T', ' ')); Assertions.assertEquals(UTC_VALUE, executor.getColumnValue(0, ColumnType.parseType("event_time", "timestamptz(6)"), new String[0])); executor.preparedStatement = Mockito.mock(java.sql.PreparedStatement.class); @@ -289,7 +289,7 @@ class JdbcTimestampSemanticsTest { executor.config = new JdbcDataSourceConfig().setTableType(org.apache.doris.thrift.TOdbcTableType.MYSQL); ResultSet resultSet = Mockito.mock(ResultSet.class); executor.resultSet = resultSet; - Mockito.when(resultSet.getObject(1, LocalDateTime.class)).thenReturn(UTC_VALUE).thenReturn(null); + Mockito.when(resultSet.getString(1)).thenReturn(UTC_VALUE.toString().replace('T', ' ')).thenReturn(null); ColumnType type = ColumnType.parseType("event_time", "timestamptz(6)"); Assertions.assertEquals(UTC_VALUE, executor.getColumnValue(0, type, new String[0])); Assertions.assertNull(executor.getColumnValue(0, type, new String[0])); diff --git a/fe/be-java-extensions/jdbc-scanner/src/test/java/org/apache/doris/jdbc/JdbcZonedTimestampIntegrationTest.java b/fe/be-java-extensions/jdbc-scanner/src/test/java/org/apache/doris/jdbc/JdbcZonedTimestampIntegrationTest.java index 9e78d19cfc8..2ba10b0b7de 100644 --- a/fe/be-java-extensions/jdbc-scanner/src/test/java/org/apache/doris/jdbc/JdbcZonedTimestampIntegrationTest.java +++ b/fe/be-java-extensions/jdbc-scanner/src/test/java/org/apache/doris/jdbc/JdbcZonedTimestampIntegrationTest.java @@ -18,6 +18,7 @@ package org.apache.doris.jdbc; import org.apache.doris.common.jni.vec.ColumnType; +import org.apache.doris.thrift.TOdbcTableType; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -55,6 +56,9 @@ class JdbcZonedTimestampIntegrationTest { // Both dialects must disambiguate the fold before the driver reconstructs the instant. withDriver(dialect, driver, connection -> { TrinoJdbcExecutor executor = Mockito.mock(TrinoJdbcExecutor.class, Mockito.CALLS_REAL_METHODS); + // Presto must exercise its untyped timestamp read and VARCHAR bind paths. + executor.config = new JdbcDataSourceConfig().setTableType( + "presto".equals(dialect) ? TOdbcTableType.PRESTO : TOdbcTableType.TRINO); String values = "ARRAY[first_value, second_value, old_value, NULL]"; String query = "WITH sample AS (SELECT " + "at_timezone(TIMESTAMP '2023-11-05 08:30:00.123456 UTC', 'America/Los_Angeles') first_value, " @@ -293,12 +297,28 @@ class JdbcZonedTimestampIntegrationTest { if (sessionSql != null) { ddl.execute(sessionSql); } + if (dialect == OracleJdbcExecutor.class) { + // A conflicting NLS format proves that both TZ column kinds use the explicit format mask. + ddl.execute("ALTER SESSION SET NLS_TIMESTAMP_TZ_FORMAT = 'DD-MON-RR HH.MI.SSXFF AM TZR'"); + } ddl.execute("CREATE TABLE " + table + " (id INT, " + String.join(", ", definitions) + ")" + suffix); try { BaseJdbcExecutor executor = Mockito.mock(dialect, Mockito.CALLS_REAL_METHODS); + executor.config = new JdbcDataSourceConfig().setTableType( + TOdbcTableType.valueOf(prefix.toUpperCase(java.util.Locale.ROOT))); String[] values = {"2020-01-02T04:01:00.111333Z", "1969-12-31T23:59:59.999999Z", "2023-11-05T08:30:00.123456Z", "2023-11-05T09:30:00.123456Z", "2020-01-02T00:00:00Z", null}; - String parameters = String.join(", ", java.util.Collections.nCopies(types.length + 1, "?")); + // Mirror JdbcTable's explicit conversion: these drivers bind instants as VARCHAR. + String timestampParameter = "?"; + if (dialect == OracleJdbcExecutor.class) { + timestampParameter = "TO_TIMESTAMP_TZ(?, 'YYYY-MM-DD HH24:MI:SS.FF6 TZH:TZM')"; + } else if ("presto".equals(prefix)) { + timestampParameter = "io.prestosql.jdbc.PrestoDriver".equals(driver) + ? "CAST(? AS TIMESTAMP(6) WITH TIME ZONE)" + : "CAST(? AS TIMESTAMP WITH TIME ZONE)"; + } + String parameters = "?, " + String.join(", ", + java.util.Collections.nCopies(types.length, timestampParameter)); try (java.sql.PreparedStatement insert = connection.prepareStatement( "INSERT INTO " + table + " VALUES (" + parameters + ")")) { executor.preparedStatement = insert; @@ -330,6 +350,10 @@ class JdbcZonedTimestampIntegrationTest { long micros = rows.getLong(col); actual = rows.wasNull() ? null : Instant.ofEpochSecond(Math.floorDiv(micros, 1_000_000), Math.floorMod(micros, 1_000_000) * 1000); + } else if ("presto".equals(prefix)) { + // PrestoDB does not implement typed getObject for zoned timestamps. + java.sql.Timestamp value = rows.getTimestamp(col); + actual = value == null ? null : value.toInstant(); } else if (dialect == TrinoJdbcExecutor.class) { java.time.ZonedDateTime value = rows.getObject(col, java.time.ZonedDateTime.class); actual = value == null ? null : value.toInstant(); @@ -349,11 +373,33 @@ class JdbcZonedTimestampIntegrationTest { }); } + @Test + void testOracleDriverIsIsolatedFromTestClasspath() throws Exception { + URL jar = oracle.jdbc.OracleDriver.class.getProtectionDomain().getCodeSource().getLocation(); + try (URLClassLoader loader = createDriverClassLoader(jar, "oracle.jdbc.OracleDriver")) { + Class<?> selected = loader.loadClass("oracle.jdbc.OracleDriver"); + Assertions.assertSame(loader, selected.getClassLoader()); + Assertions.assertNotSame(oracle.jdbc.OracleDriver.class, selected); + Assertions.assertEquals(jar, selected.getProtectionDomain().getCodeSource().getLocation()); + Assertions.assertTrue(Driver.class.isAssignableFrom(selected)); + } + } + + private URLClassLoader createDriverClassLoader(URL jar, String driverClass) { + // The Oracle mock dependency must not shadow the JAR selected for integration testing. + ClassLoader parent = driverClass.startsWith("oracle.") + ? Driver.class.getClassLoader() : getClass().getClassLoader(); + return new URLClassLoader(new URL[] {jar}, parent); + } + private void withDriver(String prefix, String driverClass, Check check) throws Exception { TimeZone original = TimeZone.getDefault(); URL jar = new File(System.getProperty(prefix + ".integration.driverJar")).toURI().toURL(); - try (URLClassLoader loader = new URLClassLoader(new URL[] {jar}, getClass().getClassLoader())) { - Driver driver = (Driver) loader.loadClass(driverClass).getDeclaredConstructor().newInstance(); + try (URLClassLoader loader = createDriverClassLoader(jar, driverClass)) { + Class<?> selected = loader.loadClass(driverClass); + Assertions.assertEquals(jar, selected.getProtectionDomain().getCodeSource().getLocation(), + "The integration test must load the configured driver JAR"); + Driver driver = (Driver) selected.getDeclaredConstructor().newInstance(); for (String zone : new String[] {"UTC", "Asia/Shanghai", "America/New_York"}) { TimeZone.setDefault(TimeZone.getTimeZone(zone)); Properties properties = new Properties(); diff --git a/fe/be-java-extensions/jdbc-scanner/src/test/java/org/apache/doris/jdbc/MySqlTimestampIntegrationTest.java b/fe/be-java-extensions/jdbc-scanner/src/test/java/org/apache/doris/jdbc/MySqlTimestampIntegrationTest.java index 365aef08755..2486e499d84 100644 --- a/fe/be-java-extensions/jdbc-scanner/src/test/java/org/apache/doris/jdbc/MySqlTimestampIntegrationTest.java +++ b/fe/be-java-extensions/jdbc-scanner/src/test/java/org/apache/doris/jdbc/MySqlTimestampIntegrationTest.java @@ -19,8 +19,6 @@ package org.apache.doris.jdbc; import org.apache.doris.common.jni.vec.ColumnType; import org.apache.doris.common.jni.vec.VectorColumn; -import org.apache.doris.thrift.TJdbcOperation; -import org.apache.doris.thrift.TOdbcTableType; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -35,6 +33,7 @@ import java.sql.Connection; import java.sql.Driver; import java.sql.PreparedStatement; import java.sql.ResultSet; +import java.sql.SQLException; import java.sql.Statement; import java.time.Instant; import java.time.LocalDateTime; @@ -61,6 +60,83 @@ class MySqlTimestampIntegrationTest { runMatrix(true); } + @Test + void testReadIgnoresCachedServerTimezoneAfterUtcReset() throws Exception { + TimeZone original = TimeZone.getDefault(); + TimeZone.setDefault(TimeZone.getTimeZone("UTC")); + URL driverUrl = new File(System.getProperty("mysql.integration.driverJar")).toURI().toURL(); + try (URLClassLoader loader = new URLClassLoader(new URL[] {driverUrl}, getClass().getClassLoader())) { + Driver driver = (Driver) loader.loadClass(System.getProperty("mysql.integration.driverClass", + "com.mysql.cj.jdbc.Driver")).getDeclaredConstructor().newInstance(); + for (boolean serverPrepared : new boolean[] {false, true}) { + String baseUrl = System.getProperty("mysql.integration.url"); + String url = baseUrl + (baseUrl.contains("?") ? "&" : "?") + + "useTimezone=true&serverTimezone=Asia/Shanghai&useServerPrepStmts=" + serverPrepared; + Properties properties = new Properties(); + properties.setProperty("user", System.getProperty("mysql.integration.user", "root")); + properties.setProperty("password", System.getProperty("mysql.integration.password", "")); + try (Connection connection = driver.connect(url, properties)) { + seed(connection, "+08:00"); + MySQLJdbcExecutor handler = handler(); + try (PreparedStatement statement = prepare(handler, connection, + "SELECT CAST(event_time AS CHAR) FROM timestamp_roundtrip ORDER BY id", 100); + ResultSet rows = statement.executeQuery()) { + for (String value : VALUES) { + Assertions.assertTrue(rows.next()); + Assertions.assertEquals(LocalDateTime.ofInstant(Instant.parse(value), ZoneOffset.UTC), + read(handler, rows, 1, INSTANT_TYPE)); + } + Assertions.assertTrue(rows.next()); + Assertions.assertNull(read(handler, rows, 1, INSTANT_TYPE)); + Assertions.assertFalse(rows.next()); + } + } + } + } finally { + TimeZone.setDefault(original); + } + } + + @Test + void testZeroTimestampRespectsDriverPolicy() throws Exception { + URL driverUrl = new File(System.getProperty("mysql.integration.driverJar")).toURI().toURL(); + try (URLClassLoader loader = new URLClassLoader(new URL[] {driverUrl}, getClass().getClassLoader())) { + Driver driver = (Driver) loader.loadClass(System.getProperty("mysql.integration.driverClass", + "com.mysql.cj.jdbc.Driver")).getDeclaredConstructor().newInstance(); + for (String policy : new String[] {"convertToNull", "round", "exception"}) { + for (boolean serverPrepared : new boolean[] {false, true}) { + String baseUrl = System.getProperty("mysql.integration.url"); + String url = baseUrl + (baseUrl.contains("?") ? "&" : "?") + + "zeroDateTimeBehavior=" + policy + "&useServerPrepStmts=" + serverPrepared; + Properties properties = new Properties(); + properties.setProperty("user", System.getProperty("mysql.integration.user", "root")); + properties.setProperty("password", System.getProperty("mysql.integration.password", "")); + try (Connection connection = driver.connect(url, properties)) { + try (Statement statement = connection.createStatement()) { + statement.execute("SET SESSION sql_mode = ''"); + statement.execute("CREATE TEMPORARY TABLE zero_timestamp (ts TIMESTAMP(6) NULL)"); + statement.execute("INSERT INTO zero_timestamp VALUES ('0000-00-00 00:00:00')"); + } + MySQLJdbcExecutor handler = handler(); + try (PreparedStatement statement = prepare(handler, connection, + "SELECT CAST(ts AS CHAR) FROM zero_timestamp", 100); + ResultSet rows = statement.executeQuery()) { + Assertions.assertTrue(rows.next()); + if (policy.equals("exception")) { + Assertions.assertThrows(SQLException.class, + () -> read(handler, rows, 1, INSTANT_TYPE)); + } else { + Assertions.assertEquals(policy.equals("round") + ? LocalDateTime.of(1, 1, 1, 0, 0) : null, + read(handler, rows, 1, INSTANT_TYPE)); + } + } + } + } + } + } + } + private void runMatrix(boolean write) throws Exception { TimeZone original = TimeZone.getDefault(); URL driverUrl = new File(System.getProperty("mysql.integration.driverJar")).toURI().toURL(); @@ -114,39 +190,37 @@ class MySqlTimestampIntegrationTest { } } - private MySQLJdbcExecutor executor(Connection connection, TJdbcOperation operation, String sql) throws Exception { - // Bypass only the JNI bootstrap; statements, result sets and timezone decoding use the real driver. + private MySQLJdbcExecutor handler() { + // OceanBase MySQL mode shares the same wire contract as MySQL. MySQLJdbcExecutor executor = Mockito.mock(MySQLJdbcExecutor.class, Mockito.CALLS_REAL_METHODS); - // The OceanBase MySQL-mode path must share this wire contract, not just the schema mapping. - TOdbcTableType tableType = TOdbcTableType.valueOf(System.getProperty("mysql.integration.tableType", "MYSQL")); - JdbcDataSourceConfig config = new JdbcDataSourceConfig().setOp(operation).setTableType(tableType); - executor.config = config; - executor.initializeStatement(connection, config, sql); + executor.config = new JdbcDataSourceConfig().setTableType(org.apache.doris.thrift.TOdbcTableType.valueOf( + System.getProperty("mysql.integration.tableType", "MYSQL"))); return executor; } private void verifyRead(Connection connection) throws Exception { - MySQLJdbcExecutor executor = executor(connection, TJdbcOperation.READ, - "SELECT event_time, local_time FROM timestamp_roundtrip ORDER BY id"); - try (PreparedStatement statement = (PreparedStatement) executor.stmt; ResultSet rows = statement.executeQuery()) { - executor.resultSet = rows; + MySQLJdbcExecutor executor = handler(); + try (PreparedStatement statement = prepare(executor, connection, + "SELECT CAST(event_time AS CHAR), local_time FROM timestamp_roundtrip ORDER BY id", 100); + ResultSet rows = statement.executeQuery()) { for (String value : VALUES) { Assertions.assertTrue(rows.next()); LocalDateTime expected = LocalDateTime.ofInstant(Instant.parse(value), ZoneOffset.UTC); - Assertions.assertEquals(expected, executor.getColumnValue(0, INSTANT_TYPE, new String[0])); - Assertions.assertEquals(expected, executor.getColumnValue(1, LOCAL_TYPE, new String[0])); + Assertions.assertEquals(expected, read(executor, rows, 1, INSTANT_TYPE)); + Assertions.assertEquals(expected, read(executor, rows, 2, LOCAL_TYPE)); } Assertions.assertTrue(rows.next()); - Assertions.assertNull(executor.getColumnValue(0, INSTANT_TYPE, new String[0])); - Assertions.assertNull(executor.getColumnValue(1, LOCAL_TYPE, new String[0])); + Assertions.assertNull(read(executor, rows, 1, INSTANT_TYPE)); + Assertions.assertNull(read(executor, rows, 2, LOCAL_TYPE)); Assertions.assertFalse(rows.next()); } } private void verifyWrite(Connection connection) throws Exception { - MySQLJdbcExecutor executor = executor(connection, TJdbcOperation.WRITE, - "INSERT INTO timestamp_roundtrip (id, event_time) VALUES (4, ?)"); + MySQLJdbcExecutor executor = handler(); Instant instant = Instant.parse(VALUES[0]); + executor.initializeStatement(connection, executor.config.setOp(org.apache.doris.thrift.TJdbcOperation.WRITE), + "INSERT INTO timestamp_roundtrip (id, event_time) VALUES (4, ?)"); VectorColumn column = Mockito.mock(VectorColumn.class); Mockito.when(column.getColumnPrimitiveType()).thenReturn(ColumnType.Type.TIMESTAMPTZ); Mockito.when(column.getTimeStampTz(0)).thenReturn(LocalDateTime.ofInstant(instant, ZoneOffset.UTC)); @@ -164,4 +238,16 @@ class MySqlTimestampIntegrationTest { rows.getBigDecimal(1).movePointRight(6).longValueExact()); } } + + private PreparedStatement prepare(MySQLJdbcExecutor executor, Connection connection, String sql, int batchSize) + throws SQLException { + executor.initializeStatement(connection, + executor.config.setOp(org.apache.doris.thrift.TJdbcOperation.READ).setBatchSize(batchSize), sql); + return (PreparedStatement) executor.stmt; + } + + private Object read(MySQLJdbcExecutor executor, ResultSet rows, int index, ColumnType type) throws SQLException { + executor.resultSet = rows; + return executor.getColumnValue(index - 1, type, new String[0]); + } } diff --git a/fe/be-java-extensions/jdbc-scanner/src/test/java/org/apache/doris/jdbc/OracleSessionTimeZoneTest.java b/fe/be-java-extensions/jdbc-scanner/src/test/java/org/apache/doris/jdbc/OracleSessionTimeZoneTest.java new file mode 100644 index 00000000000..ee525d7f1e4 --- /dev/null +++ b/fe/be-java-extensions/jdbc-scanner/src/test/java/org/apache/doris/jdbc/OracleSessionTimeZoneTest.java @@ -0,0 +1,73 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +package org.apache.doris.jdbc; + +import oracle.jdbc.OracleConnection; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.mockito.InOrder; +import org.mockito.Mockito; + +import java.sql.Connection; +import java.sql.DatabaseMetaData; +import java.sql.PreparedStatement; +import java.sql.SQLException; + +class OracleSessionTimeZoneTest { + @Test + void initializesEveryBorrowedConnectionBeforePreparingQuery() throws Exception { + OracleJdbcExecutor handler = Mockito.mock(OracleJdbcExecutor.class, Mockito.CALLS_REAL_METHODS); + for (int i = 0; i < 2; i++) { + Connection pooled = Mockito.mock(Connection.class); + OracleConnection physical = Mockito.mock(OracleConnection.class); + DatabaseMetaData metadata = Mockito.mock(DatabaseMetaData.class); + PreparedStatement statement = Mockito.mock(PreparedStatement.class); + Mockito.when(metadata.getDriverVersion()).thenReturn("23.26.2"); + Mockito.when(pooled.getMetaData()).thenReturn(metadata); + Mockito.when(pooled.unwrap(Connection.class)).thenReturn(physical); + Mockito.when(pooled.unwrap(OracleConnection.class)).thenReturn(physical); + Mockito.when(pooled.prepareStatement(Mockito.anyString(), Mockito.anyInt(), Mockito.anyInt())) + .thenReturn(statement); + handler.initializeStatement(pooled, new JdbcDataSourceConfig().setOp( + org.apache.doris.thrift.TJdbcOperation.READ).setBatchSize(100), "SELECT ts FROM t"); + InOrder order = Mockito.inOrder(physical, pooled); + order.verify(physical).setSessionTimeZone("UTC"); + order.verify(pooled).prepareStatement(Mockito.anyString(), Mockito.anyInt(), Mockito.anyInt()); + } + } + + @Test + void failedSessionInitializationAbortsBeforeQuery() throws Exception { + Connection pooled = Mockito.mock(Connection.class); + OracleConnection physical = Mockito.mock(OracleConnection.class); + DatabaseMetaData metadata = Mockito.mock(DatabaseMetaData.class); + Mockito.when(metadata.getDriverVersion()).thenReturn("23.26.2"); + Mockito.when(pooled.getMetaData()).thenReturn(metadata); + Mockito.when(pooled.unwrap(Connection.class)).thenReturn(physical); + Mockito.when(pooled.unwrap(OracleConnection.class)).thenReturn(physical); + Mockito.when(pooled.prepareStatement(Mockito.anyString(), Mockito.anyInt(), Mockito.anyInt())) + .thenReturn(Mockito.mock(PreparedStatement.class)); + Mockito.doThrow(new SQLException("session initialization failed")).when(physical).setSessionTimeZone("UTC"); + Assertions.assertThrows(SQLException.class, + () -> Mockito.mock(OracleJdbcExecutor.class, Mockito.CALLS_REAL_METHODS).initializeStatement(pooled, + new JdbcDataSourceConfig().setOp(org.apache.doris.thrift.TJdbcOperation.READ), "SELECT ts FROM t")); + Mockito.verify(pooled, Mockito.never()).prepareStatement( + Mockito.anyString(), Mockito.anyInt(), Mockito.anyInt()); + } +} diff --git a/fe/be-java-extensions/max-compute-connector/src/main/java/org/apache/doris/maxcompute/MaxComputeJniWriter.java b/fe/be-java-extensions/max-compute-connector/src/main/java/org/apache/doris/maxcompute/MaxComputeJniWriter.java index e95bb05aaf0..0bed49286d6 100644 --- a/fe/be-java-extensions/max-compute-connector/src/main/java/org/apache/doris/maxcompute/MaxComputeJniWriter.java +++ b/fe/be-java-extensions/max-compute-connector/src/main/java/org/apache/doris/maxcompute/MaxComputeJniWriter.java @@ -52,7 +52,10 @@ import org.apache.arrow.vector.Float4Vector; import org.apache.arrow.vector.Float8Vector; import org.apache.arrow.vector.IntVector; import org.apache.arrow.vector.SmallIntVector; +import org.apache.arrow.vector.TimeStampMicroTZVector; +import org.apache.arrow.vector.TimeStampMicroVector; import org.apache.arrow.vector.TimeStampMilliVector; +import org.apache.arrow.vector.TimeStampVector; import org.apache.arrow.vector.TinyIntVector; import org.apache.arrow.vector.VarBinaryVector; import org.apache.arrow.vector.VarCharVector; @@ -960,6 +963,12 @@ public class MaxComputeJniWriter extends JniWriter { BigDecimal bd = elem instanceof BigDecimal ? (BigDecimal) elem : new BigDecimal(elem.toString()); ((DecimalVector) vec).setSafe(idx, bd); + } else if (vec instanceof TimeStampMicroVector || vec instanceof TimeStampMicroTZVector) { + // Nested TIMESTAMPTZ uses the same UTC microsecond carrier as top-level TIMESTAMP writes. + java.time.Instant instant = elem instanceof java.sql.Timestamp + ? ((java.sql.Timestamp) elem).toInstant() + : ((LocalDateTime) elem).toInstant(java.time.ZoneOffset.UTC); + ((TimeStampVector) vec).setSafe(idx, instant.getEpochSecond() * 1_000_000L + instant.getNano() / 1000); } else if (vec instanceof StructVector) { StructVector structVec = (StructVector) vec; structVec.setIndexDefined(idx); diff --git a/fe/be-java-extensions/max-compute-connector/src/test/java/org/apache/doris/maxcompute/MaxComputeJniWriterTest.java b/fe/be-java-extensions/max-compute-connector/src/test/java/org/apache/doris/maxcompute/MaxComputeJniWriterTest.java index 6e1a6ad6312..cf73ca91b5f 100644 --- a/fe/be-java-extensions/max-compute-connector/src/test/java/org/apache/doris/maxcompute/MaxComputeJniWriterTest.java +++ b/fe/be-java-extensions/max-compute-connector/src/test/java/org/apache/doris/maxcompute/MaxComputeJniWriterTest.java @@ -262,4 +262,44 @@ public class MaxComputeJniWriterTest { return bytes; }; } + + @Test + public void testNestedTimestampWriteKeepsUtcMicros() throws Exception { + java.util.Map<String, String> params = new java.util.HashMap<>(); + params.put("endpoint", "http://localhost";); + params.put("project", "test_project"); + params.put("table", "events"); + params.put("txn_id", "1"); + params.put("write_session_id", "test_session"); + params.put("fe_host", "localhost"); + params.put("fe_port", "9020"); + MaxComputeJniWriter writer = new MaxComputeJniWriter(1, params); + try { + for (String timezone : new String[] {null, "UTC"}) { + try (BufferAllocator allocator = new RootAllocator(); + org.apache.arrow.vector.complex.ListVector vector = + org.apache.arrow.vector.complex.ListVector.empty("events", allocator)) { + vector.initializeChildrenFromFields(Collections.singletonList(new org.apache.arrow.vector.types.pojo.Field( + "element", org.apache.arrow.vector.types.pojo.FieldType.nullable( + new org.apache.arrow.vector.types.pojo.ArrowType.Timestamp( + org.apache.arrow.vector.types.TimeUnit.MICROSECOND, timezone)), null))); + vector.allocateNew(); + java.lang.reflect.Method write = MaxComputeJniWriter.class.getDeclaredMethod( + "writeListElement", org.apache.arrow.vector.FieldVector.class, int.class, Object.class); + write.setAccessible(true); + write.invoke(writer, vector, 0, java.util.Arrays.asList( + java.time.LocalDateTime.of(1969, 12, 31, 23, 59, 59, 999999000), null, + java.time.LocalDateTime.of(2021, 11, 7, 6, 30, 0, 123456000))); + org.apache.arrow.vector.TimeStampVector values = + (org.apache.arrow.vector.TimeStampVector) vector.getDataVector(); + Assert.assertEquals(-1L, values.get(0)); + Assert.assertTrue(values.isNull(1)); + Assert.assertEquals(1636266600123456L, values.get(2)); + } + } + } finally { + writer.close(); + } + } + } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/VarBinaryLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/VarBinaryLiteral.java index 8cc93cdd416..599692cba7d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/VarBinaryLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/VarBinaryLiteral.java @@ -92,33 +92,9 @@ public class VarBinaryLiteral extends LiteralExpr { @Override public int compareLiteral(LiteralExpr other) { if (other instanceof VarBinaryLiteral) { - byte[] thisBytes = this.value; - byte[] otherBytes = ((VarBinaryLiteral) other).value; - - int minLength = Math.min(thisBytes.length, otherBytes.length); - int i = 0; - for (i = 0; i < minLength; i++) { - if (Byte.toUnsignedInt(thisBytes[i]) < Byte.toUnsignedInt(otherBytes[i])) { - return -1; - } else if (Byte.toUnsignedInt(thisBytes[i]) > Byte.toUnsignedInt(otherBytes[i])) { - return 1; - } - } - if (thisBytes.length > otherBytes.length) { - if (thisBytes[i] == 0x00) { - return 0; - } else { - return 1; - } - } else if (thisBytes.length < otherBytes.length) { - if (otherBytes[i] == 0x00) { - return 0; - } else { - return -1; - } - } else { - return 0; - } + // VARBINARY is length-sensitive: trailing zero bytes are data, not string padding. + return com.google.common.primitives.UnsignedBytes.lexicographicalComparator() + .compare(value, ((VarBinaryLiteral) other).value); } if (other instanceof NullLiteral) { return 1; diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcTable.java index 25a4bcefaf1..bebb13e5b3b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcTable.java @@ -152,7 +152,18 @@ public class JdbcTable extends Table { if (i != 0) { sb.append(", "); } - sb.append("?"); + Column column = getColumn(insertCols.get(i)); + TOdbcTableType remoteType = getJdbcTableType(); + // Legacy drivers bind zoned timestamps as VARCHAR; parse their explicit offsets on the server. + if (column != null && column.getType().isTimeStampTz() + && (remoteType == TOdbcTableType.ORACLE || remoteType == TOdbcTableType.OCEANBASE_ORACLE)) { + sb.append("TO_TIMESTAMP_TZ(?, 'YYYY-MM-DD HH24:MI:SS.FF6 TZH:TZM')"); + } else if (column != null && column.getType().isTimeStampTz() && remoteType == TOdbcTableType.PRESTO) { + sb.append("io.prestosql.jdbc.PrestoDriver".equals(getDriverClass()) + ? "CAST(? AS TIMESTAMP(6) WITH TIME ZONE)" : "CAST(? AS TIMESTAMP WITH TIME ZONE)"); + } else { + sb.append("?"); + } } sb.append(")"); return sb.toString(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClient.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClient.java index ff4cc29eb10..445bcc2d182 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClient.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClient.java @@ -196,6 +196,32 @@ public abstract class JdbcClient { dataSource = null; } + public static Set<String> readSessionSqlMode(JdbcClientConfig config) { + // Mode lookup needs only one MySQL-protocol session, not metadata discovery or dialect-specific clients. + JdbcClient client = new JdbcMySQLClient(config, parseDbType(config.getJdbcUrl())); + try { + return client.getSessionSqlMode(); + } finally { + client.closeClient(); + } + } + + public Set<String> getSessionSqlMode() { + // Read the connected session so URL sessionVariables and server SQL mode are both honored. + try (Connection connection = getConnection(); + java.sql.Statement statement = connection.createStatement(); + ResultSet result = statement.executeQuery("SELECT @@SESSION.sql_mode")) { + if (!result.next()) { + throw new JdbcClientException("MySQL did not return its session SQL mode"); + } + return java.util.Arrays.stream(result.getString(1).split(",")) + .map(mode -> mode.trim().toUpperCase(java.util.Locale.ROOT)) + .collect(java.util.stream.Collectors.toSet()); + } catch (SQLException e) { + throw new JdbcClientException("Failed to read MySQL session SQL mode", e); + } + } + public Connection getConnection() throws JdbcClientException { ClassLoader oldClassLoader = Thread.currentThread().getContextClassLoader(); Connection conn; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/source/JdbcScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/source/JdbcScanNode.java index b66ccccf564..5ec952fc14f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/source/JdbcScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/source/JdbcScanNode.java @@ -40,6 +40,8 @@ import org.apache.doris.common.UserException; import org.apache.doris.datasource.ExternalFunctionRules; import org.apache.doris.datasource.ExternalScanNode; import org.apache.doris.datasource.jdbc.JdbcExternalTable; +import org.apache.doris.datasource.jdbc.client.JdbcClient; +import org.apache.doris.datasource.jdbc.client.JdbcClientConfig; import org.apache.doris.planner.PlanNodeId; import org.apache.doris.planner.ScanContext; import org.apache.doris.qe.ConnectContext; @@ -70,6 +72,7 @@ public class JdbcScanNode extends ExternalScanNode { private boolean isTableValuedFunction = false; private String query = ""; private boolean projectsTimestamps; + private java.util.Set<String> mysqlSqlMode; private JdbcTable tbl; private long catalogId; @@ -95,6 +98,18 @@ public class JdbcScanNode extends ExternalScanNode { jdbcType = tbl.getJdbcTableType(); tableName = tbl.getExternalTableName(); catalogId = tbl.getCatalogId(); + if (isTableValuedFunction && (jdbcType == TOdbcTableType.MYSQL || jdbcType == TOdbcTableType.OCEANBASE) + && desc.getSlots().stream().anyMatch(slot -> slot.getColumn().getType().isTimeStampTz())) { + // Bind the mode to this scan's connection snapshot, not a catalog that ALTER/DROP can replace. + // A private single-connection client avoids borrowing a pool that catalog reset can close. + mysqlSqlMode = JdbcClient.readSessionSqlMode(new JdbcClientConfig() + .setCatalog(String.valueOf(catalogId)) + .setJdbcUrl(tbl.getJdbcUrl()).setUser(tbl.getJdbcUser()).setPassword(tbl.getJdbcPasswd()) + .setDriverClass(tbl.getDriverClass()).setDriverUrl(tbl.getDriverUrl()) + .setConnectionPoolMinSize(0).setConnectionPoolMaxSize(1) + .setConnectionPoolMaxWaitTime(tbl.getConnectionPoolMaxWaitTime()) + .setConnectionPoolMaxLifeTime(tbl.getConnectionPoolMaxLifeTime())); + } } @@ -165,7 +180,12 @@ public class JdbcScanNode extends ExternalScanNode { while (leaf.isArrayType()) { leaf = ((ArrayType) leaf).getItemType(); } - if (jdbcType == TOdbcTableType.CLICKHOUSE && leaf.isTimeStampTz()) { + if ((jdbcType == TOdbcTableType.MYSQL || jdbcType == TOdbcTableType.OCEANBASE) + && col.getType().isTimeStampTz()) { + // Text projection prevents Connector/J's cached server zone from shifting UTC session fields. + columns.add("CAST(" + remoteName + " AS CHAR) AS " + remoteName); + projectsTimestamps = true; + } else if (jdbcType == TOdbcTableType.CLICKHOUSE && leaf.isTimeStampTz()) { // JDBC v1 loses the offset at DST overlaps, including for scalar ZonedDateTime. // Send epoch microseconds for both scalar and nested instants before JDBC decoding. columns.add(clickHouseTimestampProjection(remoteName, col.getType(), 0) + " AS " + remoteName); @@ -213,13 +233,178 @@ public class JdbcScanNode extends ExternalScanNode { if (!projectsTimestamps) { return query; } - String source = query.trim(); - if (source.endsWith(";")) { - source = source.substring(0, source.length() - 1); + boolean mysql = jdbcType == TOdbcTableType.MYSQL || jdbcType == TOdbcTableType.OCEANBASE; + String source = stripTerminalDelimiter(query.trim(), + mysql && mysqlSqlMode.contains("NO_BACKSLASH_ESCAPES"), mysql && mysqlSqlMode.contains("ANSI_QUOTES"), + mysql && mysqlSqlMode.contains("MSSQL")); + // WITH SESSION must stay outside the derived table; terminate trailing line comments with a newline. + int start = jdbcType == TOdbcTableType.TRINO ? trinoSessionQueryStart(source) : 0; + return source.substring(0, start) + "SELECT " + Joiner.on(", ").join(columns) + + " FROM (" + source.substring(start) + "\n) doris_jdbc_source"; + } + + private String stripTerminalDelimiter(String sql, boolean noBackslashEscapes, boolean ansiQuotes, boolean mssql) { + boolean mysql = jdbcType == TOdbcTableType.MYSQL || jdbcType == TOdbcTableType.OCEANBASE; + List<Integer> delimiters = new java.util.ArrayList<>(); + for (int i = 0; i < sql.length();) { + char c = sql.charAt(i); + if (Character.isWhitespace(c)) { + i++; + } else if (jdbcType == TOdbcTableType.CLICKHOUSE + && (sql.startsWith("//", i) || sql.startsWith("#!", i) || sql.startsWith("# ", i))) { + // ClickHouse comments may contain unmatched quotes; their contents end only at LF. + int newline = sql.indexOf('\n', i + 2); + i = newline < 0 ? sql.length() : newline + 1; + // MySQL's second dash needs a following whitespace/control character; --1 is arithmetic. + } else if ((sql.startsWith("--", i) && (!mysql || (i + 2 < sql.length() + && (Character.isWhitespace(sql.charAt(i + 2)) || Character.isISOControl(sql.charAt(i + 2)))))) + || (c == '#' + && (jdbcType == TOdbcTableType.MYSQL || jdbcType == TOdbcTableType.OCEANBASE))) { + while (i < sql.length() && sql.charAt(i) != '\n' && sql.charAt(i) != '\r') { + i++; + } + } else if (sql.startsWith("/*", i)) { + i = skipBlockComment(sql, i, jdbcType == TOdbcTableType.CLICKHOUSE); + } else if (jdbcType == TOdbcTableType.CLICKHOUSE && c == '$') { + i = skipClickHouseHeredoc(sql, i); + delimiters.clear(); + } else if (c == '\'' || c == '"' || c == '`' || (mysql && mssql && c == '[')) { + // MariaDB MSSQL identifiers use brackets; only a doubled closing bracket escapes the delimiter. + char quote = c == '[' ? ']' : c; + for (i++; i < sql.length(); i++) { + // ANSI_QUOTES makes double quotes identifiers, where backslashes never escape the closing quote. + // Both quoting modes belong to the remote session. + // ClickHouse also permits backslash escapes inside backtick-quoted identifiers. + if (sql.charAt(i) == '\\' && quote != ']' && !(mysql && noBackslashEscapes) + && !(mysql && ansiQuotes && quote == '"') + && (quote != '`' || jdbcType == TOdbcTableType.CLICKHOUSE) + && jdbcType != TOdbcTableType.TRINO && jdbcType != TOdbcTableType.PRESTO) { + i++; + } else if (sql.charAt(i) == quote) { + if (i + 1 < sql.length() && sql.charAt(i + 1) == quote) { + i++; + } else { + i++; + break; + } + } + } + delimiters.clear(); + } else { + if (c == ';') { + delimiters.add(i); + } else { + delimiters.clear(); + } + i++; + } + } + // A statement delimiter may precede trailing comments, which must stay outside SQL literals. + // Remove only the terminal delimiter run; interior delimiters still fail as multi-statements. + if (delimiters.isEmpty()) { + return sql; + } + StringBuilder result = new StringBuilder(sql); + for (int i = delimiters.size() - 1; i >= 0; i--) { + result.deleteCharAt(delimiters.get(i)); } - return "SELECT " + Joiner.on(", ").join(columns) + " FROM (" + source + ") doris_jdbc_source"; + return result.toString(); } + private static int skipClickHouseHeredoc(String sql, int start) { + // A dollar sign inside a bare identifier is not the beginning of a heredoc. + if (start > 0 && (isClickHouseWordCharacter(sql.charAt(start - 1)) || sql.charAt(start - 1) == '$')) { + return start + 1; + } + int tagEnd = start + 1; + while (tagEnd < sql.length() && isClickHouseWordCharacter(sql.charAt(tagEnd))) { + tagEnd++; + } + if (tagEnd == sql.length() || sql.charAt(tagEnd) != '$') { + return start + 1; + } + // Heredocs contain raw text: embedded quotes, comments and delimiters must not affect scanning. + String tag = sql.substring(start, tagEnd + 1); + int end = sql.indexOf(tag, tagEnd + 1); + return end < 0 ? start + 1 : end + tag.length(); + } + + private static boolean isClickHouseWordCharacter(char c) { + return c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c >= '0' && c <= '9' || c == '_'; + } + + private static int skipBlockComment(String sql, int start, boolean nested) { + // Trino, Presto and MySQL end at the first closing marker; ClickHouse permits nested comments. + int depth = 1; + int i = start + 2; + while (i < sql.length() && depth > 0) { + if (nested && sql.startsWith("/*", i)) { + depth++; + i += 2; + } else if (sql.startsWith("*/", i)) { + depth--; + i += 2; + } else { + i++; + } + } + return i; + } + + private static int trinoSessionQueryStart(String sql) { + int depth = 0; + int prefixWords = 0; + for (int i = 0; i < sql.length();) { + char c = sql.charAt(i); + if (c == '\'' || c == '"') { + char quote = c; + for (i++; i < sql.length(); i++) { + if (sql.charAt(i) == quote) { + if (i + 1 < sql.length() && sql.charAt(i + 1) == quote) { + i++; + } else { + i++; + break; + } + } + } + } else if (sql.startsWith("--", i)) { + int newline = sql.indexOf('\n', i + 2); + i = newline < 0 ? sql.length() : newline + 1; + } else if (sql.startsWith("/*", i)) { + i = skipBlockComment(sql, i, false); + } else if (c == '(') { + depth++; + i++; + } else if (c == ')') { + depth--; + i++; + } else if (Character.isLetterOrDigit(c) || c == '_') { + int start = i++; + while (i < sql.length() && (Character.isLetterOrDigit(sql.charAt(i)) || sql.charAt(i) == '_')) { + i++; + } + if (depth != 0) { + continue; + } + String word = sql.substring(start, i); + if (prefixWords < 2) { + if (!word.equalsIgnoreCase(prefixWords == 0 ? "WITH" : "SESSION")) { + return 0; + } + prefixWords++; + } else if (word.equalsIgnoreCase("SELECT") || word.equalsIgnoreCase("WITH") + || word.equalsIgnoreCase("TABLE") || word.equalsIgnoreCase("VALUES")) { + return start; + } + } else { + i++; + } + } + return 0; + } + + private String getJdbcQueryStr() { StringBuilder sql = new StringBuilder("SELECT "); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java index a3506783707..2a362ebebfd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java @@ -812,9 +812,10 @@ public class PaimonUtil { return null; } return Double.toString((Double) value); - // case binary: - // case varbinary: should not supported, because if return string with utf8, - // the data maybe be corrupted + case BINARY: + case VARBINARY: + // Native readers reconstruct omitted partition columns from lossless hex literals. + return value == null ? null : "0x" + com.google.common.io.BaseEncoding.base16().encode((byte[]) value); case DATE: if (value == null) { return null; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/VarBinaryLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/VarBinaryLiteral.java index e1f467ae2b3..0aad9d1ac58 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/VarBinaryLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/VarBinaryLiteral.java @@ -118,33 +118,9 @@ public class VarBinaryLiteral extends Literal implements ComparableLiteral { @Override public int compareTo(ComparableLiteral other) { if (other instanceof VarBinaryLiteral) { - byte[] thisBytes = this.byteValues; - byte[] otherBytes = ((VarBinaryLiteral) other).byteValues; - - int minLength = Math.min(thisBytes.length, otherBytes.length); - int i = 0; - for (i = 0; i < minLength; i++) { - if (Byte.toUnsignedInt(thisBytes[i]) < Byte.toUnsignedInt(otherBytes[i])) { - return -1; - } else if (Byte.toUnsignedInt(thisBytes[i]) > Byte.toUnsignedInt(otherBytes[i])) { - return 1; - } - } - if (thisBytes.length > otherBytes.length) { - if (thisBytes[i] == 0x00) { - return 0; - } else { - return 1; - } - } else if (thisBytes.length < otherBytes.length) { - if (otherBytes[i] == 0x00) { - return 0; - } else { - return -1; - } - } else { - return 0; - } + // VARBINARY is length-sensitive: trailing zero bytes are data, not string padding. + return com.google.common.primitives.UnsignedBytes.lexicographicalComparator() + .compare(byteValues, ((VarBinaryLiteral) other).byteValues); } if (other instanceof NullLiteral) { return 1; diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/VarBinaryLiteralOrderingTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/VarBinaryLiteralOrderingTest.java new file mode 100644 index 00000000000..4c6aa541a43 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/VarBinaryLiteralOrderingTest.java @@ -0,0 +1,34 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.analysis; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +class VarBinaryLiteralOrderingTest { + @Test + void preservesTrailingZerosAndUnsignedOrdering() throws Exception { + byte[][] values = { {}, {0}, {0, 0}, {1}, {1, 0}, {1, 0, 1}, {(byte) 0xff} }; + for (int i = 0; i < values.length; i++) { + for (int j = 0; j < values.length; j++) { + Assertions.assertEquals(Integer.compare(i, j), Integer.signum( + new VarBinaryLiteral(values[i]).compareLiteral(new VarBinaryLiteral(values[j])))); + } + } + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/jdbc/source/JdbcQueryWrappingTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/jdbc/source/JdbcQueryWrappingTest.java new file mode 100644 index 00000000000..059fbed9af5 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/jdbc/source/JdbcQueryWrappingTest.java @@ -0,0 +1,306 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.jdbc.source; + +import org.apache.doris.catalog.JdbcTable; +import org.apache.doris.datasource.jdbc.client.JdbcClientConfig; +import org.apache.doris.datasource.jdbc.client.JdbcMySQLClient; +import org.apache.doris.thrift.TOdbcTableType; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.mockito.MockedConstruction; +import org.mockito.MockedStatic; +import org.mockito.Mockito; + +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.util.Arrays; + +class JdbcQueryWrappingTest { + @Test + void mariaDbMssqlBracketIdentifiersPreserveContents() throws Exception { + JdbcScanNode node = Mockito.mock(JdbcScanNode.class, Mockito.CALLS_REAL_METHODS); + set(node, "jdbcType", TOdbcTableType.MYSQL); + set(node, "projectsTimestamps", true); + set(node, "columns", Arrays.asList("CAST(ts AS CHAR) AS ts")); + set(node, "mysqlSqlMode", java.util.Collections.singleton("MSSQL")); + Method wrap = JdbcScanNode.class.getDeclaredMethod("getTvfQuery"); + wrap.setAccessible(true); + for (String query : Arrays.asList("SELECT ts AS [x'y] FROM t", + "SELECT ts AS [x]]'y;--z] FROM t", "SELECT ts AS [x\\] FROM t")) { + set(node, "query", query + "; -- tail"); + Assertions.assertEquals("SELECT CAST(ts AS CHAR) AS ts FROM (" + query + + " -- tail\n) doris_jdbc_source", wrap.invoke(node)); + } + } + + @Test + void mysqlTvfCapturesModeBeforeCatalogReset() throws Exception { + org.apache.doris.catalog.Column column = new org.apache.doris.catalog.Column("ts", + org.apache.doris.catalog.ScalarType.createTimeStampTzType(6)); + JdbcTable table = new JdbcTable(1, "events", Arrays.asList(column), + org.apache.doris.catalog.Table.TableType.JDBC); + table.setJdbcTypeName("mysql"); + table.setExternalTableName("events"); + table.setJdbcUrl("jdbc:mysql://localhost/events"); + table.setDriverUrl("mysql-connector.jar"); + table.setDriverClass("com.mysql.cj.jdbc.Driver"); + table.setResourceName(""); + org.apache.doris.analysis.TupleDescriptor tuple = new org.apache.doris.analysis.TupleDescriptor( + new org.apache.doris.analysis.TupleId(0)); + tuple.setTable(table); + org.apache.doris.analysis.SlotDescriptor slot = new org.apache.doris.analysis.SlotDescriptor( + new org.apache.doris.analysis.SlotId(0), tuple); + slot.setColumn(column); + tuple.addSlot(slot); + try (MockedConstruction<JdbcMySQLClient> clients = Mockito.mockConstruction(JdbcMySQLClient.class, + (client, construction) -> { + JdbcClientConfig config = (JdbcClientConfig) construction.arguments().get(0); + Assertions.assertEquals(table.getJdbcUrl(), config.getJdbcUrl()); + Assertions.assertEquals(table.getDriverUrl(), config.getDriverUrl()); + Assertions.assertEquals(table.getDriverClass(), config.getDriverClass()); + if (construction.getCount() == 1) { + Mockito.when(client.getSessionSqlMode()) + .thenReturn(java.util.Collections.singleton("ANSI_QUOTES")); + } else { + Mockito.when(client.getSessionSqlMode()) + .thenThrow(new IllegalStateException("Mode lookup failed")); + } + })) { + JdbcScanNode node = new JdbcScanNode(new org.apache.doris.planner.PlanNodeId(0), tuple, true, + "SELECT ts, 1 AS \"x\\\" FROM events;", org.apache.doris.planner.ScanContext.EMPTY); + Assertions.assertEquals(1, clients.constructed().size()); + JdbcMySQLClient client = clients.constructed().get(0); + Mockito.verify(client).getSessionSqlMode(); + Mockito.verify(client).closeClient(); + // A catalog reset/drop after setup must not trigger any lookup or connection during serialization. + Mockito.reset(client); + try (MockedStatic<org.apache.doris.catalog.Env> env = Mockito.mockStatic( + org.apache.doris.catalog.Env.class)) { + env.when(org.apache.doris.catalog.Env::getCurrentEnv) + .thenThrow(new IllegalStateException("Catalog is no longer available")); + Method project = JdbcScanNode.class.getDeclaredMethod("createJdbcColumns"); + project.setAccessible(true); + project.invoke(node); + org.apache.doris.thrift.TPlanNode message = new org.apache.doris.thrift.TPlanNode(); + node.toThrift(message); + Assertions.assertEquals("SELECT CAST(`ts` AS CHAR) AS `ts` FROM (" + + "SELECT ts, 1 AS \"x\\\" FROM events\n) doris_jdbc_source", + message.jdbc_scan_node.getQueryString()); + Mockito.verifyNoInteractions(client); + } + Assertions.assertThrows(IllegalStateException.class, () -> new JdbcScanNode( + new org.apache.doris.planner.PlanNodeId(1), tuple, true, "SELECT ts FROM events", + org.apache.doris.planner.ScanContext.EMPTY)); + Assertions.assertEquals(2, clients.constructed().size()); + Mockito.verify(clients.constructed().get(1)).closeClient(); + } + } + + @Test + void preservesCommentsAndTrinoStatementPrefix() throws Exception { + JdbcScanNode node = org.mockito.Mockito.mock(JdbcScanNode.class, org.mockito.Mockito.CALLS_REAL_METHODS); + set(node, "jdbcType", TOdbcTableType.TRINO); + set(node, "projectsTimestamps", true); + set(node, "columns", Arrays.asList("at_timezone(ts, 'UTC') AS ts")); + Method wrap = JdbcScanNode.class.getDeclaredMethod("getTvfQuery"); + wrap.setAccessible(true); + set(node, "query", "SELECT ts FROM t; -- trailing"); + String sql = (String) wrap.invoke(node); + Assertions.assertTrue(sql.contains("FROM (SELECT ts FROM t -- trailing\n)"), sql); + set(node, "query", "WITH SESSION time_zone='UTC' SELECT ts FROM t;"); + sql = (String) wrap.invoke(node); + Assertions.assertTrue(sql.startsWith("WITH SESSION time_zone='UTC' SELECT at_timezone"), sql); + Assertions.assertTrue(sql.contains("FROM (SELECT ts FROM t\n)"), sql); + } + + @Test + void trinoBlockCommentsEndAtFirstClosingMarker() throws Exception { + JdbcScanNode node = org.mockito.Mockito.mock(JdbcScanNode.class, org.mockito.Mockito.CALLS_REAL_METHODS); + set(node, "jdbcType", TOdbcTableType.TRINO); + set(node, "projectsTimestamps", true); + set(node, "columns", Arrays.asList("at_timezone(ts, 'UTC') AS ts")); + Method wrap = JdbcScanNode.class.getDeclaredMethod("getTvfQuery"); + wrap.setAccessible(true); + String prefix = "WITH SESSION query_max_execution_time='1m' /* note /* marker */ "; + set(node, "query", prefix + "SELECT ts FROM t;"); + Assertions.assertEquals(prefix + "SELECT at_timezone(ts, 'UTC') AS ts FROM (SELECT ts FROM t\n)" + + " doris_jdbc_source", wrap.invoke(node)); + set(node, "query", "SELECT ts /* note /* marker */ FROM t; /* tail */"); + Assertions.assertEquals("SELECT at_timezone(ts, 'UTC') AS ts FROM (" + + "SELECT ts /* note /* marker */ FROM t /* tail */\n) doris_jdbc_source", wrap.invoke(node)); + set(node, "jdbcType", TOdbcTableType.CLICKHOUSE); + set(node, "columns", Arrays.asList("toUnixTimestamp64Micro(ts) AS ts")); + set(node, "query", "SELECT ts /* outer /* inner */ ; still comment */ FROM t;"); + Assertions.assertEquals("SELECT toUnixTimestamp64Micro(ts) AS ts FROM (" + + "SELECT ts /* outer /* inner */ ; still comment */ FROM t\n) doris_jdbc_source", wrap.invoke(node)); + } + + @Test + void mysqlAnsiQuotedIdentifierEndingInBackslash() throws Exception { + JdbcScanNode node = org.mockito.Mockito.mock(JdbcScanNode.class, org.mockito.Mockito.CALLS_REAL_METHODS); + set(node, "jdbcType", TOdbcTableType.MYSQL); + set(node, "projectsTimestamps", true); + set(node, "columns", Arrays.asList("CAST(ts AS CHAR) AS ts")); + set(node, "mysqlSqlMode", java.util.Collections.singleton("ANSI_QUOTES")); + Method wrap = JdbcScanNode.class.getDeclaredMethod("getTvfQuery"); + wrap.setAccessible(true); + String query = "SELECT ts, 1 AS \"x\\\" FROM t"; + set(node, "query", query + ";"); + Assertions.assertEquals("SELECT CAST(ts AS CHAR) AS ts FROM (" + query + "\n) doris_jdbc_source", + wrap.invoke(node)); + set(node, "mysqlSqlMode", new java.util.HashSet<>(Arrays.asList("ANSI_QUOTES", "NO_BACKSLASH_ESCAPES"))); + Assertions.assertEquals("SELECT CAST(ts AS CHAR) AS ts FROM (" + query + "\n) doris_jdbc_source", + wrap.invoke(node)); + // Without ANSI_QUOTES, a backslash escapes a double quote inside a string literal. + set(node, "mysqlSqlMode", java.util.Collections.emptySet()); + query = "SELECT ts, \"x\\\"; y\" AS value FROM t"; + set(node, "query", query + ";"); + Assertions.assertEquals("SELECT CAST(ts AS CHAR) AS ts FROM (" + query + "\n) doris_jdbc_source", + wrap.invoke(node)); + } + + @Test + void readsBothMysqlQuotingModesFromOneSession() throws Exception { + org.apache.doris.datasource.jdbc.client.JdbcClient client = org.mockito.Mockito.mock( + org.apache.doris.datasource.jdbc.client.JdbcClient.class, org.mockito.Mockito.CALLS_REAL_METHODS); + java.sql.Connection connection = org.mockito.Mockito.mock(java.sql.Connection.class); + java.sql.Statement statement = org.mockito.Mockito.mock(java.sql.Statement.class); + java.sql.ResultSet result = org.mockito.Mockito.mock(java.sql.ResultSet.class); + org.mockito.Mockito.doReturn(connection).when(client).getConnection(); + org.mockito.Mockito.when(connection.createStatement()).thenReturn(statement); + org.mockito.Mockito.when(statement.executeQuery("SELECT @@SESSION.sql_mode")).thenReturn(result); + org.mockito.Mockito.when(result.next()).thenReturn(true); + org.mockito.Mockito.when(result.getString(1)).thenReturn("ansi_quotes, NO_BACKSLASH_ESCAPES"); + Assertions.assertEquals(new java.util.HashSet<>(Arrays.asList("ANSI_QUOTES", "NO_BACKSLASH_ESCAPES")), + client.getSessionSqlMode()); + org.mockito.Mockito.verify(connection).close(); + org.mockito.Mockito.verify(statement).close(); + org.mockito.Mockito.verify(result).close(); + } + + @Test + void legacyTimestampBindsHaveExplicitServerCasts() throws Exception { + for (TOdbcTableType type : Arrays.asList(TOdbcTableType.ORACLE, TOdbcTableType.PRESTO)) { + JdbcTable table = new JdbcTable(1, "events", Arrays.asList(new org.apache.doris.catalog.Column( + "ts", org.apache.doris.catalog.ScalarType.createTimeStampTzType(6))), + org.apache.doris.catalog.Table.TableType.JDBC); + table.setJdbcTypeName(type.name().toLowerCase(java.util.Locale.ROOT)); + table.setExternalTableName("events"); + String sql = table.getInsertSql(Arrays.asList("ts")); + Assertions.assertTrue(sql.contains(type == TOdbcTableType.ORACLE ? "TO_TIMESTAMP_TZ(?" + : "CAST(? AS TIMESTAMP WITH TIME ZONE)"), sql); + } + } + + @Test + void mysqlWrappingHonorsSqlModeAndArithmeticDashes() throws Exception { + JdbcScanNode node = org.mockito.Mockito.mock(JdbcScanNode.class, org.mockito.Mockito.CALLS_REAL_METHODS); + set(node, "jdbcType", TOdbcTableType.MYSQL); + set(node, "projectsTimestamps", true); + set(node, "columns", Arrays.asList("CAST(ts AS CHAR) AS ts")); + set(node, "mysqlSqlMode", java.util.Collections.singleton("NO_BACKSLASH_ESCAPES")); + Method wrap = JdbcScanNode.class.getDeclaredMethod("getTvfQuery"); + wrap.setAccessible(true); + for (String query : Arrays.asList("SELECT ts, 1--1 AS n FROM t; -- tail", + "SELECT ts, '\\' AS text FROM t; # tail")) { + set(node, "query", query); + String sql = (String) wrap.invoke(node); + Assertions.assertFalse(sql.contains("FROM t;"), sql); + Assertions.assertTrue(sql.endsWith("\n) doris_jdbc_source"), sql); + } + set(node, "projectsTimestamps", false); + set(node, "mysqlSqlMode", null); + set(node, "query", "SELECT 1;"); + Assertions.assertEquals("SELECT 1;", wrap.invoke(node)); + } + + @Test + void mysqlProjectsOnlyInstantColumnsAsText() throws Exception { + org.apache.doris.catalog.Column instant = new org.apache.doris.catalog.Column("ts", + org.apache.doris.catalog.ScalarType.createTimeStampTzType(6)); + org.apache.doris.catalog.Column local = new org.apache.doris.catalog.Column("dt", + org.apache.doris.catalog.ScalarType.createDatetimeV2Type(6)); + JdbcTable table = new JdbcTable(1, "events", Arrays.asList(instant, local), + org.apache.doris.catalog.Table.TableType.JDBC); + table.setJdbcTypeName("mysql"); + table.setExternalTableName("events"); + org.apache.doris.analysis.TupleDescriptor tuple = new org.apache.doris.analysis.TupleDescriptor( + new org.apache.doris.analysis.TupleId(0)); + tuple.setTable(table); + int id = 0; + for (org.apache.doris.catalog.Column column : Arrays.asList(instant, local)) { + org.apache.doris.analysis.SlotDescriptor slot = new org.apache.doris.analysis.SlotDescriptor( + new org.apache.doris.analysis.SlotId(id++), tuple); + slot.setColumn(column); + tuple.addSlot(slot); + } + JdbcScanNode node = new JdbcScanNode(new org.apache.doris.planner.PlanNodeId(0), tuple, false, + org.apache.doris.planner.ScanContext.EMPTY); + Method project = JdbcScanNode.class.getDeclaredMethod("createJdbcColumns"); + project.setAccessible(true); + project.invoke(node); + Field columns = JdbcScanNode.class.getDeclaredField("columns"); + columns.setAccessible(true); + Assertions.assertEquals(Arrays.asList("CAST(`ts` AS CHAR) AS `ts`", "`dt`"), columns.get(node)); + } + + @Test + void clickHouseEscapedBackticksPreserveIdentifierContents() throws Exception { + assertClickHouseWrapped("SELECT ts, 1 AS `a\\`b` FROM t"); + assertClickHouseWrapped("SELECT ts, 1 AS `a\\`;b` FROM t"); + } + + @Test + void clickHouseLineCommentsDoNotChangeQuoteState() throws Exception { + for (String marker : Arrays.asList("//", "///", "#!", "# ")) { + for (String newline : Arrays.asList("\n", "\r\n")) { + assertClickHouseWrapped("SELECT ts FROM t " + marker + " 'note ` ;" + newline); + } + } + } + + @Test + void clickHouseHeredocsPreserveRawContents() throws Exception { + for (String literal : Arrays.asList("$h$foo'bar$h$", "$$'\\\"`; /* --$$", + "$tag_1$'text; $other$tag_1$", "$h$$h$")) { + assertClickHouseWrapped("SELECT ts FROM t WHERE length(" + literal + ") > 0"); + } + // An interior statement delimiter must remain visible to the remote parser. + assertClickHouseWrapped("SELECT ts, $h$'raw$h$ FROM t; SELECT ts FROM t"); + } + + private void assertClickHouseWrapped(String query) throws Exception { + JdbcScanNode node = org.mockito.Mockito.mock(JdbcScanNode.class, org.mockito.Mockito.CALLS_REAL_METHODS); + set(node, "jdbcType", TOdbcTableType.CLICKHOUSE); + set(node, "projectsTimestamps", true); + set(node, "columns", Arrays.asList("toUnixTimestamp64Micro(ts) AS ts")); + set(node, "query", query + ";"); + Method wrap = JdbcScanNode.class.getDeclaredMethod("getTvfQuery"); + wrap.setAccessible(true); + Assertions.assertEquals("SELECT toUnixTimestamp64Micro(ts) AS ts FROM (" + query + + "\n) doris_jdbc_source", wrap.invoke(node)); + } + + private static void set(Object target, String name, Object value) throws Exception { + Field field = JdbcScanNode.class.getDeclaredField(name); + field.setAccessible(true); + field.set(target, value); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/jdbc/source/JdbcTimestampProjectionTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/jdbc/source/JdbcTimestampProjectionTest.java index fb3623f7c38..9b05ab2acb4 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/jdbc/source/JdbcTimestampProjectionTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/jdbc/source/JdbcTimestampProjectionTest.java @@ -95,8 +95,9 @@ class JdbcTimestampProjectionTest { setField(node, "query", query); Method tvfQuery = JdbcScanNode.class.getDeclaredMethod("getTvfQuery"); tvfQuery.setAccessible(true); + // The newline prevents a trailing source comment from swallowing the derived-table closing parenthesis. Assertions.assertEquals(type.isDatetimeV2() ? query : "SELECT " + expected - + " FROM (SELECT event_time FROM event_stream) doris_jdbc_source", tvfQuery.invoke(node)); + + " FROM (SELECT event_time FROM event_stream\n) doris_jdbc_source", tvfQuery.invoke(node)); } @Test @@ -123,7 +124,7 @@ class JdbcTimestampProjectionTest { Method tvfQuery = JdbcScanNode.class.getDeclaredMethod("getTvfQuery"); tvfQuery.setAccessible(true); Assertions.assertEquals("SELECT " + columns.get(0) - + " FROM (SELECT event_time FROM event_stream) doris_jdbc_source", tvfQuery.invoke(node)); + + " FROM (SELECT event_time FROM event_stream\n) doris_jdbc_source", tvfQuery.invoke(node)); } @Test @@ -152,7 +153,7 @@ class JdbcTimestampProjectionTest { Method tvfQuery = JdbcScanNode.class.getDeclaredMethod("getTvfQuery"); tvfQuery.setAccessible(true); Assertions.assertEquals("SELECT " + columns.get(0) - + " FROM (SELECT events FROM event_stream) doris_jdbc_source", tvfQuery.invoke(node)); + + " FROM (SELECT events FROM event_stream\n) doris_jdbc_source", tvfQuery.invoke(node)); } private static void setField(JdbcScanNode node, String name, Object value) throws Exception { diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonBinaryPartitionTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonBinaryPartitionTest.java new file mode 100644 index 00000000000..e56a3f8e4a6 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonBinaryPartitionTest.java @@ -0,0 +1,36 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.paimon; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +class PaimonBinaryPartitionTest { + @Test + void binaryPartitionsPreserveBytesAndNull() throws Exception { + java.lang.reflect.Method render = PaimonUtil.class.getDeclaredMethod("serializePartitionValue", + org.apache.paimon.types.DataType.class, Object.class, String.class, boolean.class); + render.setAccessible(true); + for (org.apache.paimon.types.DataType type : new org.apache.paimon.types.DataType[] { + new org.apache.paimon.types.BinaryType(3), new org.apache.paimon.types.VarBinaryType()}) { + Assertions.assertEquals("0x00FF80", render.invoke(null, type, new byte[] {0, -1, -128}, "UTC", true)); + Assertions.assertEquals("0x", render.invoke(null, type, new byte[0], "UTC", false)); + Assertions.assertNull(render.invoke(null, type, null, "UTC", true)); + } + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/VarBinaryLiteralParserTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/VarBinaryLiteralParserTest.java index 4fb1249907b..5ef9e89590d 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/VarBinaryLiteralParserTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/VarBinaryLiteralParserTest.java @@ -78,7 +78,9 @@ public class VarBinaryLiteralParserTest extends TestWithFeService { public void testComparison() { VarBinaryLiteral v1 = extract("SELECT X'AB'"); VarBinaryLiteral v2 = extract("SELECT X'AB00'"); - Assertions.assertEquals(0, v1.compareTo(v2)); // trailing 00 considered equal + // Trailing zero bytes are part of the binary value, not padding. + Assertions.assertTrue(v1.compareTo(v2) < 0); + Assertions.assertTrue(v2.compareTo(v1) > 0); VarBinaryLiteral v3 = extract("SELECT X'AC'"); Assertions.assertTrue(v3.compareTo(v1) > 0); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/VarBinaryLiteralTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/VarBinaryLiteralTest.java index 89a17162545..4f26c514b97 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/VarBinaryLiteralTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/VarBinaryLiteralTest.java @@ -81,9 +81,10 @@ public class VarBinaryLiteralTest { Assertions.assertTrue(a.compareLiteral(c) < 0); Assertions.assertTrue(c.compareLiteral(a) > 0); - // different length with trailing zero semantics + // Trailing zero bytes are significant in VARBINARY, unlike string padding. VarBinaryLiteral withZero = new VarBinaryLiteral(new byte[] { 'a', 'b', 0x00 }); - Assertions.assertEquals(0, a.compareLiteral(withZero)); + Assertions.assertTrue(a.compareLiteral(withZero) < 0); + Assertions.assertTrue(withZero.compareLiteral(a) > 0); // null literal is treated as less than any value Assertions.assertTrue(a.compareLiteral(new NullLiteral()) > 0); diff --git a/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_binary_partitions.groovy b/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_binary_partitions.groovy index 6ebc420e749..b7e7e2a5737 100644 --- a/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_binary_partitions.groovy +++ b/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_binary_partitions.groovy @@ -73,6 +73,27 @@ suite("test_iceberg_write_binary_partitions", "p0,external,iceberg,external_dock sql "DROP TABLE IF EXISTS ${table}" } } + String identityTable = "binary_${format}_identity" + sql "DROP TABLE IF EXISTS ${identityTable}" + try { + spark_iceberg """CREATE TABLE demo.${dbName}.${identityTable} (id INT, binary_key BINARY) + USING iceberg PARTITIONED BY (binary_key) + TBLPROPERTIES ('format-version'='2', 'write.format.default'='${format}')""" + // The binary value and SQL NULL share the Iceberg path token "null", but not a partition. + sql "INSERT INTO ${identityTable} VALUES (1, X'9EE965'), (2, NULL), (3, X'9EE965')" + spark_iceberg "REFRESH TABLE demo.${dbName}.${identityTable}" + assertEquals([[2]], sql("SELECT id FROM ${identityTable} WHERE binary_key IS NULL")) + // Doris 4.1 cannot compare VARBINARY directly; Spark below still checks binary partition pruning. + assertEquals([[1], [3]], sql("SELECT id FROM ${identityTable} WHERE from_hex(binary_key) = '9EE965' ORDER BY id")) + assertSparkDorisResultEquals(spark_iceberg(""" + SELECT id FROM demo.${dbName}.${identityTable} WHERE binary_key IS NULL + """), [[2]]) + assertSparkDorisResultEquals(spark_iceberg(""" + SELECT id FROM demo.${dbName}.${identityTable} WHERE binary_key = X'9EE965' ORDER BY id + """), [[1], [3]]) + } finally { + sql "DROP TABLE IF EXISTS ${identityTable}" + } } } finally { sql "SWITCH internal" diff --git a/regression-test/suites/external_table_p0/jdbc/test_mysql_jdbc_catalog.groovy b/regression-test/suites/external_table_p0/jdbc/test_mysql_jdbc_catalog.groovy index beeddb065a3..d7c1d0086f2 100644 --- a/regression-test/suites/external_table_p0/jdbc/test_mysql_jdbc_catalog.groovy +++ b/regression-test/suites/external_table_p0/jdbc/test_mysql_jdbc_catalog.groovy @@ -395,13 +395,14 @@ suite("test_mysql_jdbc_catalog", "p0,external,mysql,external_docker,external_doc explain { sql ("SELECT timestamp0 from dt where DATE_TRUNC(date_sub(timestamp0,INTERVAL 9 HOUR),'hour') > '2011-03-03 17:39:05';") - contains "QUERY: SELECT `timestamp0` FROM `doris_test`.`dt`" + // UTC text projection prevents Connector/J from applying its cached connection time zone. + contains "QUERY: SELECT CAST(`timestamp0` AS CHAR) AS `timestamp0` FROM `doris_test`.`dt`" } explain { sql ("SELECT timestamp0 from dt where DATE_TRUNC(date_sub(timestamp0,INTERVAL 9 HOUR),'hour') > '2011-03-03 17:39:05' and timestamp0 > '2022-01-01';") // The JDBC session is UTC, while the literal is interpreted in the Doris session zone. - contains "QUERY: SELECT `timestamp0` FROM `doris_test`.`dt`" + contains "QUERY: SELECT CAST(`timestamp0` AS CHAR) AS `timestamp0` FROM `doris_test`.`dt`" notContains "WHERE" contains "PREDICATES:" contains "2021-12-31 16:00:00+00:00" --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
