Gabriel39 commented on code in PR #68873:
URL: https://github.com/apache/doris/pull/68873#discussion_r4237052758
##########
fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/OracleJdbcExecutor.java:
##########
@@ -50,8 +52,27 @@ public OracleJdbcExecutor(byte[] thriftParams) throws
Exception {
@Override
protected void setTimestampTz(int parameterIndex, LocalDateTime value)
throws SQLException {
- // An unzoned TIMESTAMP bind is session-local for both Oracle TZ and
LOCAL TIME ZONE columns.
- preparedStatement.setObject(parameterIndex,
value.atOffset(ZoneOffset.UTC), Types.TIMESTAMP_WITH_TIMEZONE);
+ // Older Oracle drivers reject JDBC type 2014; the sink SQL parses an
explicit UTC offset.
+ preparedStatement.setString(parameterIndex,
value.format(TIMESTAMP_TZ_FORMATTER) + " +00:00");
+ }
+
+ @Override
+ protected void setTimestampTzNull(int parameterIndex) throws SQLException {
+ preparedStatement.setNull(parameterIndex, Types.VARCHAR);
+ }
+
+ @Override
+ protected void initializeStatement(Connection conn, JdbcDataSourceConfig
config, String sql) throws SQLException {
+ // ALTER SESSION alone leaves the driver's TSLTZ zone unset;
initialize every borrowed connection.
+ try {
+ Connection physical = conn.unwrap(Connection.class);
+ Class<?> oracleConnection =
Class.forName("oracle.jdbc.OracleConnection", true,
+ physical.getClass().getClassLoader());
Review Comment:
Left unchanged as requested: OceanBase Oracle-mode driver compatibility is
excluded from this revision. This thread remains unresolved.
##########
fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/source/JdbcScanNode.java:
##########
@@ -213,13 +219,154 @@ private String getTvfQuery() {
if (!projectsTimestamps) {
return query;
}
- String source = query.trim();
- if (source.endsWith(";")) {
- source = source.substring(0, source.length() - 1);
+ boolean mysql = jdbcType == TOdbcTableType.MYSQL || jdbcType ==
TOdbcTableType.OCEANBASE;
+ if (mysql && noBackslashEscapes == null) {
+ // Resolve only for wrapped MySQL queries and reuse the result for
EXPLAIN and scan serialization.
+ org.apache.doris.datasource.jdbc.JdbcExternalCatalog catalog =
+ (org.apache.doris.datasource.jdbc.JdbcExternalCatalog)
org.apache.doris.catalog.Env
+
.getCurrentEnv().getCatalogMgr().getCatalog(catalogId);
+ noBackslashEscapes =
catalog.getJdbcClient().isNoBackslashEscapes();
+ }
+ String source = stripTerminalDelimiter(query.trim(), mysql &&
noBackslashEscapes);
+ // WITH SESSION must stay outside the derived table; terminate
trailing line comments with a newline.
+ int start = jdbcType == TOdbcTableType.TRINO ?
trinoSessionQueryStart(source) : 0;
+ return source.substring(0, start) + "SELECT " + Joiner.on(",
").join(columns)
+ + " FROM (" + source.substring(start) + "\n)
doris_jdbc_source";
+ }
+
+ private String stripTerminalDelimiter(String sql, boolean
noBackslashEscapes) {
+ boolean mysql = jdbcType == TOdbcTableType.MYSQL || jdbcType ==
TOdbcTableType.OCEANBASE;
+ List<Integer> delimiters = new java.util.ArrayList<>();
+ for (int i = 0; i < sql.length();) {
+ char c = sql.charAt(i);
+ if (Character.isWhitespace(c)) {
+ i++;
+ // MySQL's second dash needs a following whitespace/control
character; --1 is arithmetic.
+ } else if ((sql.startsWith("--", i) && (!mysql || (i + 2 <
sql.length()
+ && (Character.isWhitespace(sql.charAt(i + 2)) ||
Character.isISOControl(sql.charAt(i + 2))))))
+ || (c == '#'
+ && (jdbcType == TOdbcTableType.MYSQL || jdbcType ==
TOdbcTableType.OCEANBASE))) {
+ while (i < sql.length() && sql.charAt(i) != '\n' &&
sql.charAt(i) != '\r') {
+ i++;
+ }
+ } else if (sql.startsWith("/*", i)) {
+ int depth = 1;
+ i += 2;
+ while (i < sql.length() && depth > 0) {
+ if (sql.startsWith("/*", i)) {
+ depth++;
+ i += 2;
+ } else if (sql.startsWith("*/", i)) {
+ depth--;
+ i += 2;
+ } else {
+ i++;
+ }
+ }
+ } else if (c == '\'' || c == '"' || c == '`') {
+ char quote = c;
+ for (i++; i < sql.length(); i++) {
+ // SQL mode belongs to the remote connection, not the
Doris session.
+ if (sql.charAt(i) == '\\' && !(mysql && noBackslashEscapes)
+ && quote != '`' && jdbcType !=
TOdbcTableType.TRINO && jdbcType != TOdbcTableType.PRESTO) {
Review Comment:
Fixed in 88625ed939. The wrapper now reads and caches both ANSI_QUOTES and
NO_BACKSLASH_ESCAPES from the same catalog session. Added coverage for an
identifier ending in a backslash, both modes together, and ordinary escaped
string literals. The FE tests pass, and the wrapped ANSI_QUOTES query was also
executed successfully on MySQL.
##########
fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java:
##########
@@ -812,9 +812,10 @@ private static String
serializePartitionValue(org.apache.paimon.types.DataType t
return null;
}
return Double.toString((Double) value);
- // case binary:
- // case varbinary: should not supported, because if return string
with utf8,
- // the data maybe be corrupted
+ case BINARY:
+ case VARBINARY:
+ // Native readers reconstruct omitted partition columns from
lossless hex literals.
+ return value == null ? null : "0x" +
com.google.common.io.BaseEncoding.base16().encode((byte[]) value);
case DATE:
Review Comment:
Left unchanged as requested: format V1 reader changes are excluded from this
revision. This thread remains unresolved.
##########
fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/OracleJdbcExecutor.java:
##########
@@ -50,8 +52,27 @@ public OracleJdbcExecutor(byte[] thriftParams) throws
Exception {
@Override
protected void setTimestampTz(int parameterIndex, LocalDateTime value)
throws SQLException {
- // An unzoned TIMESTAMP bind is session-local for both Oracle TZ and
LOCAL TIME ZONE columns.
- preparedStatement.setObject(parameterIndex,
value.atOffset(ZoneOffset.UTC), Types.TIMESTAMP_WITH_TIMEZONE);
+ // Older Oracle drivers reject JDBC type 2014; the sink SQL parses an
explicit UTC offset.
+ preparedStatement.setString(parameterIndex,
value.format(TIMESTAMP_TZ_FORMATTER) + " +00:00");
+ }
Review Comment:
Fixed in 88625ed939. The Oracle integration test now uses the production
TO_TIMESTAMP_TZ placeholder for both TIMESTAMP WITH TIME ZONE and TIMESTAMP
WITH LOCAL TIME ZONE. It also sets a conflicting NLS_TIMESTAMP_TZ_FORMAT to
ensure the explicit mask is exercised. The real Oracle write/readback test
passed under all three JVM time zones, including null and pre-epoch values.
##########
fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/source/JdbcScanNode.java:
##########
@@ -213,13 +219,154 @@ private String getTvfQuery() {
if (!projectsTimestamps) {
return query;
}
- String source = query.trim();
- if (source.endsWith(";")) {
- source = source.substring(0, source.length() - 1);
+ boolean mysql = jdbcType == TOdbcTableType.MYSQL || jdbcType ==
TOdbcTableType.OCEANBASE;
+ if (mysql && noBackslashEscapes == null) {
+ // Resolve only for wrapped MySQL queries and reuse the result for
EXPLAIN and scan serialization.
+ org.apache.doris.datasource.jdbc.JdbcExternalCatalog catalog =
+ (org.apache.doris.datasource.jdbc.JdbcExternalCatalog)
org.apache.doris.catalog.Env
+
.getCurrentEnv().getCatalogMgr().getCatalog(catalogId);
+ noBackslashEscapes =
catalog.getJdbcClient().isNoBackslashEscapes();
+ }
+ String source = stripTerminalDelimiter(query.trim(), mysql &&
noBackslashEscapes);
+ // WITH SESSION must stay outside the derived table; terminate
trailing line comments with a newline.
+ int start = jdbcType == TOdbcTableType.TRINO ?
trinoSessionQueryStart(source) : 0;
+ return source.substring(0, start) + "SELECT " + Joiner.on(",
").join(columns)
+ + " FROM (" + source.substring(start) + "\n)
doris_jdbc_source";
+ }
+
+ private String stripTerminalDelimiter(String sql, boolean
noBackslashEscapes) {
+ boolean mysql = jdbcType == TOdbcTableType.MYSQL || jdbcType ==
TOdbcTableType.OCEANBASE;
+ List<Integer> delimiters = new java.util.ArrayList<>();
+ for (int i = 0; i < sql.length();) {
+ char c = sql.charAt(i);
+ if (Character.isWhitespace(c)) {
+ i++;
+ // MySQL's second dash needs a following whitespace/control
character; --1 is arithmetic.
+ } else if ((sql.startsWith("--", i) && (!mysql || (i + 2 <
sql.length()
+ && (Character.isWhitespace(sql.charAt(i + 2)) ||
Character.isISOControl(sql.charAt(i + 2))))))
+ || (c == '#'
+ && (jdbcType == TOdbcTableType.MYSQL || jdbcType ==
TOdbcTableType.OCEANBASE))) {
+ while (i < sql.length() && sql.charAt(i) != '\n' &&
sql.charAt(i) != '\r') {
+ i++;
+ }
+ } else if (sql.startsWith("/*", i)) {
+ int depth = 1;
+ i += 2;
+ while (i < sql.length() && depth > 0) {
+ if (sql.startsWith("/*", i)) {
+ depth++;
+ i += 2;
+ } else if (sql.startsWith("*/", i)) {
+ depth--;
+ i += 2;
+ } else {
+ i++;
+ }
+ }
+ } else if (c == '\'' || c == '"' || c == '`') {
+ char quote = c;
+ for (i++; i < sql.length(); i++) {
+ // SQL mode belongs to the remote connection, not the
Doris session.
+ if (sql.charAt(i) == '\\' && !(mysql && noBackslashEscapes)
+ && quote != '`' && jdbcType !=
TOdbcTableType.TRINO && jdbcType != TOdbcTableType.PRESTO) {
+ i++;
+ } else if (sql.charAt(i) == quote) {
+ if (i + 1 < sql.length() && sql.charAt(i + 1) ==
quote) {
+ i++;
+ } else {
+ i++;
+ break;
+ }
+ }
+ }
+ delimiters.clear();
+ } else {
+ if (c == ';') {
+ delimiters.add(i);
+ } else {
+ delimiters.clear();
+ }
+ i++;
+ }
+ }
+ // A statement delimiter may precede trailing comments, which must
stay outside SQL literals.
+ // Remove only the terminal delimiter run; interior delimiters still
fail as multi-statements.
+ if (delimiters.isEmpty()) {
+ return sql;
+ }
+ StringBuilder result = new StringBuilder(sql);
+ for (int i = delimiters.size() - 1; i >= 0; i--) {
+ result.deleteCharAt(delimiters.get(i));
}
- return "SELECT " + Joiner.on(", ").join(columns) + " FROM (" + source
+ ") doris_jdbc_source";
+ return result.toString();
}
+ private static int trinoSessionQueryStart(String sql) {
+ int depth = 0;
+ int prefixWords = 0;
+ for (int i = 0; i < sql.length();) {
+ char c = sql.charAt(i);
+ if (c == '\'' || c == '"') {
+ char quote = c;
+ for (i++; i < sql.length(); i++) {
+ if (sql.charAt(i) == quote) {
+ if (i + 1 < sql.length() && sql.charAt(i + 1) ==
quote) {
+ i++;
+ } else {
+ i++;
+ break;
+ }
+ }
+ }
+ } else if (sql.startsWith("--", i)) {
+ int newline = sql.indexOf('\n', i + 2);
+ i = newline < 0 ? sql.length() : newline + 1;
+ } else if (sql.startsWith("/*", i)) {
+ int comments = 1;
+ i += 2;
+ while (i < sql.length() && comments > 0) {
+ if (sql.startsWith("/*", i)) {
+ comments++;
Review Comment:
Fixed in 88625ed939. Both wrapper scanners share the block-comment scanner
and stop at the first closing marker for Trino/Presto/MySQL. ClickHouse retains
its nested-comment behavior. Tests cover the WITH SESSION prefix and terminal
delimiter cases, plus a ClickHouse nested-comment case. The Trino queries were
also executed successfully against a real server.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
