Gabriel39 commented on code in PR #68873:
URL: https://github.com/apache/doris/pull/68873#discussion_r4237171050
##########
fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/source/JdbcScanNode.java:
##########
@@ -213,13 +219,151 @@ private String getTvfQuery() {
if (!projectsTimestamps) {
return query;
}
- String source = query.trim();
- if (source.endsWith(";")) {
- source = source.substring(0, source.length() - 1);
+ boolean mysql = jdbcType == TOdbcTableType.MYSQL || jdbcType ==
TOdbcTableType.OCEANBASE;
+ if (mysql && mysqlSqlMode == null) {
+ // Resolve only for wrapped MySQL queries and reuse the result for
EXPLAIN and scan serialization.
+ org.apache.doris.datasource.jdbc.JdbcExternalCatalog catalog =
+ (org.apache.doris.datasource.jdbc.JdbcExternalCatalog)
org.apache.doris.catalog.Env
+
.getCurrentEnv().getCatalogMgr().getCatalog(catalogId);
+ mysqlSqlMode = catalog.getJdbcClient().getSessionSqlMode();
+ }
+ String source = stripTerminalDelimiter(query.trim(),
+ mysql && mysqlSqlMode.contains("NO_BACKSLASH_ESCAPES"), mysql
&& mysqlSqlMode.contains("ANSI_QUOTES"));
+ // WITH SESSION must stay outside the derived table; terminate
trailing line comments with a newline.
+ int start = jdbcType == TOdbcTableType.TRINO ?
trinoSessionQueryStart(source) : 0;
+ return source.substring(0, start) + "SELECT " + Joiner.on(",
").join(columns)
+ + " FROM (" + source.substring(start) + "\n)
doris_jdbc_source";
+ }
+
+ private String stripTerminalDelimiter(String sql, boolean
noBackslashEscapes, boolean ansiQuotes) {
+ boolean mysql = jdbcType == TOdbcTableType.MYSQL || jdbcType ==
TOdbcTableType.OCEANBASE;
+ List<Integer> delimiters = new java.util.ArrayList<>();
+ for (int i = 0; i < sql.length();) {
+ char c = sql.charAt(i);
+ if (Character.isWhitespace(c)) {
+ i++;
+ // MySQL's second dash needs a following whitespace/control
character; --1 is arithmetic.
+ } else if ((sql.startsWith("--", i) && (!mysql || (i + 2 <
sql.length()
+ && (Character.isWhitespace(sql.charAt(i + 2)) ||
Character.isISOControl(sql.charAt(i + 2))))))
+ || (c == '#'
+ && (jdbcType == TOdbcTableType.MYSQL || jdbcType ==
TOdbcTableType.OCEANBASE))) {
+ while (i < sql.length() && sql.charAt(i) != '\n' &&
sql.charAt(i) != '\r') {
+ i++;
+ }
+ } else if (sql.startsWith("/*", i)) {
+ i = skipBlockComment(sql, i, jdbcType ==
TOdbcTableType.CLICKHOUSE);
+ } else if (c == '\'' || c == '"' || c == '`') {
+ char quote = c;
+ for (i++; i < sql.length(); i++) {
+ // ANSI_QUOTES makes double quotes identifiers, where
backslashes never escape the closing quote.
+ // Both quoting modes belong to the remote session.
+ if (sql.charAt(i) == '\\' && !(mysql && noBackslashEscapes)
+ && !(mysql && ansiQuotes && quote == '"') && quote
!= '`'
+ && jdbcType != TOdbcTableType.TRINO && jdbcType !=
TOdbcTableType.PRESTO) {
+ i++;
+ } else if (sql.charAt(i) == quote) {
+ if (i + 1 < sql.length() && sql.charAt(i + 1) ==
quote) {
+ i++;
+ } else {
+ i++;
+ break;
+ }
+ }
+ }
+ delimiters.clear();
+ } else {
+ if (c == ';') {
+ delimiters.add(i);
+ } else {
+ delimiters.clear();
+ }
+ i++;
+ }
+ }
+ // A statement delimiter may precede trailing comments, which must
stay outside SQL literals.
+ // Remove only the terminal delimiter run; interior delimiters still
fail as multi-statements.
+ if (delimiters.isEmpty()) {
+ return sql;
+ }
+ StringBuilder result = new StringBuilder(sql);
+ for (int i = delimiters.size() - 1; i >= 0; i--) {
+ result.deleteCharAt(delimiters.get(i));
}
- return "SELECT " + Joiner.on(", ").join(columns) + " FROM (" + source
+ ") doris_jdbc_source";
+ return result.toString();
}
+ private static int skipBlockComment(String sql, int start, boolean nested)
{
+ // Trino, Presto and MySQL end at the first closing marker; ClickHouse
permits nested comments.
+ int depth = 1;
+ int i = start + 2;
+ while (i < sql.length() && depth > 0) {
+ if (nested && sql.startsWith("/*", i)) {
+ depth++;
+ i += 2;
+ } else if (sql.startsWith("*/", i)) {
+ depth--;
+ i += 2;
+ } else {
+ i++;
+ }
+ }
+ return i;
+ }
+
+ private static int trinoSessionQueryStart(String sql) {
+ int depth = 0;
+ int prefixWords = 0;
+ for (int i = 0; i < sql.length();) {
+ char c = sql.charAt(i);
+ if (c == '\'' || c == '"') {
+ char quote = c;
+ for (i++; i < sql.length(); i++) {
+ if (sql.charAt(i) == quote) {
+ if (i + 1 < sql.length() && sql.charAt(i + 1) ==
quote) {
+ i++;
+ } else {
+ i++;
+ break;
+ }
+ }
+ }
+ } else if (sql.startsWith("--", i)) {
+ int newline = sql.indexOf('\n', i + 2);
Review Comment:
Left unchanged per the requested scope. WITH SESSION queries were already
wrapped incorrectly before this PR; this CR-only variant is a remaining gap in
the new support, rather than a regression of a previously working query. This
revision is limited to newly introduced regressions.
##########
fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/source/JdbcScanNode.java:
##########
@@ -213,13 +219,151 @@ private String getTvfQuery() {
if (!projectsTimestamps) {
return query;
}
- String source = query.trim();
- if (source.endsWith(";")) {
- source = source.substring(0, source.length() - 1);
+ boolean mysql = jdbcType == TOdbcTableType.MYSQL || jdbcType ==
TOdbcTableType.OCEANBASE;
+ if (mysql && mysqlSqlMode == null) {
+ // Resolve only for wrapped MySQL queries and reuse the result for
EXPLAIN and scan serialization.
+ org.apache.doris.datasource.jdbc.JdbcExternalCatalog catalog =
+ (org.apache.doris.datasource.jdbc.JdbcExternalCatalog)
org.apache.doris.catalog.Env
+
.getCurrentEnv().getCatalogMgr().getCatalog(catalogId);
+ mysqlSqlMode = catalog.getJdbcClient().getSessionSqlMode();
+ }
+ String source = stripTerminalDelimiter(query.trim(),
+ mysql && mysqlSqlMode.contains("NO_BACKSLASH_ESCAPES"), mysql
&& mysqlSqlMode.contains("ANSI_QUOTES"));
+ // WITH SESSION must stay outside the derived table; terminate
trailing line comments with a newline.
+ int start = jdbcType == TOdbcTableType.TRINO ?
trinoSessionQueryStart(source) : 0;
+ return source.substring(0, start) + "SELECT " + Joiner.on(",
").join(columns)
+ + " FROM (" + source.substring(start) + "\n)
doris_jdbc_source";
+ }
+
+ private String stripTerminalDelimiter(String sql, boolean
noBackslashEscapes, boolean ansiQuotes) {
+ boolean mysql = jdbcType == TOdbcTableType.MYSQL || jdbcType ==
TOdbcTableType.OCEANBASE;
+ List<Integer> delimiters = new java.util.ArrayList<>();
+ for (int i = 0; i < sql.length();) {
+ char c = sql.charAt(i);
+ if (Character.isWhitespace(c)) {
+ i++;
+ // MySQL's second dash needs a following whitespace/control
character; --1 is arithmetic.
+ } else if ((sql.startsWith("--", i) && (!mysql || (i + 2 <
sql.length()
+ && (Character.isWhitespace(sql.charAt(i + 2)) ||
Character.isISOControl(sql.charAt(i + 2))))))
+ || (c == '#'
+ && (jdbcType == TOdbcTableType.MYSQL || jdbcType ==
TOdbcTableType.OCEANBASE))) {
+ while (i < sql.length() && sql.charAt(i) != '\n' &&
sql.charAt(i) != '\r') {
+ i++;
+ }
+ } else if (sql.startsWith("/*", i)) {
+ i = skipBlockComment(sql, i, jdbcType ==
TOdbcTableType.CLICKHOUSE);
+ } else if (c == '\'' || c == '"' || c == '`') {
+ char quote = c;
+ for (i++; i < sql.length(); i++) {
+ // ANSI_QUOTES makes double quotes identifiers, where
backslashes never escape the closing quote.
+ // Both quoting modes belong to the remote session.
+ if (sql.charAt(i) == '\\' && !(mysql && noBackslashEscapes)
+ && !(mysql && ansiQuotes && quote == '"') && quote
!= '`'
+ && jdbcType != TOdbcTableType.TRINO && jdbcType !=
TOdbcTableType.PRESTO) {
+ i++;
+ } else if (sql.charAt(i) == quote) {
+ if (i + 1 < sql.length() && sql.charAt(i + 1) ==
quote) {
+ i++;
+ } else {
+ i++;
+ break;
+ }
+ }
+ }
+ delimiters.clear();
+ } else {
+ if (c == ';') {
+ delimiters.add(i);
+ } else {
+ delimiters.clear();
+ }
+ i++;
+ }
+ }
+ // A statement delimiter may precede trailing comments, which must
stay outside SQL literals.
+ // Remove only the terminal delimiter run; interior delimiters still
fail as multi-statements.
+ if (delimiters.isEmpty()) {
+ return sql;
+ }
+ StringBuilder result = new StringBuilder(sql);
+ for (int i = delimiters.size() - 1; i >= 0; i--) {
+ result.deleteCharAt(delimiters.get(i));
}
- return "SELECT " + Joiner.on(", ").join(columns) + " FROM (" + source
+ ") doris_jdbc_source";
+ return result.toString();
}
+ private static int skipBlockComment(String sql, int start, boolean nested)
{
+ // Trino, Presto and MySQL end at the first closing marker; ClickHouse
permits nested comments.
+ int depth = 1;
+ int i = start + 2;
+ while (i < sql.length() && depth > 0) {
+ if (nested && sql.startsWith("/*", i)) {
+ depth++;
+ i += 2;
+ } else if (sql.startsWith("*/", i)) {
+ depth--;
+ i += 2;
+ } else {
+ i++;
+ }
+ }
+ return i;
+ }
+
+ private static int trinoSessionQueryStart(String sql) {
+ int depth = 0;
+ int prefixWords = 0;
+ for (int i = 0; i < sql.length();) {
+ char c = sql.charAt(i);
+ if (c == '\'' || c == '"') {
+ char quote = c;
+ for (i++; i < sql.length(); i++) {
+ if (sql.charAt(i) == quote) {
+ if (i + 1 < sql.length() && sql.charAt(i + 1) ==
quote) {
+ i++;
+ } else {
+ i++;
+ break;
+ }
+ }
+ }
+ } else if (sql.startsWith("--", i)) {
+ int newline = sql.indexOf('\n', i + 2);
+ i = newline < 0 ? sql.length() : newline + 1;
+ } else if (sql.startsWith("/*", i)) {
+ i = skipBlockComment(sql, i, false);
+ } else if (c == '(') {
+ depth++;
Review Comment:
Left unchanged per the requested scope. The parenthesized WITH SESSION form
already failed before this PR, as noted in the finding. Preexisting bugs are
excluded from this revision.
##########
fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcTable.java:
##########
@@ -152,7 +152,18 @@ public String getInsertSql(List<String> insertCols) {
if (i != 0) {
sb.append(", ");
}
- sb.append("?");
+ Column column = getColumn(insertCols.get(i));
+ TOdbcTableType remoteType = getJdbcTableType();
+ // Legacy drivers bind zoned timestamps as VARCHAR; parse their
explicit offsets on the server.
+ if (column != null && column.getType().isTimeStampTz()
+ && (remoteType == TOdbcTableType.ORACLE || remoteType ==
TOdbcTableType.OCEANBASE_ORACLE)) {
+ sb.append("TO_TIMESTAMP_TZ(?, 'YYYY-MM-DD HH24:MI:SS.FF6
TZH:TZM')");
+ } else if (column != null && column.getType().isTimeStampTz() &&
remoteType == TOdbcTableType.PRESTO) {
Review Comment:
Left unchanged per the requested scope: mixed-version FE/BE compatibility
fixes are excluded. This thread remains unresolved.
##########
fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/TrinoJdbcExecutor.java:
##########
@@ -42,13 +42,22 @@
public class TrinoJdbcExecutor extends BaseJdbcExecutor {
private static final DateTimeFormatter TIMESTAMP_TZ_WRITE_FORMATTER =
DateTimeFormatter.ofPattern("uuuu-MM-dd HH:mm:ss.SSSSSS");
+ private static final DateTimeFormatter ARRAY_TIMESTAMP_FORMATTER = new
java.time.format.DateTimeFormatterBuilder()
+ .appendPattern("uuuu-MM-dd HH:mm:ss")
+ .appendFraction(java.time.temporal.ChronoField.NANO_OF_SECOND, 0,
9, true)
+ .appendLiteral(' ').appendZoneOrOffsetId().toFormatter();
public TrinoJdbcExecutor(byte[] thriftParams) throws Exception {
super(thriftParams);
}
@Override
protected void setTimestampTz(int parameterIndex, LocalDateTime value)
throws SQLException {
+ if (config != null && config.getTableType() ==
org.apache.doris.thrift.TOdbcTableType.PRESTO) {
+ // PrestoDB requires VARCHAR plus a server-side cast; its driver
rejects JDBC type 2014.
+ preparedStatement.setString(parameterIndex,
value.format(TIMESTAMP_TZ_WRITE_FORMATTER) + " UTC");
+ return;
Review Comment:
Left unchanged per the requested scope: mixed-version FE/BE compatibility
fixes are excluded. This thread remains unresolved.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
