This is an automated email from the ASF dual-hosted git repository. morningman pushed a commit to branch branch-2.1-lakehouse in repository https://gitbox.apache.org/repos/asf/doris.git
commit 032cc82fd0239747158ce6da79e2163f74c6a3ff Author: morningman <[email protected]> AuthorDate: Mon Feb 17 19:15:56 2025 +0800 [feature](sql-dialect)support convert hive view and presto view use sql convertor service #46308 --- .../doris/datasource/hive/HMSExternalTable.java | 50 +++++++++++++++++++++- .../doris/nereids/rules/analysis/BindRelation.java | 5 ++- .../java/org/apache/doris/qe/ConnectProcessor.java | 29 +------------ 3 files changed, 54 insertions(+), 30 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java index b056af3ebd6..6a29446cfd9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java @@ -64,6 +64,8 @@ import com.google.common.collect.BiMap; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; +import com.google.gson.Gson; +import com.google.gson.JsonObject; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.MapUtils; import org.apache.commons.lang3.StringUtils; @@ -82,7 +84,9 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.Base64; import java.util.Collection; import java.util.Collections; import java.util.HashMap; @@ -456,9 +460,51 @@ public class HMSExternalTable extends ExternalTable implements MTMVRelatedTableI public String getViewText() { String viewText = getViewExpandedText(); if (StringUtils.isNotEmpty(viewText)) { - return viewText; + if (!viewText.equals("/* Presto View */")) { + return viewText; + } + } + + String originalText = getViewOriginalText(); + return parseTrinoViewDefinition(originalText); + } + + /** + * Parse Trino/Presto view definition from the original text. + * The definition is stored in the format: /* Presto View: <base64-encoded-json> * / + * + * The base64 encoded JSON contains the following fields: + * { + * "originalSql": "SELECT * FROM employees", // The original SQL statement + * "catalog": "hive", // The data catalog name + * "schema": "mmc_hive", // The schema name + * ... + * } + * + * @param originalText The original view definition text + * @return The parsed SQL statement, or original text if parsing fails + */ + private String parseTrinoViewDefinition(String originalText) { + if (originalText == null || !originalText.contains("/* Presto View: ")) { + return originalText; + } + + try { + String base64String = originalText.substring( + originalText.indexOf("/* Presto View: ") + "/* Presto View: ".length(), + originalText.lastIndexOf(" */") + ).trim(); + byte[] decodedBytes = Base64.getDecoder().decode(base64String); + String decodedString = new String(decodedBytes, StandardCharsets.UTF_8); + JsonObject jsonObject = new Gson().fromJson(decodedString, JsonObject.class); + + if (jsonObject.has("originalSql")) { + return jsonObject.get("originalSql").getAsString(); + } + } catch (Exception e) { + LOG.warn("Decoding Presto view definition failed", e); } - return getViewOriginalText(); + return originalText; } public String getViewExpandedText() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java index d412348973f..a1ba0ad7f34 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java @@ -46,6 +46,7 @@ import org.apache.doris.nereids.analyzer.UnboundResultSink; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.hint.LeadingHint; import org.apache.doris.nereids.parser.NereidsParser; +import org.apache.doris.nereids.parser.SqlDialectHelper; import org.apache.doris.nereids.pattern.MatchingContext; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.properties.PhysicalProperties; @@ -451,11 +452,12 @@ public class BindRelation extends OneAnalysisRuleFactory { ConnectContext ctx = cascadesContext.getConnectContext(); String previousCatalog = ctx.getCurrentCatalog().getName(); String previousDb = ctx.getDatabase(); + String convertedSql = SqlDialectHelper.convertSqlByDialect(ddlSql, ctx.getSessionVariable()); // change catalog and db to hive catalog and db, so that we can parse and analyze the view sql in hive context. ctx.changeDefaultCatalog(hiveCatalog); ctx.setDatabase(hiveDb); try { - return parseAndAnalyzeView(table, ddlSql, cascadesContext); + return parseAndAnalyzeView(table, convertedSql, cascadesContext); } finally { // restore catalog and db in connect context ctx.changeDefaultCatalog(previousCatalog); @@ -515,4 +517,5 @@ public class BindRelation extends OneAnalysisRuleFactory { return part.getId(); }).collect(ImmutableList.toImmutableList()); } + } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java index 6e0f07ec3b8..a7c9a93a9f5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java @@ -57,14 +57,12 @@ import org.apache.doris.nereids.exceptions.NotSupportedException; import org.apache.doris.nereids.exceptions.ParseException; import org.apache.doris.nereids.glue.LogicalPlanAdapter; import org.apache.doris.nereids.minidump.MinidumpUtils; -import org.apache.doris.nereids.parser.Dialect; import org.apache.doris.nereids.parser.NereidsParser; +import org.apache.doris.nereids.parser.SqlDialectHelper; import org.apache.doris.nereids.stats.StatsErrorEstimator; import org.apache.doris.nereids.trees.plans.commands.ExplainCommand; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.trees.plans.logical.LogicalSqlCache; -import org.apache.doris.plugin.DialectConverterPlugin; -import org.apache.doris.plugin.PluginMgr; import org.apache.doris.proto.Data; import org.apache.doris.qe.QueryState.MysqlStateType; import org.apache.doris.qe.cache.CacheAnalyzer; @@ -79,7 +77,6 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import org.apache.commons.codec.digest.DigestUtils; -import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.thrift.TException; @@ -92,7 +89,6 @@ import java.util.List; import java.util.Map; import java.util.Optional; import java.util.UUID; -import javax.annotation.Nullable; /** * Process one connection, the life cycle is the same as connection @@ -235,7 +231,7 @@ public abstract class ConnectProcessor { MetricRepo.COUNTER_REQUEST_ALL.increase(1L); } - String convertedStmt = convertOriginStmt(originStmt); + String convertedStmt = SqlDialectHelper.convertSqlByDialect(originStmt, ctx.getSessionVariable()); String sqlHash = DigestUtils.md5Hex(convertedStmt); ctx.setSqlHash(sqlHash); @@ -431,27 +427,6 @@ public abstract class ConnectProcessor { return null; } - private String convertOriginStmt(String originStmt) { - String convertedStmt = originStmt; - @Nullable Dialect sqlDialect = Dialect.getByName(ctx.getSessionVariable().getSqlDialect()); - if (sqlDialect != null && sqlDialect != Dialect.DORIS) { - PluginMgr pluginMgr = Env.getCurrentEnv().getPluginMgr(); - List<DialectConverterPlugin> plugins = pluginMgr.getActiveDialectPluginList(sqlDialect); - for (DialectConverterPlugin plugin : plugins) { - try { - String convertedSql = plugin.convertSql(originStmt, ctx.getSessionVariable()); - if (StringUtils.isNotEmpty(convertedSql)) { - convertedStmt = convertedSql; - break; - } - } catch (Throwable throwable) { - LOG.warn("Convert sql with dialect {} failed, plugin: {}, sql: {}, use origin sql.", - sqlDialect, plugin.getClass().getSimpleName(), originStmt, throwable); - } - } - } - return convertedStmt; - } // Use a handler for exception to avoid big try catch block which is a little hard to understand protected void handleQueryException(Throwable throwable, String origStmt, --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
