This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.2 by this push:
new df6eb31b45f branch-4.2: [fix](statistics) Quote identifiers in
statistics queries #67703 (#70088)
df6eb31b45f is described below
commit df6eb31b45f0530084478f69a7cb7bcb8bdf7b16
Author: morrySnow <[email protected]>
AuthorDate: Wed Nov 4 08:25:48 2026 +0800
branch-4.2: [fix](statistics) Quote identifiers in statistics queries
#67703 (#70088)
picked from master #67703
---
.../apache/doris/statistics/BaseAnalysisTask.java | 73 ++++++++++----------
.../doris/statistics/ExternalAnalysisTask.java | 14 ++--
.../apache/doris/statistics/HMSAnalysisTask.java | 4 +-
.../org/apache/doris/statistics/HistogramTask.java | 17 ++---
.../apache/doris/statistics/OlapAnalysisTask.java | 25 +++----
.../org/apache/doris/statistics/AnalyzeTest.java | 11 +++
.../doris/statistics/BaseAnalysisTaskTest.java | 6 +-
.../doris/statistics/HMSAnalysisTaskTest.java | 4 ++
.../doris/statistics/OlapAnalysisTaskTest.java | 80 ++++++++++++++++------
9 files changed, 147 insertions(+), 87 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java
index 2e0c5cc4a98..c374cccbb50 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java
@@ -28,6 +28,7 @@ import org.apache.doris.catalog.Type;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.Status;
import org.apache.doris.common.util.DebugUtil;
+import org.apache.doris.common.util.SqlUtils;
import org.apache.doris.datasource.CatalogIf;
import org.apache.doris.metric.MetricRepo;
import org.apache.doris.qe.AuditLogHelper;
@@ -89,8 +90,8 @@ public abstract class BaseAnalysisTask {
public static final String ANALYZE_SKIP_LONG_STRING_COLUMN_MARKER =
"ANALYZE_SKIP_LONG_STRING_COLUMN";
protected static final String FULL_ANALYZE_TEMPLATE = "WITH cte1 AS ("
- + "SELECT `${colName}`${lengthAssert} "
- + "FROM `${catalogName}`.`${dbName}`.`${tblName}` ${index}), "
+ + "SELECT ${colName}${lengthAssert} "
+ + "FROM ${catalogName}.${dbName}.${tblName} ${index}), "
+ "cte2 AS ("
+ "SELECT CONCAT(${tblId}, '-', ${idxId}, '-', '${colId}') AS
`id`, "
+ "${catalogId} AS `catalog_id`, "
@@ -100,10 +101,10 @@ public abstract class BaseAnalysisTask {
+ "'${colId}' AS `col_id`, "
+ "NULL AS `part_id`, "
+ "COUNT(1) AS `row_count`, "
- + "NDV(`${colName}`) AS `ndv`, "
- + "COUNT(1) - COUNT(`${colName}`) AS `null_count`, "
- + "SUBSTRING(CAST(MIN(`${colName}`) AS STRING), 1, 1024) AS
`min`, "
- + "SUBSTRING(CAST(MAX(`${colName}`) AS STRING), 1, 1024) AS
`max`, "
+ + "NDV(${colName}) AS `ndv`, "
+ + "COUNT(1) - COUNT(${colName}) AS `null_count`, "
+ + "SUBSTRING(CAST(MIN(${colName}) AS STRING), 1, 1024) AS
`min`, "
+ + "SUBSTRING(CAST(MAX(${colName}) AS STRING), 1, 1024) AS
`max`, "
+ "${dataSizeFunction} AS `data_size`, "
+ "NOW() "
+ "FROM cte1), "
@@ -114,9 +115,9 @@ public abstract class BaseAnalysisTask {
+ "as `hot_value` "
+ "FROM ("
+ "SELECT ${subStringColName} as `hash_value`, "
- + "MAX(`${colName}`) as `column_key`, "
+ + "MAX(${colName}) as `column_key`, "
+ "COUNT(1) AS `count` "
- + "FROM cte1 WHERE `${colName}` IS NOT NULL "
+ + "FROM cte1 WHERE ${colName} IS NOT NULL "
+ "GROUP BY `hash_value` ORDER BY `count` DESC LIMIT
${hotValueCollectCount}) t) "
+ "SELECT * FROM cte2 CROSS JOIN cte3";
@@ -129,19 +130,19 @@ public abstract class BaseAnalysisTask {
+ "'${colId}' AS `col_id`, "
+ "NULL AS `part_id`, "
+ "COUNT(1) AS `row_count`, "
- + "NDV(`${colName}`) AS `ndv`, "
- + "COUNT(1) - COUNT(`${colName}`) AS `null_count`, "
- + "SUBSTRING(CAST(MIN(`${colName}`) AS STRING), 1, 1024) AS
`min`, "
- + "SUBSTRING(CAST(MAX(`${colName}`) AS STRING), 1, 1024) AS
`max`, "
+ + "NDV(${colName}) AS `ndv`, "
+ + "COUNT(1) - COUNT(${colName}) AS `null_count`, "
+ + "SUBSTRING(CAST(MIN(${colName}) AS STRING), 1, 1024) AS
`min`, "
+ + "SUBSTRING(CAST(MAX(${colName}) AS STRING), 1, 1024) AS
`max`, "
+ "${dataSizeFunction} AS `data_size`, "
+ "NOW() AS `update_time`, "
+ "null as `hot_value` "
- + "FROM (SELECT `${colName}`${lengthAssert} "
- + "FROM `${catalogName}`.`${dbName}`.`${tblName}` ${index})
__lc_t";
+ + "FROM (SELECT ${colName}${lengthAssert} "
+ + "FROM ${catalogName}.${dbName}.${tblName} ${index}) __lc_t";
protected static final String LINEAR_ANALYZE_TEMPLATE = "WITH cte1 AS ("
- + "SELECT `${colName}`${lengthAssert} "
- + "FROM `${catalogName}`.`${dbName}`.`${tblName}` ${index}
${sampleHints} ${limit} ${preAggHint}), "
+ + "SELECT ${colName}${lengthAssert} "
+ + "FROM ${catalogName}.${dbName}.${tblName} ${index}
${sampleHints} ${limit} ${preAggHint}), "
+ "cte2 AS ("
+ "SELECT CONCAT(${tblId}, '-', ${idxId}, '-', '${colId}') AS
`id`, "
+ "${catalogId} AS `catalog_id`, "
@@ -152,7 +153,7 @@ public abstract class BaseAnalysisTask {
+ "NULL AS `part_id`, "
+ "${rowCount} AS `row_count`, "
+ "${ndvFunction} as `ndv`, "
- + "ROUND(SUM(CASE WHEN `${colName}` IS NULL THEN 1 ELSE 0 END)
* ${scaleFactor}) AS `null_count`, "
+ + "ROUND(SUM(CASE WHEN ${colName} IS NULL THEN 1 ELSE 0 END) *
${scaleFactor}) AS `null_count`, "
+ "SUBSTRING(CAST(${min} AS STRING), 1, 1024) AS `min`, "
+ "SUBSTRING(CAST(${max} AS STRING), 1, 1024) AS `max`, "
+ "${dataSizeFunction} * ${scaleFactor} AS `data_size`, "
@@ -164,9 +165,9 @@ public abstract class BaseAnalysisTask {
+ "as `hot_value` "
+ "FROM ("
+ "SELECT ${subStringColName} as `hash_value`, "
- + "MAX(`${colName}`) as `column_key`, "
+ + "MAX(${colName}) as `column_key`, "
+ "COUNT(1) AS `count` "
- + "FROM cte1 WHERE `${colName}` IS NOT NULL "
+ + "FROM cte1 WHERE ${colName} IS NOT NULL "
+ "GROUP BY `hash_value` ORDER BY `count` DESC LIMIT
${hotValueCollectCount}) t) "
+ "SELECT * FROM cte2 CROSS JOIN cte3";
@@ -175,9 +176,9 @@ public abstract class BaseAnalysisTask {
+ "FROM "
+ "(SELECT "
+ "${subStringColName} AS `hash_value`, "
- + "`${colName}` AS `col_value`, "
- + "LENGTH(`${colName}`) as `len`${lengthAssert} "
- + "FROM `${catalogName}`.`${dbName}`.`${tblName}` ${index}
${sampleHints} ${limit}) as `t0` "
+ + "${colName} AS `col_value`, "
+ + "LENGTH(${colName}) as `len`${lengthAssert} "
+ + "FROM ${catalogName}.${dbName}.${tblName} ${index}
${sampleHints} ${limit}) as `t0` "
+ "${preAggHint} GROUP BY `t0`.`hash_value`), "
+ "cte2 AS ( "
+ "SELECT CONCAT('${tblId}', '-', '${idxId}', '-', '${colId}')
AS `id`, "
@@ -239,13 +240,13 @@ public abstract class BaseAnalysisTask {
+ "${partId} AS `part_id`, "
+ "'${colId}' AS `col_id`, "
+ "COUNT(1) AS `row_count`, "
- + "HLL_UNION(HLL_HASH(`${colName}`)) as ndv, "
- + "COUNT(1) - COUNT(`${colName}`) AS `null_count`, "
- + "SUBSTRING(CAST(MIN(`${colName}`) AS STRING), 1, 1024) AS `min`,
"
- + "SUBSTRING(CAST(MAX(`${colName}`) AS STRING), 1, 1024) AS `max`,
"
+ + "HLL_UNION(HLL_HASH(${colName})) as ndv, "
+ + "COUNT(1) - COUNT(${colName}) AS `null_count`, "
+ + "SUBSTRING(CAST(MIN(${colName}) AS STRING), 1, 1024) AS `min`, "
+ + "SUBSTRING(CAST(MAX(${colName}) AS STRING), 1, 1024) AS `max`, "
+ "${dataSizeFunction} AS `data_size`, "
+ "NOW() AS `update_time` "
- + "FROM `${catalogName}`.`${dbName}`.`${tblName}` ${index}
${partitionInfo}";
+ + "FROM ${catalogName}.${dbName}.${tblName} ${index}
${partitionInfo}";
protected static final String MERGE_PARTITION_TEMPLATE =
"SELECT CONCAT(${tblId}, '-', ${idxId}, '-', '${colId}') AS `id`, "
@@ -416,7 +417,7 @@ public abstract class BaseAnalysisTask {
}
} else {
if (column.getType().isStringType()) {
- return "SUM(LENGTH(`${colName}`))";
+ return "SUM(LENGTH(${colName}))";
} else {
return "COUNT(1) * " + column.getType().getSlotSize();
}
@@ -425,15 +426,15 @@ public abstract class BaseAnalysisTask {
protected String getStringTypeColName(Column column) {
if (column.getType().isStringType()) {
- return "xxhash_64(SUBSTRING(CAST(`${colName}` AS STRING), 1,
1024))";
+ return "xxhash_64(SUBSTRING(CAST(${colName} AS STRING), 1, 1024))";
} else {
- return "`${colName}`";
+ return "${colName}";
}
}
protected String getMinFunction() {
if (tableSample == null) {
- return "CAST(MIN(`${colName}`) as ${type}) ";
+ return "CAST(MIN(${colName}) as ${type}) ";
} else {
// Min value is not accurate while sample, so set it to NULL to
avoid optimizer generate bad plan.
return "NULL";
@@ -454,7 +455,7 @@ public abstract class BaseAnalysisTask {
// Max value is not accurate while sample, so set it to NULL to avoid
optimizer generate bad plan.
protected String getMaxFunction() {
if (tableSample == null) {
- return "CAST(MAX(`${colName}`) as ${type}) ";
+ return "CAST(MAX(${colName}) as ${type}) ";
} else {
return "NULL";
}
@@ -552,7 +553,7 @@ public abstract class BaseAnalysisTask {
}
}
}
- params.put("partName", "'" + StatisticsUtil.escapeColumnName(part)
+ "'");
+ params.put("partName",
StatisticsUtil.quote(StatisticsUtil.escapeSQL(part)));
params.put("partitionInfo", getPartitionInfo(part));
StringSubstitutor stringSubstitutor = new
StringSubstitutor(params);
sqls.add(stringSubstitutor.replace(PARTITION_ANALYZE_TEMPLATE));
@@ -636,11 +637,11 @@ public abstract class BaseAnalysisTask {
protected void addLengthAssertParam(Map<String, String> params) {
long maxLen =
org.apache.doris.common.Config.statistics_max_string_column_length;
if (col != null && col.getType().isStringType() && maxLen > 0) {
- String escapedColName =
StatisticsUtil.escapeColumnName(String.valueOf(info.colName));
+ String quotedColName =
SqlUtils.getIdentSql(String.valueOf(info.colName));
// The StringSubstitutor used by callers already has ${colName}
populated,
- // so we inline the escaped column name directly here.
+ // so we inline the quoted column name directly here.
params.put("lengthAssert",
- ", assert_true(`" + escapedColName + "` IS NULL OR
LENGTH(`" + escapedColName + "`) <= "
+ ", assert_true(" + quotedColName + " IS NULL OR LENGTH(" +
quotedColName + ") <= "
+ maxLen + ", '" +
ANALYZE_SKIP_LONG_STRING_COLUMN_MARKER + "') AS `__lc`");
} else {
params.put("lengthAssert", "");
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/ExternalAnalysisTask.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/ExternalAnalysisTask.java
index d37b3740f9d..d244201586a 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/statistics/ExternalAnalysisTask.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/statistics/ExternalAnalysisTask.java
@@ -20,8 +20,10 @@ package org.apache.doris.statistics;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.FeConstants;
import org.apache.doris.common.NotImplementedException;
+import org.apache.doris.common.util.SqlUtils;
import org.apache.doris.datasource.ExternalTable;
import org.apache.doris.qe.SessionVariable;
+import org.apache.doris.statistics.util.StatisticsUtil;
import org.apache.commons.text.StringSubstitutor;
@@ -67,7 +69,7 @@ public class ExternalAnalysisTask extends BaseAnalysisTask {
if (shouldCollectHotValue()) {
params.put("hotValueCollectCount",
String.valueOf(SessionVariable.getHotValueCollectCount()));
params.put("subStringColName", getStringTypeColName(col));
- params.put("rowCount2", "(SELECT COUNT(1) FROM cte1 WHERE
`${colName}` IS NOT NULL)");
+ params.put("rowCount2", "(SELECT COUNT(1) FROM cte1 WHERE
${colName} IS NOT NULL)");
template = FULL_ANALYZE_TEMPLATE;
} else {
template = FULL_ANALYZE_WITHOUT_HOT_VALUE_TEMPLATE;
@@ -86,11 +88,11 @@ public class ExternalAnalysisTask extends BaseAnalysisTask {
params.put("dbId", String.valueOf(db.getId()));
params.put("tblId", String.valueOf(tbl.getId()));
params.put("idxId", "-1");
- params.put("colName", info.colName);
- params.put("colId", info.colName);
- params.put("catalogName", catalog.getName());
- params.put("dbName", db.getFullName());
- params.put("tblName", tbl.getName());
+ params.put("colName", SqlUtils.getIdentSql(info.colName));
+ params.put("colId", StatisticsUtil.escapeSQL(info.colName));
+ params.put("catalogName", SqlUtils.getIdentSql(catalog.getName()));
+ params.put("dbName", SqlUtils.getIdentSql(db.getFullName()));
+ params.put("tblName", SqlUtils.getIdentSql(tbl.getName()));
params.put("sampleHints", getSampleHint());
params.put("limit", "");
params.put("scaleFactor", "1");
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/HMSAnalysisTask.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/HMSAnalysisTask.java
index 3b611f60000..16bd4941223 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/HMSAnalysisTask.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/HMSAnalysisTask.java
@@ -250,9 +250,9 @@ public class HMSAnalysisTask extends ExternalAnalysisTask {
if (distributionColumns.size() == 1 &&
distributionColumns.contains(col.getName().toLowerCase())) {
bucketFlag = true;
sb.append(LINEAR_ANALYZE_TEMPLATE);
- params.put("ndvFunction", "ROUND(NDV(`${colName}`) *
${scaleFactor})");
+ params.put("ndvFunction", "ROUND(NDV(${colName}) *
${scaleFactor})");
params.put("rowCount", "ROUND(COUNT(1) * ${scaleFactor})");
- params.put("rowCount2", "(SELECT COUNT(1) FROM cte1 WHERE
`${colName}` IS NOT NULL)");
+ params.put("rowCount2", "(SELECT COUNT(1) FROM cte1 WHERE
${colName} IS NOT NULL)");
} else {
sb.append(DUJ1_ANALYZE_TEMPLATE);
params.put("subStringColName", getStringTypeColName(col));
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/HistogramTask.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/HistogramTask.java
index aae2566700e..eeab8dc118f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/HistogramTask.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/HistogramTask.java
@@ -20,6 +20,7 @@ package org.apache.doris.statistics;
import org.apache.doris.catalog.Env;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.FeConstants;
+import org.apache.doris.common.util.SqlUtils;
import org.apache.doris.statistics.AnalysisInfo.AnalysisMethod;
import org.apache.doris.statistics.util.StatisticsUtil;
@@ -43,10 +44,10 @@ public class HistogramTask extends BaseAnalysisTask {
+ " ${idxId} AS idx_id, "
+ " '${colId}' AS col_id, "
+ " ${sampleRate} AS sample_rate, "
- + " HISTOGRAM(`${colName}`, ${maxBucketNum}) AS buckets, "
+ + " HISTOGRAM(${colName}, ${maxBucketNum}) AS buckets, "
+ " NOW() AS create_time "
+ "FROM "
- + " `${dbName}`.`${tblName}`";
+ + " ${dbName}.${tblName}";
public HistogramTask(AnalysisInfo info) {
super(info);
@@ -55,16 +56,16 @@ public class HistogramTask extends BaseAnalysisTask {
@Override
public void doExecute() throws Exception {
Map<String, String> params = new HashMap<>();
- params.put("internalDB", FeConstants.INTERNAL_DB_NAME);
- params.put("histogramStatTbl", StatisticConstants.HISTOGRAM_TBL_NAME);
+ params.put("internalDB",
SqlUtils.getIdentSql(FeConstants.INTERNAL_DB_NAME));
+ params.put("histogramStatTbl",
SqlUtils.getIdentSql(StatisticConstants.HISTOGRAM_TBL_NAME));
params.put("catalogId", String.valueOf(catalog.getId()));
params.put("dbId", String.valueOf(db.getId()));
params.put("tblId", String.valueOf(tbl.getId()));
params.put("idxId", String.valueOf(info.indexId));
- params.put("colId", String.valueOf(info.colName));
- params.put("dbName", db.getFullName());
- params.put("tblName", tbl.getName());
- params.put("colName", String.valueOf(info.colName));
+ params.put("colId",
StatisticsUtil.escapeSQL(String.valueOf(info.colName)));
+ params.put("dbName", SqlUtils.getIdentSql(db.getFullName()));
+ params.put("tblName", SqlUtils.getIdentSql(tbl.getName()));
+ params.put("colName",
SqlUtils.getIdentSql(String.valueOf(info.colName)));
params.put("sampleRate", getSampleRateFunction());
params.put("maxBucketNum", String.valueOf(info.maxBucketNum));
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java
index a7ac364c984..5b1f451004a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java
@@ -34,6 +34,7 @@ import org.apache.doris.common.FeConstants;
import org.apache.doris.common.Pair;
import org.apache.doris.common.util.DebugPointUtil;
import org.apache.doris.common.util.DebugUtil;
+import org.apache.doris.common.util.SqlUtils;
import org.apache.doris.qe.AutoCloseConnectContext;
import org.apache.doris.qe.SessionVariable;
import org.apache.doris.qe.StmtExecutor;
@@ -62,9 +63,9 @@ import java.util.stream.Collectors;
public class OlapAnalysisTask extends BaseAnalysisTask {
private static final String BASIC_STATS_TEMPLATE = "SELECT "
- + "SUBSTRING(CAST(MIN(`${colName}`) AS STRING), 1, 1024) as min, "
- + "SUBSTRING(CAST(MAX(`${colName}`) AS STRING), 1, 1024) as max "
- + "FROM `${dbName}`.`${tblName}` ${index}";
+ + "SUBSTRING(CAST(MIN(${colName}) AS STRING), 1, 1024) as min, "
+ + "SUBSTRING(CAST(MAX(${colName}) AS STRING), 1, 1024) as max "
+ + "FROM ${dbName}.${tblName} ${index}";
private boolean keyColumnSampleTooManyRows = false;
private boolean partitionColumnSampleTooManyRows = false;
@@ -360,12 +361,12 @@ public class OlapAnalysisTask extends BaseAnalysisTask {
params.put("dataSizeFunction", getDataSizeFunction(col, true));
params.put("rowCount2", "(SELECT SUM(`count`) FROM cte1 WHERE
`col_value` IS NOT NULL)");
} else {
- params.put("rowCount2", "(SELECT COUNT(1) FROM cte1 WHERE
`${colName}` IS NOT NULL)");
+ params.put("rowCount2", "(SELECT COUNT(1) FROM cte1 WHERE
${colName} IS NOT NULL)");
// For single unique key, use count as ndv.
if (isSingleUniqueKey()) {
params.put("ndvFunction", String.valueOf(tableRowCount));
} else {
- params.put("ndvFunction", "ROUND(NDV(`${colName}`) *
${scaleFactor})");
+ params.put("ndvFunction", "ROUND(NDV(${colName}) *
${scaleFactor})");
}
}
}
@@ -382,7 +383,7 @@ public class OlapAnalysisTask extends BaseAnalysisTask {
if (shouldCollectHotValue()) {
params.put("hotValueCollectCount",
String.valueOf(SessionVariable.getHotValueCollectCount()));
params.put("subStringColName", getStringTypeColName(col));
- params.put("rowCount2", "(SELECT COUNT(1) FROM cte1 WHERE
`${colName}` IS NOT NULL)");
+ params.put("rowCount2", "(SELECT COUNT(1) FROM cte1 WHERE
${colName} IS NOT NULL)");
runQuery(stringSubstitutor.replace(FULL_ANALYZE_TEMPLATE));
} else {
runQuery(stringSubstitutor.replace(FULL_ANALYZE_WITHOUT_HOT_VALUE_TEMPLATE));
@@ -435,7 +436,7 @@ public class OlapAnalysisTask extends BaseAnalysisTask {
@Override
protected String getPartitionInfo(String partitionName) {
- return "partition " + partitionName;
+ return "partition " + SqlUtils.getIdentSql(partitionName);
}
@Override
@@ -449,10 +450,10 @@ public class OlapAnalysisTask extends BaseAnalysisTask {
params.put("idxId", String.valueOf(info.indexId));
params.put("colId",
StatisticsUtil.escapeSQL(String.valueOf(info.colName)));
params.put("dataSizeFunction", getDataSizeFunction(col, false));
- params.put("catalogName", catalog.getName());
- params.put("dbName", db.getFullName());
- params.put("colName",
StatisticsUtil.escapeColumnName(String.valueOf(info.colName)));
- params.put("tblName", String.valueOf(tbl.getName()));
+ params.put("catalogName", SqlUtils.getIdentSql(catalog.getName()));
+ params.put("dbName", SqlUtils.getIdentSql(db.getFullName()));
+ params.put("colName",
SqlUtils.getIdentSql(String.valueOf(info.colName)));
+ params.put("tblName",
SqlUtils.getIdentSql(String.valueOf(tbl.getName())));
params.put("index", getIndex());
params.put("preAggHint", "");
addLengthAssertParam(params);
@@ -464,7 +465,7 @@ public class OlapAnalysisTask extends BaseAnalysisTask {
return "";
} else {
OlapTable olapTable = (OlapTable) this.tbl;
- return "index `" + olapTable.getIndexNameById(info.indexId) + "`";
+ return "index " +
SqlUtils.getIdentSql(olapTable.getIndexNameById(info.indexId));
}
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/statistics/AnalyzeTest.java
b/fe/fe-core/src/test/java/org/apache/doris/statistics/AnalyzeTest.java
index b41c464b9e9..a8c8f61cffb 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/statistics/AnalyzeTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/statistics/AnalyzeTest.java
@@ -105,6 +105,17 @@ public class AnalyzeTest extends TestWithFeService {
Database database,
@Mocked OlapTable olapTable)
throws Exception {
+ new Expectations() {
+ {
+ catalog.getName();
+ result = "internal";
+ database.getFullName();
+ result = "analysis_job_test";
+ olapTable.getName();
+ result = "t1";
+ }
+ };
+
new MockUp<OlapTable>() {
@Mock
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/statistics/BaseAnalysisTaskTest.java
b/fe/fe-core/src/test/java/org/apache/doris/statistics/BaseAnalysisTaskTest.java
index 9e2ea6d3510..78418370503 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/statistics/BaseAnalysisTaskTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/statistics/BaseAnalysisTaskTest.java
@@ -36,7 +36,7 @@ public class BaseAnalysisTaskTest {
String dataSizeFunction = olapAnalysisTask.getDataSizeFunction(column,
true);
Assertions.assertEquals("SUM(`column_length`)", dataSizeFunction);
dataSizeFunction = olapAnalysisTask.getDataSizeFunction(column, false);
- Assertions.assertEquals("SUM(LENGTH(`${colName}`))", dataSizeFunction);
+ Assertions.assertEquals("SUM(LENGTH(${colName}))", dataSizeFunction);
column = new Column("int_column", PrimitiveType.INT);
dataSizeFunction = olapAnalysisTask.getDataSizeFunction(column, false);
@@ -45,14 +45,14 @@ public class BaseAnalysisTaskTest {
Assertions.assertEquals("SUM(t1.count) * 4", dataSizeFunction);
String minFunction = olapAnalysisTask.getMinFunction();
- Assertions.assertEquals("CAST(MIN(`${colName}`) as ${type}) ",
minFunction);
+ Assertions.assertEquals("CAST(MIN(${colName}) as ${type}) ",
minFunction);
olapAnalysisTask.tableSample = new TableSample(true, 20L);
minFunction = olapAnalysisTask.getMinFunction();
Assertions.assertEquals("NULL", minFunction);
olapAnalysisTask.tableSample = null;
String maxFunction = olapAnalysisTask.getMaxFunction();
- Assertions.assertEquals("CAST(MAX(`${colName}`) as ${type}) ",
maxFunction);
+ Assertions.assertEquals("CAST(MAX(${colName}) as ${type}) ",
maxFunction);
olapAnalysisTask.tableSample = new TableSample(true, 20L);
maxFunction = olapAnalysisTask.getMaxFunction();
Assertions.assertEquals("NULL", maxFunction);
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/statistics/HMSAnalysisTaskTest.java
b/fe/fe-core/src/test/java/org/apache/doris/statistics/HMSAnalysisTaskTest.java
index 33721e5f8e4..22d0f30f14c 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/statistics/HMSAnalysisTaskTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/statistics/HMSAnalysisTaskTest.java
@@ -316,12 +316,16 @@ public class HMSAnalysisTaskTest {
{
tableIf.getId();
result = 30001;
+ tableIf.getName();
+ result = "test";
catalogIf.getId();
result = 10001;
catalogIf.getName();
result = "hms";
databaseIf.getId();
result = 20001;
+ databaseIf.getFullName();
+ result = "default";
}
};
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/statistics/OlapAnalysisTaskTest.java
b/fe/fe-core/src/test/java/org/apache/doris/statistics/OlapAnalysisTaskTest.java
index dd0113bf930..26aacda24ab 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/statistics/OlapAnalysisTaskTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/statistics/OlapAnalysisTaskTest.java
@@ -40,6 +40,7 @@ import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.FeConstants;
import org.apache.doris.common.Pair;
import org.apache.doris.common.util.DebugPointUtil;
+import org.apache.doris.common.util.SqlUtils;
import org.apache.doris.datasource.CatalogIf;
import org.apache.doris.persist.gson.GsonUtils;
import org.apache.doris.qe.SessionVariable;
@@ -153,6 +154,8 @@ public class OlapAnalysisTaskTest {
result = "catalogName";
databaseIf.getId();
result = 20001;
+ databaseIf.getFullName();
+ result = "dbName";
}
};
@@ -181,7 +184,7 @@ public class OlapAnalysisTaskTest {
@Mock
public void runQuery(String sql) {
- Assertions.assertEquals("WITH cte1 AS (SELECT `null` FROM
`catalogName`.`${dbName}`.`null` "
+ Assertions.assertEquals("WITH cte1 AS (SELECT `null` FROM
`catalogName`.`dbName`.`null` "
+ "${sampleHints} ${limit} ), cte2 AS (SELECT
CONCAT(30001, '-', -1, '-', 'null') AS `id`, "
+ "10001 AS `catalog_id`, 20001 AS `db_id`, 30001 AS
`tbl_id`, -1 AS `idx_id`, "
+ "'null' AS `col_id`, NULL AS `part_id`, ${rowCount}
AS `row_count`, ${ndvFunction} as `ndv`, "
@@ -215,7 +218,7 @@ public class OlapAnalysisTaskTest {
public void runQuery(String sql) {
Assertions.assertEquals("WITH cte1 AS (SELECT
MAX(t0.`col_value`) as `col_value`, COUNT(1) as `count`,"
+ " SUM(`len`) as `column_length` FROM (SELECT
${subStringColName} AS `hash_value`, "
- + "`null` AS `col_value`, LENGTH(`null`) as `len` FROM
`catalogName`.`${dbName}`.`null` "
+ + "`null` AS `col_value`, LENGTH(`null`) as `len` FROM
`catalogName`.`dbName`.`null` "
+ "${sampleHints} ${limit}) as `t0` GROUP BY
`t0`.`hash_value`), "
+ "cte2 AS ( SELECT CONCAT('30001', '-', '-1', '-',
'null') AS `id`, 10001 AS `catalog_id`, "
+ "20001 AS `db_id`, 30001 AS `tbl_id`, -1 AS
`idx_id`, 'null' AS `col_id`, NULL AS `part_id`, "
@@ -413,9 +416,9 @@ public class OlapAnalysisTaskTest {
new
OlapAnalysisTask.SampleCollectInfo(AnalyzeSampleAlgorithm.FULL, null));
Assertions.assertEquals("1", params.get("scaleFactor"));
Assertions.assertEquals("", params.get("sampleHints"));
- Assertions.assertEquals("(SELECT COUNT(1) FROM cte1 WHERE `${colName}`
IS NOT NULL)",
+ Assertions.assertEquals("(SELECT COUNT(1) FROM cte1 WHERE ${colName}
IS NOT NULL)",
params.get("rowCount2"));
- Assertions.assertEquals("ROUND(NDV(`${colName}`) * ${scaleFactor})",
params.get("ndvFunction"));
+ Assertions.assertEquals("ROUND(NDV(${colName}) * ${scaleFactor})",
params.get("ndvFunction"));
Assertions.assertNull(params.get("preAggHint"));
Assertions.assertEquals("COUNT(1)", params.get("rowCount"));
params.clear();
@@ -425,9 +428,9 @@ public class OlapAnalysisTaskTest {
new
OlapAnalysisTask.SampleCollectInfo(AnalyzeSampleAlgorithm.LINEAR,
Pair.of(Lists.newArrayList(1L, 2L), 100L)));
Assertions.assertEquals("TABLET(1, 2)", params.get("sampleHints"));
- Assertions.assertEquals("(SELECT COUNT(1) FROM cte1 WHERE `${colName}`
IS NOT NULL)",
+ Assertions.assertEquals("(SELECT COUNT(1) FROM cte1 WHERE ${colName}
IS NOT NULL)",
params.get("rowCount2"));
- Assertions.assertEquals("ROUND(NDV(`${colName}`) * ${scaleFactor})",
params.get("ndvFunction"));
+ Assertions.assertEquals("ROUND(NDV(${colName}) * ${scaleFactor})",
params.get("ndvFunction"));
params.clear();
// DUJ1 algorithm with sample tablets: rowCount2 and ndvFunction must
reference the cte1
@@ -459,7 +462,7 @@ public class OlapAnalysisTaskTest {
Assertions.assertEquals("TABLET(1, 2)", params.get("sampleHints"));
Assertions.assertEquals("SUM(`t1`.`count`) * COUNT(`t1`.`col_value`) /
(SUM(`t1`.`count`) - SUM(IF(`t1`.`count` = 1 and `t1`.`col_value` is not null,
1, 0)) + SUM(IF(`t1`.`count` = 1 and `t1`.`col_value` is not null, 1, 0)) *
SUM(`t1`.`count`) / 1000)", params.get("ndvFunction"));
Assertions.assertEquals("SUM(t1.count) * 4",
params.get("dataSizeFunction"));
- Assertions.assertEquals("`${colName}`",
params.get("subStringColName"));
+ Assertions.assertEquals("${colName}", params.get("subStringColName"));
Assertions.assertEquals("/*+PREAGGOPEN*/", params.get("preAggHint"));
params.clear();
@@ -518,7 +521,7 @@ public class OlapAnalysisTaskTest {
Pair.of(Lists.newArrayList(1L, 2L), 100L)));
Assertions.assertEquals("10.0", params.get("scaleFactor"));
Assertions.assertEquals("TABLET(1, 2)", params.get("sampleHints"));
- Assertions.assertEquals("ROUND(NDV(`${colName}`) * ${scaleFactor})",
params.get("ndvFunction"));
+ Assertions.assertEquals("ROUND(NDV(${colName}) * ${scaleFactor})",
params.get("ndvFunction"));
params.clear();
new MockUp<OlapAnalysisTask>() {
@@ -947,6 +950,43 @@ public class OlapAnalysisTaskTest {
}
}
+ @Test
+ public void testBuildSqlParamsQuotesIdentifiers() {
+ CatalogIf catalog = Mockito.mock(CatalogIf.class);
+ DatabaseIf database = Mockito.mock(DatabaseIf.class);
+ OlapTable table = Mockito.mock(OlapTable.class);
+ Mockito.when(catalog.getId()).thenReturn(1L);
+ Mockito.when(catalog.getName()).thenReturn("cat`alog");
+ Mockito.when(database.getId()).thenReturn(2L);
+ Mockito.when(database.getFullName()).thenReturn("db`name");
+ Mockito.when(table.getId()).thenReturn(3L);
+ Mockito.when(table.getName()).thenReturn("table`name");
+ Mockito.when(table.getIndexNameById(4L)).thenReturn("index`name");
+
+ OlapAnalysisTask task = new OlapAnalysisTask();
+ task.catalog = catalog;
+ task.db = database;
+ task.tbl = table;
+ task.col = new Column("col`'name", PrimitiveType.INT);
+ task.info = new AnalysisInfoBuilder()
+ .setIndexId(4L)
+ .setColName("col`'name")
+ .build();
+
+ Map<String, String> params = task.buildSqlParams();
+ Assertions.assertEquals("`cat``alog`", params.get("catalogName"));
+ Assertions.assertEquals("`db``name`", params.get("dbName"));
+ Assertions.assertEquals("`table``name`", params.get("tblName"));
+ Assertions.assertEquals("`col``'name`", params.get("colName"));
+ Assertions.assertEquals("col`''name", params.get("colId"));
+ Assertions.assertEquals("index `index``name`", params.get("index"));
+ Assertions.assertEquals("partition `part``name`",
task.getPartitionInfo("part`name"));
+
+ String sql = new
StringSubstitutor(params).replace(BaseAnalysisTask.FULL_ANALYZE_WITHOUT_HOT_VALUE_TEMPLATE);
+ Assertions.assertTrue(sql.contains("FROM (SELECT `col``'name` FROM "
+ + "`cat``alog`.`db``name`.`table``name` index `index``name`)
__lc_t"), sql);
+ }
+
@Test
public void testFullAnalyzeTemplateRendersLengthAssert() {
// Confirm the rendered FULL_ANALYZE_TEMPLATE wraps the base table in
a subquery
@@ -960,10 +1000,10 @@ public class OlapAnalysisTaskTest {
params.put("idxId", "3");
params.put("colId", "s");
params.put("dataSizeFunction", "100");
- params.put("catalogName", "internal");
- params.put("dbName", "db1");
- params.put("colName", "s");
- params.put("tblName", "tbl1");
+ params.put("catalogName", SqlUtils.getIdentSql("internal"));
+ params.put("dbName", SqlUtils.getIdentSql("db1"));
+ params.put("colName", SqlUtils.getIdentSql("s"));
+ params.put("tblName", SqlUtils.getIdentSql("tbl1"));
params.put("index", "");
params.put("lengthAssert",
", assert_true(`s` IS NULL OR LENGTH(`s`) <= 1024, '"
@@ -987,10 +1027,10 @@ public class OlapAnalysisTaskTest {
params.put("idxId", "3");
params.put("colId", "id");
params.put("dataSizeFunction", "100");
- params.put("catalogName", "internal");
- params.put("dbName", "db1");
- params.put("colName", "id");
- params.put("tblName", "tbl1");
+ params.put("catalogName", SqlUtils.getIdentSql("internal"));
+ params.put("dbName", SqlUtils.getIdentSql("db1"));
+ params.put("colName", SqlUtils.getIdentSql("id"));
+ params.put("tblName", SqlUtils.getIdentSql("tbl1"));
params.put("index", "");
params.put("lengthAssert", "");
StringSubstitutor stringSubstitutor = new StringSubstitutor(params);
@@ -1009,11 +1049,11 @@ public class OlapAnalysisTaskTest {
params.put("tblId", "2");
params.put("idxId", "3");
params.put("colId", "col1");
- params.put("colName", "col1");
+ params.put("colName", SqlUtils.getIdentSql("col1"));
params.put("dataSizeFunction", "SUM(LENGTH(`col1`))");
- params.put("catalogName", "internal");
- params.put("dbName", "db1");
- params.put("tblName", "tbl1");
+ params.put("catalogName", SqlUtils.getIdentSql("internal"));
+ params.put("dbName", SqlUtils.getIdentSql("db1"));
+ params.put("tblName", SqlUtils.getIdentSql("tbl1"));
params.put("index", "");
params.put("hotValueCollectCount", "10");
params.put("subStringColName", "`col1`");
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
