This is an automated email from the ASF dual-hosted git repository. kxiao pushed a commit to branch branch-2.0 in repository https://gitbox.apache.org/repos/asf/doris.git
commit 2fedad33e4b7d64c09032759142199015c878f35 Author: AKIRA <[email protected]> AuthorDate: Mon Oct 16 16:49:53 2023 +0800 [refactor](stats) Use id instead name in analysis info (#25213) --- .../org/apache/doris/analysis/AnalyzeTblStmt.java | 6 ++ .../org/apache/doris/datasource/CatalogMgr.java | 9 ++- .../java/org/apache/doris/qe/ShowExecutor.java | 15 +++- .../org/apache/doris/statistics/AnalysisInfo.java | 87 ++++++---------------- .../doris/statistics/AnalysisInfoBuilder.java | 32 ++++---- .../apache/doris/statistics/AnalysisManager.java | 50 ++++++++----- .../apache/doris/statistics/BaseAnalysisTask.java | 31 +++----- .../apache/doris/statistics/HMSAnalysisTask.java | 6 +- .../org/apache/doris/statistics/HistogramTask.java | 4 +- .../apache/doris/statistics/JdbcAnalysisTask.java | 4 +- .../apache/doris/statistics/MVAnalysisTask.java | 4 +- .../apache/doris/statistics/OlapAnalysisTask.java | 12 +-- .../doris/statistics/StatisticsAutoCollector.java | 35 +-------- .../apache/doris/statistics/StatisticsCleaner.java | 2 +- .../doris/statistics/StatisticsCollector.java | 2 +- .../apache/doris/statistics/util/DBObjects.java | 8 +- .../doris/statistics/util/StatisticsUtil.java | 47 ++++++++++-- .../apache/doris/statistics/AnalysisJobTest.java | 34 ++++++++- .../doris/statistics/AnalysisTaskExecutorTest.java | 57 +++++++++++--- .../apache/doris/statistics/HistogramTaskTest.java | 31 +++++++- .../suites/statistics/analyze_stats.groovy | 2 +- 21 files changed, 275 insertions(+), 203 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/AnalyzeTblStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/AnalyzeTblStmt.java index 185bee1d132..8329d2c451a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/AnalyzeTblStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/AnalyzeTblStmt.java @@ -89,6 +89,7 @@ public class AnalyzeTblStmt extends AnalyzeStmt { private boolean isAllColumns; // after analyzed + private long catalogId; private long dbId; private TableIf table; @@ -130,6 +131,7 @@ public class AnalyzeTblStmt extends AnalyzeStmt { String tblName = tableName.getTbl(); CatalogIf catalog = analyzer.getEnv().getCatalogMgr() .getCatalogOrAnalysisException(catalogName); + this.catalogId = catalog.getId(); DatabaseIf db = catalog.getDbOrAnalysisException(dbName); dbId = db.getId(); table = db.getTableOrAnalysisException(tblName); @@ -329,4 +331,8 @@ public class AnalyzeTblStmt extends AnalyzeStmt { public boolean isAllColumns() { return isAllColumns; } + + public long getCatalogId() { + return catalogId; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java index 356c116ae42..5dd8a464119 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java @@ -92,7 +92,7 @@ public class CatalogMgr implements Writable, GsonPostProcessable { private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock(true); @SerializedName(value = "idToCatalog") - private final Map<Long, CatalogIf> idToCatalog = Maps.newConcurrentMap(); + private final Map<Long, CatalogIf<? extends DatabaseIf<? extends TableIf>>> idToCatalog = Maps.newConcurrentMap(); // this map will be regenerated from idToCatalog, so not need to persist. private final Map<String, CatalogIf> nameToCatalog = Maps.newConcurrentMap(); // record last used database of every catalog @@ -163,7 +163,7 @@ public class CatalogMgr implements Writable, GsonPostProcessable { return nameToCatalog.get(name); } - public CatalogIf getCatalog(long id) { + public CatalogIf<? extends DatabaseIf<? extends TableIf>> getCatalog(long id) { return idToCatalog.get(id); } @@ -173,7 +173,8 @@ public class CatalogMgr implements Writable, GsonPostProcessable { ErrorCode.ERR_UNKNOWN_CATALOG)); } - public <E extends Exception> CatalogIf getCatalogOrException(long id, Function<Long, E> e) throws E { + public <E extends Exception> CatalogIf<? extends DatabaseIf<? extends TableIf>> + getCatalogOrException(long id, Function<Long, E> e) throws E { CatalogIf catalog = idToCatalog.get(id); if (catalog == null) { throw e.apply(id); @@ -1174,7 +1175,7 @@ public class CatalogMgr implements Writable, GsonPostProcessable { internalCatalog = (InternalCatalog) idToCatalog.get(InternalCatalog.INTERNAL_CATALOG_ID); } - public Map<Long, CatalogIf> getIdToCatalog() { + public Map<Long, CatalogIf<? extends DatabaseIf<? extends TableIf>>> getIdToCatalog() { return idToCatalog; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java index 2c3cdf8ff32..5eb2e4187dd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java @@ -197,6 +197,7 @@ import org.apache.doris.statistics.Histogram; import org.apache.doris.statistics.StatisticsRepository; import org.apache.doris.statistics.TableStatsMeta; import org.apache.doris.statistics.query.QueryStatsUtil; +import org.apache.doris.statistics.util.StatisticsUtil; import org.apache.doris.system.Backend; import org.apache.doris.system.Diagnoser; import org.apache.doris.system.SystemInfoService; @@ -239,6 +240,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.function.Predicate; @@ -2586,9 +2588,16 @@ public class ShowExecutor { for (AnalysisInfo analysisInfo : results) { List<String> row = new ArrayList<>(); row.add(String.valueOf(analysisInfo.jobId)); - row.add(analysisInfo.catalogName); - row.add(analysisInfo.dbName); - row.add(analysisInfo.tblName); + CatalogIf<? extends DatabaseIf<? extends TableIf>> c = StatisticsUtil.findCatalog(analysisInfo.catalogId); + row.add(c.getName()); + Optional<? extends DatabaseIf<? extends TableIf>> databaseIf = c.getDb(analysisInfo.dbId); + row.add(databaseIf.isPresent() ? databaseIf.get().getFullName() : "DB may get deleted"); + if (databaseIf.isPresent()) { + Optional<? extends TableIf> table = databaseIf.get().getTable(analysisInfo.tblId); + row.add(table.isPresent() ? table.get().getName() : "Table may get deleted"); + } else { + row.add("DB may get deleted"); + } row.add(analysisInfo.colName); row.add(analysisInfo.jobType.toString()); row.add(analysisInfo.analysisType.toString()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfo.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfo.java index 4df6f9cf7af..2bf06d0c2c4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfo.java @@ -17,8 +17,6 @@ package org.apache.doris.statistics; -import org.apache.doris.catalog.Env; -import org.apache.doris.common.FeMetaVersion; import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; import org.apache.doris.persist.gson.GsonUtils; @@ -37,8 +35,6 @@ import java.io.DataOutput; import java.io.IOException; import java.lang.reflect.Type; import java.text.ParseException; -import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -91,14 +87,14 @@ public class AnalysisInfo implements Writable { @SerializedName("taskIds") public final List<Long> taskIds; - @SerializedName("catalogName") - public final String catalogName; + @SerializedName("catalogId") + public final long catalogId; - @SerializedName("dbName") - public final String dbName; + @SerializedName("dbId") + public final long dbId; - @SerializedName("tblName") - public final String tblName; + @SerializedName("tblId") + public final long tblId; // TODO: Map here is wired, List is enough @SerializedName("colToPartitions") @@ -183,7 +179,7 @@ public class AnalysisInfo implements Writable { @SerializedName("forceFull") public final boolean forceFull; - public AnalysisInfo(long jobId, long taskId, List<Long> taskIds, String catalogName, String dbName, String tblName, + public AnalysisInfo(long jobId, long taskId, List<Long> taskIds, long catalogId, long dbId, long tblId, Map<String, Set<String>> colToPartitions, Set<String> partitionNames, String colName, Long indexId, JobType jobType, AnalysisMode analysisMode, AnalysisMethod analysisMethod, AnalysisType analysisType, int samplePercent, long sampleRows, int maxBucketNum, long periodTimeInMs, String message, @@ -193,9 +189,9 @@ public class AnalysisInfo implements Writable { this.jobId = jobId; this.taskId = taskId; this.taskIds = taskIds; - this.catalogName = catalogName; - this.dbName = dbName; - this.tblName = tblName; + this.catalogId = catalogId; + this.dbId = dbId; + this.tblId = tblId; this.colToPartitions = colToPartitions; this.partitionNames = partitionNames; this.colName = colName; @@ -229,9 +225,9 @@ public class AnalysisInfo implements Writable { public String toString() { StringJoiner sj = new StringJoiner("\n", getClass().getName() + ":\n", "\n"); sj.add("JobId: " + jobId); - sj.add("CatalogName: " + catalogName); - sj.add("DBName: " + dbName); - sj.add("TableName: " + tblName); + sj.add("catalogId: " + catalogId); + sj.add("dbId: " + dbId); + sj.add("TableName: " + tblId); sj.add("ColumnName: " + colName); sj.add("TaskType: " + analysisType); sj.add("TaskMode: " + analysisMode); @@ -291,7 +287,8 @@ public class AnalysisInfo implements Writable { return null; } Gson gson = new Gson(); - Type type = new TypeToken<Map<String, Set<String>>>() {}.getType(); + Type type = new TypeToken<Map<String, Set<String>>>() { + }.getType(); return gson.fromJson(colToPartitionStr, type); } @@ -302,53 +299,15 @@ public class AnalysisInfo implements Writable { } public static AnalysisInfo read(DataInput dataInput) throws IOException { - if (Env.getCurrentEnvJournalVersion() < FeMetaVersion.VERSION_123) { - AnalysisInfoBuilder analysisInfoBuilder = new AnalysisInfoBuilder(); - analysisInfoBuilder.setJobId(dataInput.readLong()); - long taskId = dataInput.readLong(); - analysisInfoBuilder.setTaskId(taskId); - analysisInfoBuilder.setCatalogName(Text.readString(dataInput)); - analysisInfoBuilder.setDbName(Text.readString(dataInput)); - analysisInfoBuilder.setTblName(Text.readString(dataInput)); - int size = dataInput.readInt(); - Map<String, Set<String>> colToPartitions = new HashMap<>(); - for (int i = 0; i < size; i++) { - String k = Text.readString(dataInput); - int partSize = dataInput.readInt(); - Set<String> parts = new HashSet<>(); - for (int j = 0; j < partSize; j++) { - parts.add(Text.readString(dataInput)); - } - colToPartitions.put(k, parts); + String json = Text.readString(dataInput); + AnalysisInfo analysisInfo = GsonUtils.GSON.fromJson(json, AnalysisInfo.class); + if (analysisInfo.cronExprStr != null) { + try { + analysisInfo.cronExpression = new CronExpression(analysisInfo.cronExprStr); + } catch (ParseException e) { + LOG.warn("Cron expression of job is invalid, there is a bug", e); } - analysisInfoBuilder.setColToPartitions(colToPartitions); - analysisInfoBuilder.setColName(Text.readString(dataInput)); - analysisInfoBuilder.setIndexId(dataInput.readLong()); - analysisInfoBuilder.setJobType(JobType.valueOf(Text.readString(dataInput))); - analysisInfoBuilder.setAnalysisMode(AnalysisMode.valueOf(Text.readString(dataInput))); - analysisInfoBuilder.setAnalysisMethod(AnalysisMethod.valueOf(Text.readString(dataInput))); - analysisInfoBuilder.setAnalysisType(AnalysisType.valueOf(Text.readString(dataInput))); - analysisInfoBuilder.setSamplePercent(dataInput.readInt()); - analysisInfoBuilder.setSampleRows(dataInput.readInt()); - analysisInfoBuilder.setMaxBucketNum(dataInput.readInt()); - analysisInfoBuilder.setPeriodTimeInMs(dataInput.readLong()); - analysisInfoBuilder.setLastExecTimeInMs(dataInput.readLong()); - analysisInfoBuilder.setState(AnalysisState.valueOf(Text.readString(dataInput))); - analysisInfoBuilder.setScheduleType(ScheduleType.valueOf(Text.readString(dataInput))); - analysisInfoBuilder.setMessage(Text.readString(dataInput)); - analysisInfoBuilder.setExternalTableLevelTask(dataInput.readBoolean()); - return analysisInfoBuilder.build(); - } else { - String json = Text.readString(dataInput); - AnalysisInfo analysisInfo = GsonUtils.GSON.fromJson(json, AnalysisInfo.class); - if (analysisInfo.cronExprStr != null) { - try { - analysisInfo.cronExpression = new CronExpression(analysisInfo.cronExprStr); - } catch (ParseException e) { - LOG.warn("Cron expression of job is invalid, there is a bug", e); - } - } - return analysisInfo; } + return analysisInfo; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfoBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfoBuilder.java index b6d9aafe853..b6dc7b5dd74 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfoBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfoBuilder.java @@ -33,9 +33,9 @@ public class AnalysisInfoBuilder { private long jobId; private long taskId; private List<Long> taskIds; - private String catalogName; - private String dbName; - private String tblName; + private long catalogId; + private long dbId; + private long tblId; private Map<String, Set<String>> colToPartitions; private Set<String> partitionNames; private String colName; @@ -68,9 +68,9 @@ public class AnalysisInfoBuilder { jobId = info.jobId; taskId = info.taskId; taskIds = info.taskIds; - catalogName = info.catalogName; - dbName = info.dbName; - tblName = info.tblName; + catalogId = info.catalogId; + dbId = info.dbId; + tblId = info.tblId; colToPartitions = info.colToPartitions; partitionNames = info.partitionNames; colName = info.colName; @@ -112,18 +112,18 @@ public class AnalysisInfoBuilder { return this; } - public AnalysisInfoBuilder setCatalogName(String catalogName) { - this.catalogName = catalogName; + public AnalysisInfoBuilder setCatalogId(long catalogId) { + this.catalogId = catalogId; return this; } - public AnalysisInfoBuilder setDbName(String dbName) { - this.dbName = dbName; + public AnalysisInfoBuilder setDBId(long dbId) { + this.dbId = dbId; return this; } - public AnalysisInfoBuilder setTblName(String tblName) { - this.tblName = tblName; + public AnalysisInfoBuilder setTblId(long tblId) { + this.tblId = tblId; return this; } @@ -246,7 +246,7 @@ public class AnalysisInfoBuilder { } public AnalysisInfo build() { - return new AnalysisInfo(jobId, taskId, taskIds, catalogName, dbName, tblName, colToPartitions, partitionNames, + return new AnalysisInfo(jobId, taskId, taskIds, catalogId, dbId, tblId, colToPartitions, partitionNames, colName, indexId, jobType, analysisMode, analysisMethod, analysisType, samplePercent, sampleRows, maxBucketNum, periodTimeInMs, message, lastExecTimeInMs, timeCostInMs, state, scheduleType, externalTableLevelTask, partitionOnly, samplingPartition, isAllPartition, partitionCount, @@ -258,9 +258,9 @@ public class AnalysisInfoBuilder { .setJobId(jobId) .setTaskId(taskId) .setTaskIds(taskIds) - .setCatalogName(catalogName) - .setDbName(dbName) - .setTblName(tblName) + .setCatalogId(catalogId) + .setDBId(dbId) + .setTblId(tblId) .setColToPartitions(colToPartitions) .setColName(colName) .setIndexId(indexId) diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java index 9fa576f85cd..0d8308c4ebd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java @@ -44,6 +44,7 @@ import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; import org.apache.doris.common.util.Daemon; import org.apache.doris.common.util.Util; +import org.apache.doris.datasource.CatalogIf; import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.persist.AnalyzeDeletionLog; import org.apache.doris.persist.TableStatsDeletionLog; @@ -56,6 +57,7 @@ import org.apache.doris.statistics.AnalysisInfo.AnalysisMode; import org.apache.doris.statistics.AnalysisInfo.AnalysisType; import org.apache.doris.statistics.AnalysisInfo.JobType; import org.apache.doris.statistics.AnalysisInfo.ScheduleType; +import org.apache.doris.statistics.util.DBObjects; import org.apache.doris.statistics.util.SimpleQueue; import org.apache.doris.statistics.util.StatisticsUtil; @@ -84,6 +86,7 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Map.Entry; +import java.util.Optional; import java.util.Set; import java.util.StringJoiner; import java.util.TreeMap; @@ -365,7 +368,7 @@ public class AnalysisManager extends Daemon implements Writable { Map<Long, BaseAnalysisTask> analysisTaskInfos = new HashMap<>(); createTaskForEachColumns(jobInfo, analysisTaskInfos, isSync); if (!jobInfo.partitionOnly && stmt.isAllColumns() - && StatisticsUtil.isExternalTable(jobInfo.catalogName, jobInfo.dbName, jobInfo.tblName)) { + && StatisticsUtil.isExternalTable(jobInfo.catalogId, jobInfo.dbId, jobInfo.tblId)) { createTableLevelTaskForExternalTable(jobInfo, analysisTaskInfos, isSync); } if (isSync) { @@ -398,9 +401,16 @@ public class AnalysisManager extends Daemon implements Writable { } List<String> row = new ArrayList<>(); row.add(String.valueOf(analysisInfo.jobId)); - row.add(analysisInfo.catalogName); - row.add(analysisInfo.dbName); - row.add(analysisInfo.tblName); + CatalogIf<? extends DatabaseIf<? extends TableIf>> c = StatisticsUtil.findCatalog(analysisInfo.catalogId); + row.add(c.getName()); + Optional<? extends DatabaseIf<? extends TableIf>> databaseIf = c.getDb(analysisInfo.dbId); + row.add(databaseIf.isPresent() ? databaseIf.get().getFullName() : "DB may get deleted"); + if (databaseIf.isPresent()) { + Optional<? extends TableIf> table = databaseIf.get().getTable(analysisInfo.tblId); + row.add(table.isPresent() ? table.get().getName() : "Table may get deleted"); + } else { + row.add("DB not exists anymore"); + } row.add(analysisInfo.colName); resultRows.add(row); } @@ -487,11 +497,6 @@ public class AnalysisManager extends Daemon implements Writable { public AnalysisInfo buildAnalysisJobInfo(AnalyzeTblStmt stmt) throws DdlException { AnalysisInfoBuilder infoBuilder = new AnalysisInfoBuilder(); long jobId = Env.getCurrentEnv().getNextId(); - String catalogName = stmt.getCatalogName(); - String db = stmt.getDBName(); - TableName tbl = stmt.getTblName(); - StatisticsUtil.convertTableNameToObjects(tbl); - String tblName = tbl.getTbl(); TableIf table = stmt.getTable(); Set<String> columnNames = stmt.getColumnNames(); Set<String> partitionNames = stmt.getPartitionNames(); @@ -508,9 +513,9 @@ public class AnalysisManager extends Daemon implements Writable { CronExpression cronExpression = stmt.getCron(); infoBuilder.setJobId(jobId); - infoBuilder.setCatalogName(catalogName); - infoBuilder.setDbName(db); - infoBuilder.setTblName(tblName); + infoBuilder.setCatalogId(stmt.getCatalogId()); + infoBuilder.setDBId(stmt.getDbId()); + infoBuilder.setTblId(stmt.getTable().getId()); // TODO: Refactor later, DON'T MODIFY IT RIGHT NOW StringJoiner stringJoiner = new StringJoiner(",", "[", "]"); for (String colName : columnNames) { @@ -638,8 +643,8 @@ public class AnalysisManager extends Daemon implements Writable { @VisibleForTesting public void updateTableStats(AnalysisInfo jobInfo) { - TableIf tbl = StatisticsUtil.findTable(jobInfo.catalogName, - jobInfo.dbName, jobInfo.tblName); + TableIf tbl = StatisticsUtil.findTable(jobInfo.catalogId, + jobInfo.dbId, jobInfo.tblId); // External Table update table stats after table level task finished. if (tbl instanceof ExternalTable) { return; @@ -666,11 +671,15 @@ public class AnalysisManager extends Daemon implements Writable { protected List<AnalysisInfo> findShowAnalyzeResult(Collection<AnalysisInfo> analysisInfos, ShowAnalyzeStmt stmt) { String state = stmt.getStateValue(); TableName tblName = stmt.getDbTableName(); + TableIf tbl = null; + if (tblName != null) { + tbl = StatisticsUtil.findTable(tblName.getCtl(), tblName.getDb(), tblName.getTbl()); + } + long tblId = tbl == null ? -1 : tbl.getId(); return analysisInfos.stream() .filter(a -> stmt.getJobId() == 0 || a.jobId == stmt.getJobId()) .filter(a -> state == null || a.state.equals(AnalysisState.valueOf(state))) - .filter(a -> tblName == null || a.catalogName.equals(tblName.getCtl()) - && a.dbName.equals(tblName.getDb()) && a.tblName.equals(tblName.getTbl())) + .filter(a -> tblName == null || a.tblId == tblId) .sorted(Comparator.comparingLong(a -> a.jobId)) .collect(Collectors.toList()); } @@ -777,8 +786,11 @@ public class AnalysisManager extends Daemon implements Writable { } private void checkPriv(AnalysisInfo analysisInfo) { + DBObjects dbObjects = StatisticsUtil.convertIdToObjects(analysisInfo.catalogId, + analysisInfo.dbId, analysisInfo.tblId); if (!Env.getCurrentEnv().getAccessManager() - .checkTblPriv(ConnectContext.get(), analysisInfo.dbName, analysisInfo.tblName, PrivPredicate.SELECT)) { + .checkTblPriv(ConnectContext.get(), dbObjects.catalog.getName(), dbObjects.db.getFullName(), + dbObjects.table.getName(), PrivPredicate.SELECT)) { throw new RuntimeException("You need at least SELECT PRIV to corresponding table to kill this analyze" + " job"); } @@ -793,8 +805,8 @@ public class AnalysisManager extends Daemon implements Writable { private BaseAnalysisTask createTask(AnalysisInfo analysisInfo) throws DdlException { try { - TableIf table = StatisticsUtil.findTable(analysisInfo.catalogName, - analysisInfo.dbName, analysisInfo.tblName); + TableIf table = StatisticsUtil.findTable(analysisInfo.catalogId, + analysisInfo.dbId, analysisInfo.tblId); return table.createAnalysisTask(analysisInfo); } catch (Throwable t) { LOG.warn("Failed to find table", t); diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java index af85b528247..04b775bcd63 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java @@ -30,6 +30,7 @@ import org.apache.doris.qe.QueryState.MysqlStateType; import org.apache.doris.qe.StmtExecutor; import org.apache.doris.statistics.AnalysisInfo.AnalysisMethod; import org.apache.doris.statistics.AnalysisInfo.AnalysisType; +import org.apache.doris.statistics.util.DBObjects; import org.apache.doris.statistics.util.StatisticsUtil; import com.google.common.annotations.VisibleForTesting; @@ -94,9 +95,9 @@ public abstract class BaseAnalysisTask { protected AnalysisInfo info; - protected CatalogIf catalog; + protected CatalogIf<? extends DatabaseIf<? extends TableIf>> catalog; - protected DatabaseIf db; + protected DatabaseIf<? extends TableIf> db; protected TableIf tbl; @@ -119,25 +120,11 @@ public abstract class BaseAnalysisTask { } protected void init(AnalysisInfo info) { - catalog = Env.getCurrentEnv().getCatalogMgr().getCatalog(info.catalogName); - if (catalog == null) { - Env.getCurrentEnv().getAnalysisManager().updateTaskStatus(info, AnalysisState.FAILED, - String.format("Catalog with name: %s not exists", info.dbName), System.currentTimeMillis()); - return; - } - db = (DatabaseIf) catalog.getDb(info.dbName).orElse(null); - if (db == null) { - Env.getCurrentEnv().getAnalysisManager().updateTaskStatus(info, AnalysisState.FAILED, - String.format("DB with name %s not exists", info.dbName), System.currentTimeMillis()); - return; - } - tbl = (TableIf) db.getTable(info.tblName).orElse(null); - if (tbl == null) { - Env.getCurrentEnv().getAnalysisManager().updateTaskStatus( - info, AnalysisState.FAILED, - String.format("Table with name %s not exists", info.tblName), System.currentTimeMillis()); - } tableSample = getTableSample(); + DBObjects dbObjects = StatisticsUtil.convertIdToObjects(info.catalogId, info.dbId, info.tblId); + catalog = dbObjects.catalog; + db = dbObjects.db; + tbl = dbObjects.table; // External Table level task doesn't contain a column. Don't need to do the column related analyze. if (info.externalTableLevelTask) { return; @@ -146,7 +133,7 @@ public abstract class BaseAnalysisTask { || info.analysisType.equals(AnalysisType.HISTOGRAM))) { col = tbl.getColumn(info.colName); if (col == null) { - throw new RuntimeException(String.format("Column with name %s not exists", info.tblName)); + throw new RuntimeException(String.format("Column with name %s not exists", tbl.getName())); } Preconditions.checkArgument(!StatisticsUtil.isUnsupportedType(col.getType()), String.format("Column with type %s is not supported", col.getType().toString())); @@ -261,7 +248,7 @@ public abstract class BaseAnalysisTask { QueryState queryState = stmtExecutor.getContext().getState(); if (queryState.getStateType().equals(MysqlStateType.ERR)) { throw new RuntimeException(String.format("Failed to analyze %s.%s.%s, error: %s sql: %s", - info.catalogName, info.dbName, info.colName, stmtExecutor.getOriginStmt().toString(), + catalog.getName(), db.getFullName(), info.colName, stmtExecutor.getOriginStmt().toString(), queryState.getErrorMessage())); } } finally { diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/HMSAnalysisTask.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/HMSAnalysisTask.java index 076479aae8f..7ef87ac9401 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/HMSAnalysisTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/HMSAnalysisTask.java @@ -224,7 +224,7 @@ public class HMSAnalysisTask extends BaseAnalysisTask { QueryState queryState = r.connectContext.getState(); if (queryState.getStateType().equals(QueryState.MysqlStateType.ERR)) { throw new RuntimeException(String.format("Failed to analyze %s.%s.%s, error: %s sql: %s", - info.catalogName, info.dbName, info.colName, partitionCollectSQL, + catalog.getName(), db.getFullName(), info.colName, partitionCollectSQL, queryState.getErrorMessage())); } } @@ -254,11 +254,11 @@ public class HMSAnalysisTask extends BaseAnalysisTask { QueryState queryState = r.connectContext.getState(); if (queryState.getStateType().equals(QueryState.MysqlStateType.ERR)) { LOG.warn(String.format("Failed to analyze %s.%s.%s, sql: [%s], error: [%s]", - info.catalogName, info.dbName, info.colName, sql, queryState.getErrorMessage())); + catalog.getName(), db.getFullName(), info.colName, sql, queryState.getErrorMessage())); throw new RuntimeException(queryState.getErrorMessage()); } LOG.debug(String.format("Analyze %s.%s.%s done. SQL: [%s]. Cost %d ms.", - info.catalogName, info.dbName, info.colName, sql, (System.currentTimeMillis() - startTime))); + catalog.getName(), db.getFullName(), info.colName, sql, (System.currentTimeMillis() - startTime))); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/HistogramTask.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/HistogramTask.java index 11274ec79e6..3e02c47497b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/HistogramTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/HistogramTask.java @@ -67,8 +67,8 @@ public class HistogramTask extends BaseAnalysisTask { params.put("tblId", String.valueOf(tbl.getId())); params.put("idxId", String.valueOf(info.indexId)); params.put("colId", String.valueOf(info.colName)); - params.put("dbName", info.dbName); - params.put("tblName", String.valueOf(info.tblName)); + params.put("dbName", db.getFullName()); + params.put("tblName", tbl.getName()); params.put("colName", String.valueOf(info.colName)); params.put("sampleRate", getSampleRateFunction()); params.put("maxBucketNum", String.valueOf(info.maxBucketNum)); diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/JdbcAnalysisTask.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/JdbcAnalysisTask.java index 58be1510b44..0c148b5ad8d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/JdbcAnalysisTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/JdbcAnalysisTask.java @@ -130,11 +130,11 @@ public class JdbcAnalysisTask extends BaseAnalysisTask { QueryState queryState = r.connectContext.getState(); if (queryState.getStateType().equals(QueryState.MysqlStateType.ERR)) { LOG.warn(String.format("Failed to analyze %s.%s.%s, sql: [%s], error: [%s]", - info.catalogName, info.dbName, info.colName, sql, queryState.getErrorMessage())); + catalog.getName(), db.getFullName(), info.colName, sql, queryState.getErrorMessage())); throw new RuntimeException(queryState.getErrorMessage()); } LOG.debug(String.format("Analyze %s.%s.%s done. SQL: [%s]. Cost %d ms.", - info.catalogName, info.dbName, info.colName, sql, (System.currentTimeMillis() - startTime))); + catalog.getName(), db.getFullName(), info.colName, sql, (System.currentTimeMillis() - startTime))); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/MVAnalysisTask.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/MVAnalysisTask.java index 89d4313cf2f..6a43c5092fa 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/MVAnalysisTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/MVAnalysisTask.java @@ -114,9 +114,9 @@ public class MVAnalysisTask extends BaseAnalysisTask { String.valueOf(olapTable.getPartition(partName).getId()); params.put("partId", partId); params.put("dataSizeFunction", getDataSizeFunction(column)); - params.put("dbName", info.dbName); + params.put("dbName", db.getFullName()); params.put("colName", colName); - params.put("tblName", String.valueOf(info.tblName)); + params.put("tblName", tbl.getName()); params.put("sql", sql); StatisticsUtil.execUpdate(ANALYZE_MV_PART, params); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java index 868bd1a8882..d14cbc0f0a1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java @@ -137,9 +137,9 @@ public class OlapAnalysisTask extends BaseAnalysisTask { params.put("idxId", String.valueOf(info.indexId)); params.put("colId", String.valueOf(info.colName)); params.put("dataSizeFunction", getDataSizeFunction(col)); - params.put("dbName", info.dbName); - params.put("colName", String.valueOf(info.colName)); - params.put("tblName", String.valueOf(info.tblName)); + params.put("dbName", db.getFullName()); + params.put("colName", info.colName); + params.put("tblName", tbl.getName()); params.put("scaleFactor", String.valueOf(scaleFactor)); params.put("tablets", tabletStr.isEmpty() ? "" : String.format("TABLET(%s)", tabletStr)); StringSubstitutor stringSubstitutor = new StringSubstitutor(params); @@ -207,9 +207,9 @@ public class OlapAnalysisTask extends BaseAnalysisTask { params.put("idxId", String.valueOf(info.indexId)); params.put("colId", String.valueOf(info.colName)); params.put("dataSizeFunction", getDataSizeFunction(col)); - params.put("dbName", info.dbName); + params.put("dbName", db.getFullName()); params.put("colName", String.valueOf(info.colName)); - params.put("tblName", String.valueOf(info.tblName)); + params.put("tblName", String.valueOf(tbl.getName())); List<String> partitionAnalysisSQLs = new ArrayList<>(); try { tbl.readLock(); @@ -249,7 +249,7 @@ public class OlapAnalysisTask extends BaseAnalysisTask { QueryState queryState = r.connectContext.getState(); if (queryState.getStateType().equals(MysqlStateType.ERR)) { throw new RuntimeException(String.format("Failed to analyze %s.%s.%s, error: %s sql: %s", - info.catalogName, info.dbName, info.colName, partitionCollectSQL, + catalog.getName(), db.getFullName(), info.colName, partitionCollectSQL, queryState.getErrorMessage())); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java index fe535b0fb4a..20ebc5848d0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java @@ -17,7 +17,6 @@ package org.apache.doris.statistics; -import org.apache.doris.analysis.TableName; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.DatabaseIf; import org.apache.doris.catalog.Env; @@ -32,7 +31,6 @@ import org.apache.doris.statistics.AnalysisInfo.JobType; import org.apache.doris.statistics.AnalysisInfo.ScheduleType; import org.apache.doris.statistics.util.StatisticsUtil; -import com.google.common.collect.Maps; import org.apache.hudi.common.util.VisibleForTesting; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -126,13 +124,11 @@ public class StatisticsAutoCollector extends StatisticsCollector { List<AnalysisInfo> analysisInfos, TableIf table) { AnalysisMethod analysisMethod = table.getDataSize(true) > Config.huge_table_lower_bound_size_in_bytes ? AnalysisMethod.SAMPLE : AnalysisMethod.FULL; - TableName tableName = new TableName(db.getCatalog().getName(), db.getFullName(), - table.getName()); AnalysisInfo jobInfo = new AnalysisInfoBuilder() .setJobId(Env.getCurrentEnv().getNextId()) - .setCatalogName(db.getCatalog().getName()) - .setDbName(db.getFullName()) - .setTblName(tableName.getTbl()) + .setCatalogId(db.getCatalog().getId()) + .setDBId(db.getId()) + .setTblId(table.getId()) .setColName( table.getBaseSchema().stream().filter(c -> !StatisticsUtil.isUnsupportedType(c.getType())) .map( @@ -153,7 +149,7 @@ public class StatisticsAutoCollector extends StatisticsCollector { @VisibleForTesting protected AnalysisInfo getReAnalyzeRequiredPart(AnalysisInfo jobInfo) { TableIf table = StatisticsUtil - .findTable(jobInfo.catalogName, jobInfo.dbName, jobInfo.tblName); + .findTable(jobInfo.catalogId, jobInfo.dbId, jobInfo.tblId); AnalysisManager analysisManager = Env.getServingEnv().getAnalysisManager(); TableStatsMeta tblStats = analysisManager.findTableStatsStatus(table.getId()); @@ -170,27 +166,4 @@ public class StatisticsAutoCollector extends StatisticsCollector { return new AnalysisInfoBuilder(jobInfo).setColToPartitions(needRunPartitions).build(); } - @VisibleForTesting - protected AnalysisInfo getAnalysisJobInfo(AnalysisInfo jobInfo, TableIf table, - Set<String> needRunPartitions) { - Map<String, Set<String>> newColToPartitions = Maps.newHashMap(); - Map<String, Set<String>> colToPartitions = jobInfo.colToPartitions; - if (colToPartitions == null) { - for (Column c : table.getColumns()) { - if (StatisticsUtil.isUnsupportedType(c.getType())) { - continue; - } - newColToPartitions.put(c.getName(), needRunPartitions); - } - } else { - colToPartitions.keySet().forEach(colName -> { - Column column = table.getColumn(colName); - if (column != null) { - newColToPartitions.put(colName, needRunPartitions); - } - }); - } - return new AnalysisInfoBuilder(jobInfo) - .setColToPartitions(newColToPartitions).build(); - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCleaner.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCleaner.java index 6521a8b4a59..88fa098e57b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCleaner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCleaner.java @@ -55,7 +55,7 @@ public class StatisticsCleaner extends MasterDaemon { private OlapTable colStatsTbl; private OlapTable histStatsTbl; - private Map<Long, CatalogIf> idToCatalog; + private Map<Long, CatalogIf<? extends DatabaseIf<? extends TableIf>>> idToCatalog; private Map<Long, DatabaseIf> idToDb; private Map<Long, TableIf> idToTbl; diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCollector.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCollector.java index 2d5c4816835..c2f1db6bc4a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCollector.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCollector.java @@ -76,7 +76,7 @@ public abstract class StatisticsCollector extends MasterDaemon { Map<Long, BaseAnalysisTask> analysisTaskInfos = new HashMap<>(); AnalysisManager analysisManager = Env.getCurrentEnv().getAnalysisManager(); analysisManager.createTaskForEachColumns(jobInfo, analysisTaskInfos, false); - if (StatisticsUtil.isExternalTable(jobInfo.catalogName, jobInfo.dbName, jobInfo.tblName)) { + if (StatisticsUtil.isExternalTable(jobInfo.catalogId, jobInfo.dbId, jobInfo.tblId)) { analysisManager.createTableLevelTaskForExternalTable(jobInfo, analysisTaskInfos, false); } Env.getCurrentEnv().getAnalysisManager().registerSysJob(jobInfo, analysisTaskInfos); diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/DBObjects.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/DBObjects.java index 3e68d0e0274..3e3a258030c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/DBObjects.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/DBObjects.java @@ -23,14 +23,14 @@ import org.apache.doris.datasource.CatalogIf; public class DBObjects { - public final CatalogIf<DatabaseIf> catalog; + public final CatalogIf<? extends DatabaseIf<? extends TableIf>> catalog; - public final DatabaseIf<TableIf> db; + public final DatabaseIf<? extends TableIf> db; public final TableIf table; - public DBObjects(CatalogIf<DatabaseIf> catalog, - DatabaseIf<TableIf> db, TableIf table) { + public DBObjects(CatalogIf<? extends DatabaseIf<? extends TableIf>> catalog, + DatabaseIf<? extends TableIf> db, TableIf table) { this.catalog = catalog; this.db = db; this.table = table; diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java index f482c812879..985dd20b635 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java @@ -301,13 +301,13 @@ public class StatisticsUtil { } - public static DBObjects convertTableNameToObjects(TableName tableName) { - CatalogIf<DatabaseIf> catalogIf = Env.getCurrentEnv().getCatalogMgr().getCatalog(tableName.getCtl()); + CatalogIf<? extends DatabaseIf<? extends TableIf>> catalogIf = + Env.getCurrentEnv().getCatalogMgr().getCatalog(tableName.getCtl()); if (catalogIf == null) { throw new IllegalStateException(String.format("Catalog:%s doesn't exist", tableName.getCtl())); } - DatabaseIf<TableIf> databaseIf = catalogIf.getDbNullable(tableName.getDb()); + DatabaseIf<? extends TableIf> databaseIf = catalogIf.getDbNullable(tableName.getDb()); if (databaseIf == null) { throw new IllegalStateException(String.format("DB:%s doesn't exist", tableName.getDb())); } @@ -318,12 +318,17 @@ public class StatisticsUtil { return new DBObjects(catalogIf, databaseIf, tableIf); } + public static DBObjects convertIdToObjects(long catalogId, long dbId, long tblId) { + return new DBObjects(findCatalog(catalogId), findDatabase(catalogId, dbId), findTable(catalogId, dbId, tblId)); + } + public static Column findColumn(long catalogId, long dbId, long tblId, long idxId, String columnName) { - CatalogIf<DatabaseIf<TableIf>> catalogIf = Env.getCurrentEnv().getCatalogMgr().getCatalog(catalogId); + CatalogIf<? extends DatabaseIf<? extends TableIf>> catalogIf = + Env.getCurrentEnv().getCatalogMgr().getCatalog(catalogId); if (catalogIf == null) { return null; } - DatabaseIf<TableIf> db = catalogIf.getDb(dbId).orElse(null); + DatabaseIf<? extends TableIf> db = catalogIf.getDb(dbId).orElse(null); if (db == null) { return null; } @@ -361,6 +366,16 @@ public class StatisticsUtil { } } + public static TableIf findTable(long catalogId, long dbId, long tblId) { + try { + DatabaseIf<? extends TableIf> db = findDatabase(catalogId, dbId); + return db.getTableOrException(tblId, + t -> new RuntimeException("Table: " + t + " not exists")); + } catch (Throwable t) { + throw new RuntimeException("Table: `" + catalogId + "." + dbId + "." + tblId + "` not exists"); + } + } + /** * Throw RuntimeException if database not exists. */ @@ -371,6 +386,12 @@ public class StatisticsUtil { d -> new RuntimeException("DB: " + d + " not exists")); } + public static DatabaseIf<? extends TableIf> findDatabase(long catalogId, long dbId) { + CatalogIf<? extends DatabaseIf<? extends TableIf>> catalog = findCatalog(catalogId); + return catalog.getDbOrException(dbId, + d -> new RuntimeException("DB: " + d + " not exists")); + } + /** * Throw RuntimeException if catalog not exists. */ @@ -380,6 +401,11 @@ public class StatisticsUtil { .getCatalogOrException(catalogName, c -> new RuntimeException("Catalog: " + c + " not exists")); } + public static CatalogIf<? extends DatabaseIf<? extends TableIf>> findCatalog(long catalogId) { + return Env.getCurrentEnv().getCatalogMgr().getCatalogOrException(catalogId, + c -> new RuntimeException("Catalog: " + c + " not exists")); + } + public static boolean isNullOrEmpty(String str) { return Optional.ofNullable(str) .map(String::trim) @@ -763,6 +789,17 @@ public class StatisticsUtil { return table instanceof ExternalTable; } + public static boolean isExternalTable(long catalogId, long dbId, long tblId) { + TableIf table; + try { + table = findTable(catalogId, dbId, tblId); + } catch (Throwable e) { + LOG.warn(e.getMessage()); + return false; + } + return table instanceof ExternalTable; + } + public static boolean inAnalyzeTime(LocalTime now) { try { Pair<LocalTime, LocalTime> range = findRangeFromGlobalSessionVar(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/statistics/AnalysisJobTest.java b/fe/fe-core/src/test/java/org/apache/doris/statistics/AnalysisJobTest.java index 9624c201498..f01485f642f 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/statistics/AnalysisJobTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/statistics/AnalysisJobTest.java @@ -17,8 +17,13 @@ package org.apache.doris.statistics; +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Database; import org.apache.doris.catalog.InternalSchemaInitializer; +import org.apache.doris.catalog.OlapTable; +import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.common.FeConstants; +import org.apache.doris.datasource.InternalCatalog; import org.apache.doris.qe.AutoCloseConnectContext; import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.StmtExecutor; @@ -26,6 +31,7 @@ import org.apache.doris.statistics.AnalysisInfo.AnalysisMethod; import org.apache.doris.statistics.AnalysisInfo.AnalysisMode; import org.apache.doris.statistics.AnalysisInfo.AnalysisType; import org.apache.doris.statistics.AnalysisInfo.JobType; +import org.apache.doris.statistics.util.DBObjects; import org.apache.doris.statistics.util.StatisticsUtil; import org.apache.doris.utframe.TestWithFeService; @@ -41,6 +47,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Set; public class AnalysisJobTest extends TestWithFeService { @@ -95,8 +102,18 @@ public class AnalysisJobTest extends TestWithFeService { } @Test - public void testJobExecution(@Mocked StmtExecutor stmtExecutor) + public void testJobExecution(@Mocked StmtExecutor stmtExecutor, @Mocked InternalCatalog catalog, @Mocked + Database database, + @Mocked OlapTable olapTable) throws Exception { + new MockUp<OlapTable>() { + + @Mock + public Column getColumn(String name) { + return new Column("col1", PrimitiveType.INT); + } + }; + new MockUp<StatisticsUtil>() { @Mock @@ -107,6 +124,11 @@ public class AnalysisJobTest extends TestWithFeService { @Mock public void execUpdate(String sql) throws Exception { } + + @Mock + public DBObjects convertIdToObjects(long catalogId, long dbId, long tblId) { + return new DBObjects(catalog, database, olapTable); + } }; new MockUp<StatisticsCache>() { @@ -126,10 +148,18 @@ public class AnalysisJobTest extends TestWithFeService { return new ArrayList<>(); } }; + + new MockUp<OlapAnalysisTask>() { + + @Mock + public void execSQLs(List<String> partitionAnalysisSQLs, Map<String, String> params) throws Exception {} + }; HashMap<String, Set<String>> colToPartitions = Maps.newHashMap(); colToPartitions.put("col1", Collections.singleton("t1")); AnalysisInfo analysisJobInfo = new AnalysisInfoBuilder().setJobId(0).setTaskId(0) - .setCatalogName("internal").setDbName("default_cluster:analysis_job_test").setTblName("t1") + .setCatalogId(0) + .setDBId(0) + .setTblId(0) .setColName("col1").setJobType(JobType.MANUAL) .setAnalysisMode(AnalysisMode.FULL) .setAnalysisMethod(AnalysisMethod.FULL) diff --git a/fe/fe-core/src/test/java/org/apache/doris/statistics/AnalysisTaskExecutorTest.java b/fe/fe-core/src/test/java/org/apache/doris/statistics/AnalysisTaskExecutorTest.java index 196ac8ad9a0..19d7798041a 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/statistics/AnalysisTaskExecutorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/statistics/AnalysisTaskExecutorTest.java @@ -17,24 +17,32 @@ package org.apache.doris.statistics; +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Database; import org.apache.doris.catalog.InternalSchemaInitializer; +import org.apache.doris.catalog.OlapTable; +import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.common.jmockit.Deencapsulation; +import org.apache.doris.datasource.InternalCatalog; import org.apache.doris.qe.StmtExecutor; import org.apache.doris.statistics.AnalysisInfo.AnalysisMethod; import org.apache.doris.statistics.AnalysisInfo.AnalysisMode; import org.apache.doris.statistics.AnalysisInfo.AnalysisType; import org.apache.doris.statistics.AnalysisInfo.JobType; +import org.apache.doris.statistics.util.DBObjects; +import org.apache.doris.statistics.util.StatisticsUtil; import org.apache.doris.utframe.TestWithFeService; import com.google.common.collect.Maps; -import mockit.Expectations; import mockit.Mock; import mockit.MockUp; +import mockit.Mocked; import org.junit.jupiter.api.Test; import java.util.Collections; import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.concurrent.BlockingQueue; @@ -58,9 +66,26 @@ public class AnalysisTaskExecutorTest extends TestWithFeService { } @Test - public void testExpiredJobCancellation() throws Exception { + public void testExpiredJobCancellation(@Mocked InternalCatalog catalog, @Mocked Database database, + @Mocked OlapTable olapTable) throws Exception { + new MockUp<StatisticsUtil>() { + + @Mock + public DBObjects convertIdToObjects(long catalogId, long dbId, long tblId) { + return new DBObjects(catalog, database, olapTable); + } + }; + new MockUp<OlapTable>() { + + @Mock + public Column getColumn(String name) { + return new Column("col1", PrimitiveType.INT); + } + }; AnalysisInfo analysisJobInfo = new AnalysisInfoBuilder().setJobId(0).setTaskId(0) - .setCatalogName("internal").setDbName("default_cluster:analysis_job_test").setTblName("t1") + .setCatalogId(0) + .setDBId(0) + .setTblId(0) .setColName("col1").setJobType(JobType.MANUAL) .setAnalysisMode(AnalysisMode.FULL) .setAnalysisMethod(AnalysisMethod.FULL) @@ -77,8 +102,22 @@ public class AnalysisTaskExecutorTest extends TestWithFeService { } @Test - public void testTaskExecution() throws Exception { + public void testTaskExecution(@Mocked InternalCatalog catalog, @Mocked Database database, + @Mocked OlapTable olapTable) throws Exception { + new MockUp<StatisticsUtil>() { + + @Mock + public DBObjects convertIdToObjects(long catalogId, long dbId, long tblId) { + return new DBObjects(catalog, database, olapTable); + } + }; + new MockUp<OlapTable>() { + @Mock + public Column getColumn(String name) { + return new Column("col1", PrimitiveType.INT); + } + }; new MockUp<StmtExecutor>() { @Mock public List<ResultRow> executeInternalQuery() { @@ -88,7 +127,7 @@ public class AnalysisTaskExecutorTest extends TestWithFeService { new MockUp<OlapAnalysisTask>() { @Mock - public void execSQLs(List<String> sqls) throws Exception { + public void execSQLs(List<String> partitionAnalysisSQLs, Map<String, String> params) throws Exception { } @Mock @@ -108,7 +147,7 @@ public class AnalysisTaskExecutorTest extends TestWithFeService { HashMap<String, Set<String>> colToPartitions = Maps.newHashMap(); colToPartitions.put("col1", Collections.singleton("t1")); AnalysisInfo analysisInfo = new AnalysisInfoBuilder().setJobId(0).setTaskId(0) - .setCatalogName("internal").setDbName("default_cluster:analysis_job_test").setTblName("t1") + .setCatalogId(0).setDBId(0).setTblId(0) .setColName("col1").setJobType(JobType.MANUAL) .setAnalysisMode(AnalysisMode.FULL) .setAnalysisMethod(AnalysisMethod.FULL) @@ -122,11 +161,7 @@ public class AnalysisTaskExecutorTest extends TestWithFeService { @Mock public void updateTaskStatus(AnalysisInfo info, AnalysisState jobState, String message, long time) {} }; - new Expectations() { - { - task.doExecute(); - } - }; + Deencapsulation.invoke(analysisTaskExecutor, "submitTask", task); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/statistics/HistogramTaskTest.java b/fe/fe-core/src/test/java/org/apache/doris/statistics/HistogramTaskTest.java index 0660c994a12..b856d374bef 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/statistics/HistogramTaskTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/statistics/HistogramTaskTest.java @@ -17,19 +17,27 @@ package org.apache.doris.statistics; +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Database; import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.OlapTable; +import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.common.FeConstants; import org.apache.doris.common.jmockit.Deencapsulation; +import org.apache.doris.datasource.InternalCatalog; import org.apache.doris.qe.StmtExecutor; import org.apache.doris.statistics.AnalysisInfo.AnalysisMethod; import org.apache.doris.statistics.AnalysisInfo.AnalysisMode; import org.apache.doris.statistics.AnalysisInfo.AnalysisType; import org.apache.doris.statistics.AnalysisInfo.JobType; +import org.apache.doris.statistics.util.DBObjects; +import org.apache.doris.statistics.util.StatisticsUtil; import org.apache.doris.system.SystemInfoService; import org.apache.doris.utframe.TestWithFeService; import mockit.Mock; import mockit.MockUp; +import mockit.Mocked; import org.junit.FixMethodOrder; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -84,18 +92,33 @@ public class HistogramTaskTest extends TestWithFeService { for (Entry<Long, BaseAnalysisTask> infoEntry : taskInfo.entrySet()) { BaseAnalysisTask task = infoEntry.getValue(); Assertions.assertEquals(AnalysisType.HISTOGRAM, task.info.analysisType); - Assertions.assertEquals("t1", task.info.tblName); Assertions.assertEquals("col1", task.info.colName); } } } @Test - public void test2TaskExecution() throws Exception { + public void test2TaskExecution(@Mocked InternalCatalog catalog, @Mocked Database database, + @Mocked OlapTable olapTable) throws Exception { + new MockUp<StatisticsUtil>() { + + @Mock + public DBObjects convertIdToObjects(long catalogId, long dbId, long tblId) { + return new DBObjects(catalog, database, olapTable); + } + }; + new MockUp<OlapTable>() { + + @Mock + public Column getColumn(String name) { + return new Column("col1", PrimitiveType.INT); + } + }; AnalysisTaskExecutor analysisTaskExecutor = new AnalysisTaskExecutor(1); AnalysisInfo analysisInfo = new AnalysisInfoBuilder() - .setJobId(0).setTaskId(0).setCatalogName("internal") - .setDbName(SystemInfoService.DEFAULT_CLUSTER + ":" + "histogram_task_test").setTblName("t1") + .setJobId(0).setTaskId(0).setCatalogId(0) + .setDBId(0) + .setTblId(0) .setColName("col1").setJobType(JobType.MANUAL) .setAnalysisMode(AnalysisMode.FULL) .setAnalysisMethod(AnalysisMethod.FULL) diff --git a/regression-test/suites/statistics/analyze_stats.groovy b/regression-test/suites/statistics/analyze_stats.groovy index b168230b2d5..04bd9c730d4 100644 --- a/regression-test/suites/statistics/analyze_stats.groovy +++ b/regression-test/suites/statistics/analyze_stats.groovy @@ -18,7 +18,7 @@ import java.util.stream.Collectors // under the License. suite("test_analyze") { - String db = "test_analyze" + String db = "regression_test_statistics" String tbl = "analyzetestlimited_duplicate_all" sql """ --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
