This is an automated email from the ASF dual-hosted git repository.
englefly pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 10792ca0f7 [fix](nereids) Mistaken stats when analyzing table
incrementally and partition number less than 512 #23507
10792ca0f7 is described below
commit 10792ca0f75167b34a496c4a5dd1803578f9c303
Author: AKIRA <[email protected]>
AuthorDate: Mon Aug 28 17:31:36 2023 +0800
[fix](nereids) Mistaken stats when analyzing table incrementally and
partition number less than 512 #23507
Fix bug that mistaken stats when analyzing table incrementally and
partition number less than 512
Fix bug that cron expression lost during analyzing
Mark system job as running after registered to AnalysisManager to avoid
submit same jobs if previous one take long time
---
.../java/org/apache/doris/qe/ShowExecutor.java | 7 +-
.../java/org/apache/doris/qe/StmtExecutor.java | 1 +
.../org/apache/doris/statistics/AnalysisInfo.java | 3 +
.../doris/statistics/AnalysisInfoBuilder.java | 1 +
.../apache/doris/statistics/AnalysisManager.java | 60 +++---
.../org/apache/doris/statistics/AnalysisState.java | 2 +
.../apache/doris/statistics/BaseAnalysisTask.java | 1 +
.../apache/doris/statistics/OlapAnalysisTask.java | 92 ++-------
.../doris/statistics/StatisticConstants.java | 5 +-
.../doris/statistics/StatisticsAutoAnalyzer.java | 9 +-
.../apache/doris/statistics/AnalysisJobTest.java | 1 +
.../doris/statistics/AnalysisManagerTest.java | 7 +-
.../doris/statistics/AnalysisTaskExecutorTest.java | 1 +
.../suites/statistics/analyze_stats.groovy | 213 ++++++++++++---------
14 files changed, 199 insertions(+), 204 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
index f505089cd7..5faf28b294 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
@@ -2670,7 +2670,12 @@ public class ShowExecutor {
LocalDateTime.ofInstant(Instant.ofEpochMilli(analysisInfo.lastExecTimeInMs),
ZoneId.systemDefault())));
row.add(analysisInfo.state.toString());
-
row.add(Env.getCurrentEnv().getAnalysisManager().getJobProgress(analysisInfo.jobId));
+ try {
+
row.add(Env.getCurrentEnv().getAnalysisManager().getJobProgress(analysisInfo.jobId));
+ } catch (Exception e) {
+ row.add("N/A");
+ LOG.warn("Failed to get progress for job: {}", analysisInfo,
e);
+ }
row.add(analysisInfo.scheduleType.toString());
resultRows.add(row);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
index 85235ec159..8f654dc292 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
@@ -2438,6 +2438,7 @@ public class StmtExecutor {
}
public List<ResultRow> executeInternalQuery() {
+ LOG.debug("INTERNAL QUERY: " + originStmt.toString());
try {
List<ResultRow> resultRows = new ArrayList<>();
try {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfo.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfo.java
index 73441db2e2..8f33480640 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfo.java
@@ -200,6 +200,9 @@ public class AnalysisInfo implements Writable {
this.partitionOnly = partitionOnly;
this.samplingPartition = samplingPartition;
this.cronExpression = cronExpression;
+ if (cronExpression != null) {
+ this.cronExprStr = cronExpression.getCronExpression();
+ }
}
@Override
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfoBuilder.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfoBuilder.java
index dc368fcdf1..081ee4554c 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfoBuilder.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfoBuilder.java
@@ -89,6 +89,7 @@ public class AnalysisInfoBuilder {
externalTableLevelTask = info.externalTableLevelTask;
partitionOnly = info.partitionOnly;
samplingPartition = info.samplingPartition;
+ cronExpression = info.cronExpression;
}
public AnalysisInfoBuilder setJobId(long jobId) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
index 5fbfb56829..bdd325e6d1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
@@ -347,7 +347,9 @@ public class AnalysisManager extends Daemon implements
Writable {
analysisJobIdToTaskMap.put(jobInfo.jobId, analysisTaskInfos);
// TODO: maybe we should update table stats only when all task
succeeded.
updateTableStats(jobInfo);
- analysisTaskInfos.values().forEach(taskExecutor::submitTask);
+ if (!jobInfo.scheduleType.equals(ScheduleType.PERIOD)) {
+ analysisTaskInfos.values().forEach(taskExecutor::submitTask);
+ }
return jobInfo;
}
@@ -464,7 +466,7 @@ public class AnalysisManager extends Daemon implements
Writable {
@VisibleForTesting
public AnalysisInfo buildAnalysisJobInfo(AnalyzeTblStmt stmt) throws
DdlException {
- AnalysisInfoBuilder info = new AnalysisInfoBuilder();
+ AnalysisInfoBuilder infoBuilder = new AnalysisInfoBuilder();
long jobId = Env.getCurrentEnv().getNextId();
String catalogName = stmt.getCatalogName();
String db = stmt.getDBName();
@@ -484,49 +486,49 @@ public class AnalysisManager extends Daemon implements
Writable {
ScheduleType scheduleType = stmt.getScheduleType();
CronExpression cronExpression = stmt.getCron();
- info.setJobId(jobId);
- info.setCatalogName(catalogName);
- info.setDbName(db);
- info.setTblName(tblName);
+ infoBuilder.setJobId(jobId);
+ infoBuilder.setCatalogName(catalogName);
+ infoBuilder.setDbName(db);
+ infoBuilder.setTblName(tblName);
StringJoiner stringJoiner = new StringJoiner(",", "[", "]");
for (String colName : columnNames) {
stringJoiner.add(colName);
}
- info.setColName(stringJoiner.toString());
- info.setPartitionNames(partitionNames);
- info.setPartitionOnly(partitionOnly);
- info.setSamplingPartition(isSamplingPartition);
- info.setJobType(JobType.MANUAL);
- info.setState(AnalysisState.PENDING);
- info.setLastExecTimeInMs(System.currentTimeMillis());
- info.setAnalysisType(analysisType);
- info.setAnalysisMode(analysisMode);
- info.setAnalysisMethod(analysisMethod);
- info.setScheduleType(scheduleType);
- info.setLastExecTimeInMs(0);
- info.setCronExpression(cronExpression);
+ infoBuilder.setColName(stringJoiner.toString());
+ infoBuilder.setPartitionNames(partitionNames);
+ infoBuilder.setPartitionOnly(partitionOnly);
+ infoBuilder.setSamplingPartition(isSamplingPartition);
+ infoBuilder.setJobType(JobType.MANUAL);
+ infoBuilder.setState(AnalysisState.PENDING);
+ infoBuilder.setLastExecTimeInMs(System.currentTimeMillis());
+ infoBuilder.setAnalysisType(analysisType);
+ infoBuilder.setAnalysisMode(analysisMode);
+ infoBuilder.setAnalysisMethod(analysisMethod);
+ infoBuilder.setScheduleType(scheduleType);
+ infoBuilder.setLastExecTimeInMs(0);
+ infoBuilder.setCronExpression(cronExpression);
if (analysisMethod == AnalysisMethod.SAMPLE) {
- info.setSamplePercent(samplePercent);
- info.setSampleRows(sampleRows);
+ infoBuilder.setSamplePercent(samplePercent);
+ infoBuilder.setSampleRows(sampleRows);
}
if (analysisType == AnalysisType.HISTOGRAM) {
int numBuckets = stmt.getNumBuckets();
int maxBucketNum = numBuckets > 0 ? numBuckets
: StatisticConstants.HISTOGRAM_MAX_BUCKET_NUM;
- info.setMaxBucketNum(maxBucketNum);
+ infoBuilder.setMaxBucketNum(maxBucketNum);
}
long periodTimeInMs = stmt.getPeriodTimeInMs();
- info.setPeriodTimeInMs(periodTimeInMs);
+ infoBuilder.setPeriodTimeInMs(periodTimeInMs);
Map<String, Set<String>> colToPartitions =
validateAndGetPartitions(table, columnNames,
partitionNames, analysisType, analysisMode);
- info.setColToPartitions(colToPartitions);
- info.setTaskIds(Lists.newArrayList());
+ infoBuilder.setColToPartitions(colToPartitions);
+ infoBuilder.setTaskIds(Lists.newArrayList());
- return info.build();
+ return infoBuilder.build();
}
@VisibleForTesting
@@ -820,7 +822,7 @@ public class AnalysisManager extends Daemon implements
Writable {
public List<AnalysisInfo> findPeriodicJobs() {
synchronized (analysisJobInfoMap) {
Predicate<AnalysisInfo> p = a -> {
- if (a.state.equals(AnalysisState.RUNNING) ||
a.state.equals(AnalysisState.PENDING)) {
+ if (a.state.equals(AnalysisState.RUNNING)) {
return false;
}
if (a.cronExpression == null) {
@@ -844,7 +846,8 @@ public class AnalysisManager extends Daemon implements
Writable {
public List<AnalysisInfo> findTasksByTaskIds(long jobId) {
AnalysisInfo jobInfo = analysisJobInfoMap.get(jobId);
if (jobInfo != null && jobInfo.taskIds != null) {
- return jobInfo.taskIds.stream().map(id ->
analysisTaskInfoMap.get(id)).collect(Collectors.toList());
+ return
jobInfo.taskIds.stream().map(analysisTaskInfoMap::get).filter(i -> i != null)
+ .collect(Collectors.toList());
}
return null;
}
@@ -944,6 +947,7 @@ public class AnalysisManager extends Daemon implements
Writable {
}
public void registerSysJob(AnalysisInfo jobInfo, Map<Long,
BaseAnalysisTask> taskInfos) {
+ jobInfo.state = AnalysisState.RUNNING;
systemJobInfoMap.put(jobInfo.jobId, jobInfo);
analysisJobIdToTaskMap.put(jobInfo.jobId, taskInfos);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisState.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisState.java
index bab8a462e8..3abc4c224f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisState.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisState.java
@@ -18,7 +18,9 @@
package org.apache.doris.statistics;
public enum AnalysisState {
+ // When analyze job/task created, but never run
PENDING,
+ // When analyze job/task is in running queue
RUNNING,
FINISHED,
FAILED;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java
index 235861119a..0323929081 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java
@@ -236,6 +236,7 @@ public abstract class BaseAnalysisTask {
if (killed) {
return;
}
+ LOG.debug("execute internal sql: {}", stmtExecutor.getOriginStmt());
stmtExecutor.execute();
QueryState queryState = stmtExecutor.getContext().getState();
if (queryState.getStateType().equals(MysqlStateType.ERR)) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java
index 257a104254..ef26d7349e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java
@@ -30,6 +30,7 @@ import com.google.common.collect.Lists;
import org.apache.commons.text.StringSubstitutor;
import java.util.ArrayList;
+import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -109,6 +110,8 @@ public class OlapAnalysisTask extends BaseAnalysisTask {
@VisibleForTesting
public void execSQLs(List<String> partitionAnalysisSQLs, Map<String,
String> params) throws Exception {
+ long startTime = System.currentTimeMillis();
+ LOG.debug("analyze task {} start at {}", info.toString(), new Date());
try (AutoCloseConnectContext r = StatisticsUtil.buildConnectContext())
{
List<List<String>> sqlGroups =
Lists.partition(partitionAnalysisSQLs, StatisticConstants.UNION_ALL_LIMIT);
for (List<String> group : sqlGroups) {
@@ -127,80 +130,25 @@ public class OlapAnalysisTask extends BaseAnalysisTask {
queryState.getErrorMessage()));
}
}
- if (buf.size() > 1) {
- for (List<ColStatsData> colStatsDataList : buf) {
- StringBuilder batchInsertSQL =
- new StringBuilder("INSERT INTO
__internal_schema.column_statistics VALUES ");
- StringJoiner sj = new StringJoiner(",");
- colStatsDataList.forEach(c -> sj.add(c.toSQL(true)));
- batchInsertSQL.append(sj.toString());
- stmtExecutor = new StmtExecutor(r.connectContext,
batchInsertSQL.toString());
- executeWithExceptionOnFail(stmtExecutor);
- }
- params.put("type", col.getType().toString());
- StringSubstitutor stringSubstitutor = new
StringSubstitutor(params);
- String sql =
stringSubstitutor.replace(ANALYZE_COLUMN_SQL_TEMPLATE);
- stmtExecutor = new StmtExecutor(r.connectContext, sql);
- executeWithExceptionOnFail(stmtExecutor);
- } else {
- List<ColStatsData> colStatsDataList = buf.get(0);
- String batchInsertSQLTemplate = "INSERT INTO
__internal_schema.column_statistics "
- + "SELECT id, catalog_id, db_id, tbl_id, idx_id,
col_id, part_id, row_count,"
- + "ndv, null_count, CAST(min AS string), CAST(max AS
string), data_size, update_time FROM ("
- + "SELECT CONCAT(${tblId}, '-', ${idxId}, '-',
'${colId}') AS id, "
- + " ${catalogId} AS catalog_id, "
- + " ${dbId} AS db_id, "
- + " ${tblId} AS tbl_id, "
- + " ${idxId} AS idx_id, "
- + " '${colId}' AS col_id, "
- + " NULL AS part_id, "
- + " SUM(count) AS row_count,"
- + " SUM(null_count) AS null_count, "
- + " MIN(CAST (min AS ${type})) AS min, "
- + " MAX(CAST (max AS ${type})) AS max, "
- + " SUM(data_size_in_bytes) AS data_size, "
- + " NOW() AS update_time"
- + " FROM (${partitionStatsView}) psv) t1, "
- + " (SELECT NDV(`${colName}`) AS ndv "
- + " FROM `${dbName}`.`${tblName}` ${sampleExpr})
t2 UNION ALL ${partitionStatsView}";
- StringJoiner sj = new StringJoiner(" UNION ALL ");
- String selectPartitionTemplate =
- "SELECT %s AS id,"
- + "%s AS catalog_id,"
- + "%s AS db_id,"
- + "%s AS tbl_id,"
- + "%s AS idx_id,"
- + "%s AS col_id,"
- + "%s AS part_id,"
- + "%s AS count,"
- + "%s AS ndv,"
- + "%s AS null_count,"
- + "%s as min,"
- + "%s as max,"
- + "%s as data_size_in_bytes,"
- + "%s AS update_time";
- colStatsDataList.forEach(c ->
sj.add(String.format(selectPartitionTemplate,
- StatisticsUtil.quote(c.statsId.id),
- c.statsId.catalogId,
- c.statsId.dbId,
- c.statsId.tblId,
- c.statsId.idxId,
- StatisticsUtil.quote(c.statsId.colId),
- c.statsId.partId,
- c.count,
- c.ndv,
- c.nullCount,
- c.minLit == null ? null :
StatisticsUtil.quote(StatisticsUtil.escapeSQL(c.minLit)),
- c.maxLit == null ? null :
StatisticsUtil.quote(StatisticsUtil.escapeSQL(c.maxLit)),
- c.dataSizeInBytes,
- StatisticsUtil.quote(c.updateTime))));
- params.put("partitionStatsView", sj.toString());
- params.put("type", col.getType().toString());
- StringSubstitutor stringSubstitutor = new
StringSubstitutor(params);
- String insertSQL =
stringSubstitutor.replace(batchInsertSQLTemplate);
- stmtExecutor = new StmtExecutor(r.connectContext, insertSQL);
+ for (List<ColStatsData> colStatsDataList : buf) {
+ StringBuilder batchInsertSQL =
+ new StringBuilder("INSERT INTO " +
StatisticConstants.FULL_QUALIFIED_STATS_TBL_NAME
+ + " VALUES ");
+ StringJoiner sj = new StringJoiner(",");
+ colStatsDataList.forEach(c -> sj.add(c.toSQL(true)));
+ batchInsertSQL.append(sj.toString());
+ stmtExecutor = new StmtExecutor(r.connectContext,
batchInsertSQL.toString());
executeWithExceptionOnFail(stmtExecutor);
}
+ params.put("type", col.getType().toString());
+ StringSubstitutor stringSubstitutor = new
StringSubstitutor(params);
+ String sql =
stringSubstitutor.replace(ANALYZE_COLUMN_SQL_TEMPLATE);
+ stmtExecutor = new StmtExecutor(r.connectContext, sql);
+ executeWithExceptionOnFail(stmtExecutor);
+ } finally {
+ LOG.debug("analyze task {} end. cost {}ms", info,
+ System.currentTimeMillis() - startTime);
}
+
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticConstants.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticConstants.java
index ce512c1952..f0f6f60277 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticConstants.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticConstants.java
@@ -28,15 +28,12 @@ import java.util.concurrent.TimeUnit;
public class StatisticConstants {
public static final String STATISTIC_TBL_NAME = "column_statistics";
-
public static final String HISTOGRAM_TBL_NAME = "histogram_statistics";
public static final int MAX_NAME_LEN = 64;
public static final int ID_LEN = 4096;
- public static final int STATISTICS_CACHE_VALID_DURATION_IN_HOURS = 24 * 2;
-
public static final int STATISTICS_CACHE_REFRESH_INTERVAL = 24 * 2;
public static final int ROW_COUNT_CACHE_VALID_DURATION_IN_HOURS = 12;
@@ -76,6 +73,8 @@ public class StatisticConstants {
public static final String DB_NAME = SystemInfoService.DEFAULT_CLUSTER +
":" + FeConstants.INTERNAL_DB_NAME;
+ public static final String FULL_QUALIFIED_STATS_TBL_NAME =
FeConstants.INTERNAL_DB_NAME + "." + STATISTIC_TBL_NAME;
+
public static final int STATISTIC_INTERNAL_TABLE_REPLICA_NUM = 3;
public static final int RETRY_LOAD_QUEUE_SIZE = 1000;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoAnalyzer.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoAnalyzer.java
index e07a4c1ddf..ad70769427 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoAnalyzer.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoAnalyzer.java
@@ -53,7 +53,7 @@ public class StatisticsAutoAnalyzer extends MasterDaemon {
private static final Logger LOG =
LogManager.getLogger(StatisticsAutoAnalyzer.class);
- private AnalysisTaskExecutor analysisTaskExecutor;
+ private final AnalysisTaskExecutor analysisTaskExecutor;
public StatisticsAutoAnalyzer() {
super("Automatic Analyzer",
@@ -70,6 +70,7 @@ public class StatisticsAutoAnalyzer extends MasterDaemon {
if (!StatisticsUtil.statsTblAvailable()) {
return;
}
+ analyzePeriodically();
if
(!checkAnalyzeTime(LocalTime.now(TimeUtils.getTimeZone().toZoneId()))) {
return;
}
@@ -78,7 +79,6 @@ public class StatisticsAutoAnalyzer extends MasterDaemon {
return;
}
- analyzePeriodically();
if (Config.enable_full_auto_analyze) {
analyzeAll();
}
@@ -145,10 +145,9 @@ public class StatisticsAutoAnalyzer extends MasterDaemon {
AnalysisManager analysisManager =
Env.getCurrentEnv().getAnalysisManager();
List<AnalysisInfo> jobInfos = analysisManager.findPeriodicJobs();
for (AnalysisInfo jobInfo : jobInfos) {
- jobInfo = new
AnalysisInfoBuilder(jobInfo).setJobType(JobType.SYSTEM).build();
createSystemAnalysisJob(jobInfo);
}
- } catch (DdlException e) {
+ } catch (Exception e) {
LOG.warn("Failed to periodically analyze the statistics." + e);
}
}
@@ -238,7 +237,7 @@ public class StatisticsAutoAnalyzer extends MasterDaemon {
// Analysis job created by the system
@VisibleForTesting
- public void createSystemAnalysisJob(AnalysisInfo jobInfo)
+ protected void createSystemAnalysisJob(AnalysisInfo jobInfo)
throws DdlException {
if (jobInfo.colToPartitions.isEmpty()) {
// No statistics need to be collected or updated
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/statistics/AnalysisJobTest.java
b/fe/fe-core/src/test/java/org/apache/doris/statistics/AnalysisJobTest.java
index e85bbf2c28..9624c20149 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/statistics/AnalysisJobTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/statistics/AnalysisJobTest.java
@@ -135,6 +135,7 @@ public class AnalysisJobTest extends TestWithFeService {
.setAnalysisMethod(AnalysisMethod.FULL)
.setAnalysisType(AnalysisType.FUNDAMENTALS)
.setColToPartitions(colToPartitions)
+ .setState(AnalysisState.RUNNING)
.build();
new OlapAnalysisTask(analysisJobInfo).doExecute();
new Expectations() {
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/statistics/AnalysisManagerTest.java
b/fe/fe-core/src/test/java/org/apache/doris/statistics/AnalysisManagerTest.java
index 4f7b1a9c5d..16ef1705d8 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/statistics/AnalysisManagerTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/statistics/AnalysisManagerTest.java
@@ -24,6 +24,7 @@ import org.apache.doris.analysis.TableName;
import org.apache.doris.common.DdlException;
import org.apache.doris.statistics.AnalysisInfo.AnalysisType;
import org.apache.doris.statistics.AnalysisInfo.JobType;
+import org.apache.doris.statistics.AnalysisInfo.ScheduleType;
import org.apache.doris.statistics.util.StatisticsUtil;
import com.google.common.annotations.VisibleForTesting;
@@ -182,7 +183,9 @@ public class AnalysisManagerTest {
// test build async job
@Test
public void testBuildAndAssignJob2(@Injectable OlapAnalysisTask
analysisTask) throws Exception {
- AnalysisInfo analysisInfo = new
AnalysisInfoBuilder().setColToPartitions(new HashMap<>()).build();
+ AnalysisInfo analysisInfo = new
AnalysisInfoBuilder().setColToPartitions(new HashMap<>())
+ .setScheduleType(ScheduleType.PERIOD)
+ .build();
new MockUp<StatisticsUtil>() {
@Mock
@@ -240,9 +243,9 @@ public class AnalysisManagerTest {
}, new AnalyzeProperties(new HashMap<String, String>() {
{
put(AnalyzeProperties.PROPERTY_SYNC, "false");
+ put(AnalyzeProperties.PROPERTY_PERIOD_SECONDS, "100");
}
}));
-
AnalysisManager analysisManager = new AnalysisManager();
analysisInfo.colToPartitions.put("c1", new HashSet<String>() {
{
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/statistics/AnalysisTaskExecutorTest.java
b/fe/fe-core/src/test/java/org/apache/doris/statistics/AnalysisTaskExecutorTest.java
index 6136b8efd4..196ac8ad9a 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/statistics/AnalysisTaskExecutorTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/statistics/AnalysisTaskExecutorTest.java
@@ -113,6 +113,7 @@ public class AnalysisTaskExecutorTest extends
TestWithFeService {
.setAnalysisMode(AnalysisMode.FULL)
.setAnalysisMethod(AnalysisMethod.FULL)
.setAnalysisType(AnalysisType.FUNDAMENTALS)
+ .setState(AnalysisState.RUNNING)
.setColToPartitions(colToPartitions)
.build();
OlapAnalysisTask task = new OlapAnalysisTask(analysisInfo);
diff --git a/regression-test/suites/statistics/analyze_stats.groovy
b/regression-test/suites/statistics/analyze_stats.groovy
index 40f7720256..50420613b5 100644
--- a/regression-test/suites/statistics/analyze_stats.groovy
+++ b/regression-test/suites/statistics/analyze_stats.groovy
@@ -143,99 +143,99 @@ suite("test_analyze") {
ANALYZE DATABASE ${db} WITH SYNC WITH SAMPLE PERCENT 10
"""
-// a_result_2 = sql """
-// ANALYZE DATABASE ${db} WITH SYNC WITH SAMPLE PERCENT 5
-// """
-//
-// a_result_3 = sql """
-// ANALYZE DATABASE ${db} WITH SAMPLE PERCENT 5
-// """
-//
-// show_result = sql """
-// SHOW ANALYZE
-// """
-//
-// def contains_expected_table = { r ->
-// for (int i = 0; i < r.size; i++) {
-// if (r[i][3] == "${tbl}") {
-// return true
-// }
-// }
-// return false
-// }
-//
-// def stats_job_removed = { r, id ->
-// for (int i = 0; i < r.size; i++) {
-// if (r[i][0] == id) {
-// return false
-// }
-// }
-// return true
-// }
-//
-// assert contains_expected_table(show_result)
-//
-// sql """
-// DROP ANALYZE JOB ${a_result_3[0][4]}
-// """
-//
-// show_result = sql """
-// SHOW ANALYZE
-// """
-//
-// assert stats_job_removed(show_result, a_result_3[0][4])
-//
-// sql """
-// ANALYZE DATABASE ${db} WITH SAMPLE ROWS 5 WITH PERIOD 100000
-// """
-//
-// sql """
-// DROP TABLE IF EXISTS analyze_partitioned_tbl_test
-// """
-
-// sql """
-// CREATE TABLE analyze_partitioned_tbl_test (col1 int, col2 int, col3
int)
-// PARTITION BY RANGE(`col2`) (
-// PARTITION `p1` VALUES LESS THAN ('5'),
-// PARTITION `p2` VALUES LESS THAN ('10'),
-// PARTITION `P3` VALUES LESS THAN ('15'),
-// PARTITION `P4` VALUES LESS THAN ('20'),
-// PARTITION `P5` VALUES LESS THAN ('25'),
-// PARTITION `P6` VALUES LESS THAN ('30'))
-// DISTRIBUTED BY HASH(col3)
-// BUCKETS 3
-// PROPERTIES(
-// "replication_num"="1"
-// )
-// """
-//
-// sql """insert into analyze_partitioned_tbl_test values(1,3,1) """
-// sql """insert into analyze_partitioned_tbl_test values(6,6,6) """
-// sql """insert into analyze_partitioned_tbl_test values(11,6,6) """
-// sql """insert into analyze_partitioned_tbl_test values(16,6,6) """
-// sql """insert into analyze_partitioned_tbl_test values(21,6,6) """
-// sql """insert into analyze_partitioned_tbl_test values(26,6,6) """
-//
-// sql """
-// ANALYZE TABLE analyze_partitioned_tbl_test WITH SYNC
-// """
-//
-// part_tbl_analyze_result = sql """
-// SHOW COLUMN CACHED STATS analyze_partitioned_tbl_test(col1)
-// """
-//
-// def expected_result = { r ->
-// for (int i = 0; i < r.size; i++) {
-// if ((int) Double.parseDouble(r[i][1]) == 6) {
-// return true
-// } else {
-// return false
-// }
-// }
-// return false
-// }
-//
-// assert expected_result(part_tbl_analyze_result)
+ a_result_2 = sql """
+ ANALYZE DATABASE ${db} WITH SYNC WITH SAMPLE PERCENT 5
+ """
+
+ a_result_3 = sql """
+ ANALYZE DATABASE ${db} WITH SAMPLE PERCENT 5
+ """
+
+ show_result = sql """
+ SHOW ANALYZE
+ """
+
+ def contains_expected_table = { r ->
+ for (int i = 0; i < r.size; i++) {
+ if (r[i][3] == "${tbl}") {
+ return true
+ }
+ }
+ return false
+ }
+
+ def stats_job_removed = { r, id ->
+ for (int i = 0; i < r.size; i++) {
+ if (r[i][0] == id) {
+ return false
+ }
+ }
+ return true
+ }
+
+ assert contains_expected_table(show_result)
+
+ sql """
+ DROP ANALYZE JOB ${a_result_3[0][4]}
+ """
+
+ show_result = sql """
+ SHOW ANALYZE
+ """
+
+ assert stats_job_removed(show_result, a_result_3[0][4])
+
+ sql """
+ ANALYZE DATABASE ${db} WITH SAMPLE ROWS 5 WITH PERIOD 100000
+ """
+
+ sql """
+ DROP TABLE IF EXISTS analyze_partitioned_tbl_test
+ """
+
+ sql """
+ CREATE TABLE analyze_partitioned_tbl_test (col1 int, col2 int, col3
int)
+ PARTITION BY RANGE(`col2`) (
+ PARTITION `p1` VALUES LESS THAN ('5'),
+ PARTITION `p2` VALUES LESS THAN ('10'),
+ PARTITION `P3` VALUES LESS THAN ('15'),
+ PARTITION `P4` VALUES LESS THAN ('20'),
+ PARTITION `P5` VALUES LESS THAN ('25'),
+ PARTITION `P6` VALUES LESS THAN ('30'))
+ DISTRIBUTED BY HASH(col3)
+ BUCKETS 3
+ PROPERTIES(
+ "replication_num"="1"
+ )
+ """
+
+ sql """insert into analyze_partitioned_tbl_test values(1,3,1) """
+ sql """insert into analyze_partitioned_tbl_test values(6,6,6) """
+ sql """insert into analyze_partitioned_tbl_test values(11,6,6) """
+ sql """insert into analyze_partitioned_tbl_test values(16,6,6) """
+ sql """insert into analyze_partitioned_tbl_test values(21,6,6) """
+ sql """insert into analyze_partitioned_tbl_test values(26,6,6) """
+
+ sql """
+ ANALYZE TABLE analyze_partitioned_tbl_test WITH SYNC
+ """
+
+ part_tbl_analyze_result = sql """
+ SHOW COLUMN CACHED STATS analyze_partitioned_tbl_test(col1)
+ """
+
+ def expected_result = { r ->
+ for (int i = 0; i < r.size; i++) {
+ if ((int) Double.parseDouble(r[i][1]) == 6) {
+ return true
+ } else {
+ return false
+ }
+ }
+ return false
+ }
+
+ assert expected_result(part_tbl_analyze_result)
sql """
DROP TABLE IF EXISTS test_600_partition_table_analyze;
@@ -877,4 +877,31 @@ PARTITION `p599` VALUES IN (599)
assert expected_id_col_stats(id_col_stats, 600, 1)
assert expected_id_col_stats(id_col_stats, 599, 7)
assert expected_id_col_stats(id_col_stats, 0, 6)
+
+ sql """DROP TABLE IF EXISTS increment_analyze_test"""
+ sql """
+ CREATE TABLE increment_analyze_test (
+ id BIGINT
+ ) DUPLICATE KEY(`id`)
+ PARTITION BY RANGE(`id`)
+ (
+ PARTITION `p1` VALUES LESS THAN ('5')
+ )
+
+ DISTRIBUTED BY HASH(`id`) BUCKETS 3
+ PROPERTIES (
+ "replication_num"="1"
+ );
+ """
+
+ sql """INSERT INTO increment_analyze_test VALUES(1),(2),(3)"""
+ sql """ANALYZE TABLE increment_analyze_test WITH SYNC"""
+ sql """ALTER TABLE increment_analyze_test ADD PARTITION p2 VALUES LESS
THAN('10')"""
+
+ sql """INSERT INTO increment_analyze_test VALUES(6),(7),(8)"""
+ sql """ANALYZE TABLE increment_analyze_test WITH SYNC WITH INCREMENTAL"""
+ def inc_res = sql """
+ SHOW COLUMN CACHED STATS increment_analyze_test(id)
+ """
+ expected_id_col_stats(inc_res, 6, 1)
}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
