This is an automated email from the ASF dual-hosted git repository.
lijibing pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new acd7d8db250 Record partition update rows for each column. (#35227)
acd7d8db250 is described below
commit acd7d8db250c9a5b00f4802befdf1e1842dc99a1
Author: Jibing-Li <[email protected]>
AuthorDate: Thu May 23 19:47:19 2024 +0800
Record partition update rows for each column. (#35227)
---
.../apache/doris/analysis/ShowColumnStatsStmt.java | 48 +++++++++++++++-------
.../org/apache/doris/statistics/AnalysisInfo.java | 9 +++-
.../doris/statistics/AnalysisInfoBuilder.java | 10 ++++-
.../apache/doris/statistics/AnalysisManager.java | 4 ++
.../org/apache/doris/statistics/ColStatsMeta.java | 11 ++++-
.../doris/statistics/StatisticsAutoCollector.java | 1 +
.../doris/statistics/StatisticsRepository.java | 4 ++
.../apache/doris/statistics/TableStatsMeta.java | 12 ++++--
.../doris/statistics/util/StatisticsUtilTest.java | 10 ++---
9 files changed, 84 insertions(+), 25 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowColumnStatsStmt.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowColumnStatsStmt.java
index 01612af9f9f..6c54175784f 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowColumnStatsStmt.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowColumnStatsStmt.java
@@ -20,6 +20,8 @@ package org.apache.doris.analysis;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.DatabaseIf;
import org.apache.doris.catalog.Env;
+import org.apache.doris.catalog.OlapTable;
+import org.apache.doris.catalog.Partition;
import org.apache.doris.catalog.ScalarType;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.common.ErrorCode;
@@ -35,6 +37,7 @@ import org.apache.doris.statistics.AnalysisManager;
import org.apache.doris.statistics.ColStatsMeta;
import org.apache.doris.statistics.ColumnStatistic;
import org.apache.doris.statistics.ResultRow;
+import org.apache.doris.statistics.TableStatsMeta;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
@@ -186,22 +189,39 @@ public class ShowColumnStatsStmt extends ShowStmt {
public ShowResultSet constructPartitionResultSet(List<ResultRow>
resultRows, TableIf tableIf) {
List<List<String>> result = Lists.newArrayList();
- resultRows.forEach(r -> {
+ for (ResultRow r : resultRows) {
List<String> row = Lists.newArrayList();
- row.add(r.get(0));
- row.add(r.get(1));
- row.add(r.get(2)); // TODO: Get index name.
- row.add(r.get(3));
- row.add(r.get(4));
- row.add(r.get(5));
- row.add(r.get(6));
- row.add(r.get(7));
- row.add(r.get(8));
- row.add(r.get(9));
- row.add("N/A");
- row.add("Manual");
+ row.add(r.get(0)); // column_name
+ row.add(r.get(1)); // partition_name
+ long indexId = Long.parseLong(r.get(2));
+ String indexName = indexId == -1 ? tableIf.getName() :
((OlapTable) tableIf).getIndexNameById(indexId);
+ row.add(indexName); // index_name.
+ row.add(r.get(3)); // count
+ row.add(r.get(4)); // ndv
+ row.add(r.get(5)); // num_null
+ row.add(r.get(6)); // min
+ row.add(r.get(7)); // max
+ row.add(r.get(8)); // data_size
+ row.add(r.get(9)); // updated_time
+ String updateRows = "N/A";
+ TableStatsMeta tableStats =
Env.getCurrentEnv().getAnalysisManager().findTableStatsStatus(tableIf.getId());
+ if (tableStats != null && tableIf instanceof OlapTable) {
+ OlapTable olapTable = (OlapTable) tableIf;
+ ColStatsMeta columnStatsMeta =
tableStats.findColumnStatsMeta(indexName, r.get(0));
+ if (columnStatsMeta != null &&
columnStatsMeta.partitionUpdateRows != null) {
+ Partition partition = olapTable.getPartition(r.get(1));
+ if (partition != null) {
+ Long rows =
columnStatsMeta.partitionUpdateRows.get(partition.getId());
+ if (rows != null) {
+ updateRows = rows.toString();
+ }
+ }
+ }
+ }
+ row.add(updateRows); // update_rows
+ row.add("Manual"); // trigger. Manual or System
result.add(row);
- });
+ }
return new ShowResultSet(getMetaData(), result);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfo.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfo.java
index 4e4960f6ecf..ef0d84e66c9 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfo.java
@@ -35,7 +35,9 @@ import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.text.ParseException;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import java.util.Set;
import java.util.StringJoiner;
@@ -188,6 +190,8 @@ public class AnalysisInfo implements Writable {
@SerializedName("updateRows")
public final long updateRows;
+
+ public final Map<Long, Long> partitionUpdateRows = new HashMap();
/**
*
* Used to store the newest partition version of tbl when creating this
job.
@@ -209,7 +213,7 @@ public class AnalysisInfo implements Writable {
boolean partitionOnly, boolean samplingPartition,
boolean isAllPartition, long partitionCount, CronExpression
cronExpression, boolean forceFull,
boolean usingSqlForExternalTable, long tblUpdateTime, long
rowCount, boolean userInject,
- long updateRows, JobPriority priority) {
+ long updateRows, JobPriority priority, Map<Long, Long>
partitionUpdateRows) {
this.jobId = jobId;
this.taskId = taskId;
this.taskIds = taskIds;
@@ -248,6 +252,9 @@ public class AnalysisInfo implements Writable {
this.userInject = userInject;
this.updateRows = updateRows;
this.priority = priority;
+ if (partitionUpdateRows != null) {
+ this.partitionUpdateRows.putAll(partitionUpdateRows);
+ }
}
@Override
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfoBuilder.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfoBuilder.java
index 97e96112d79..e8cef0f64d6 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfoBuilder.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfoBuilder.java
@@ -27,6 +27,7 @@ import org.apache.doris.statistics.AnalysisInfo.ScheduleType;
import org.apache.logging.log4j.core.util.CronExpression;
import java.util.List;
+import java.util.Map;
import java.util.Set;
public class AnalysisInfoBuilder {
@@ -65,6 +66,7 @@ public class AnalysisInfoBuilder {
private boolean userInject;
private long updateRows;
private JobPriority priority;
+ private Map<Long, Long> partitionUpdateRows;
public AnalysisInfoBuilder() {
}
@@ -105,6 +107,7 @@ public class AnalysisInfoBuilder {
userInject = info.userInject;
updateRows = info.updateRows;
priority = info.priority;
+ partitionUpdateRows = info.partitionUpdateRows;
}
public AnalysisInfoBuilder setJobId(long jobId) {
@@ -282,13 +285,18 @@ public class AnalysisInfoBuilder {
return this;
}
+ public AnalysisInfoBuilder setPartitionUpdateRows(Map<Long, Long>
partitionUpdateRows) {
+ this.partitionUpdateRows = partitionUpdateRows;
+ return this;
+ }
+
public AnalysisInfo build() {
return new AnalysisInfo(jobId, taskId, taskIds, catalogId, dbId,
tblId, jobColumns, partitionNames,
colName, indexId, jobType, analysisMode, analysisMethod,
analysisType, samplePercent,
sampleRows, maxBucketNum, periodTimeInMs, message,
lastExecTimeInMs, timeCostInMs, state, scheduleType,
partitionOnly, samplingPartition, isAllPartition,
partitionCount,
cronExpression, forceFull, usingSqlForExternalTable,
tblUpdateTime, rowCount, userInject, updateRows,
- priority);
+ priority, partitionUpdateRows);
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
index 57109400ee7..18f7f79b4c9 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
@@ -381,6 +381,7 @@ public class AnalysisManager implements Writable {
TableStatsMeta tableStatsStatus = findTableStatsStatus(table.getId());
infoBuilder.setUpdateRows(tableStatsStatus == null ? 0 :
tableStatsStatus.updatedRows.get());
infoBuilder.setPriority(JobPriority.MANUAL);
+ infoBuilder.setPartitionUpdateRows(tableStatsStatus == null ? null :
tableStatsStatus.partitionUpdateRows);
return infoBuilder.build();
}
@@ -514,6 +515,9 @@ public class AnalysisManager implements Writable {
if (jobInfo.partitionNames != null) {
jobInfo.partitionNames.clear();
}
+ if (jobInfo.partitionUpdateRows != null) {
+ jobInfo.partitionUpdateRows.clear();
+ }
}
@VisibleForTesting
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/ColStatsMeta.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/ColStatsMeta.java
index 7e317d67bd7..6cb2ced9286 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/ColStatsMeta.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/ColStatsMeta.java
@@ -23,6 +23,9 @@ import org.apache.doris.statistics.AnalysisInfo.JobType;
import com.google.gson.annotations.SerializedName;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.atomic.AtomicLong;
public class ColStatsMeta {
@@ -49,8 +52,11 @@ public class ColStatsMeta {
@SerializedName("rowCount")
public long rowCount;
+ @SerializedName("pur")
+ public ConcurrentMap<Long, Long> partitionUpdateRows = new
ConcurrentHashMap<>();
+
public ColStatsMeta(long updatedTime, AnalysisMethod analysisMethod,
AnalysisType analysisType, JobType jobType,
- long queriedTimes, long rowCount, long updatedRows) {
+ long queriedTimes, long rowCount, long updatedRows, Map<Long,
Long> partitionUpdateRows) {
this.updatedTime = updatedTime;
this.analysisMethod = analysisMethod;
this.analysisType = analysisType;
@@ -58,5 +64,8 @@ public class ColStatsMeta {
this.queriedTimes.addAndGet(queriedTimes);
this.updatedRows = updatedRows;
this.rowCount = rowCount;
+ if (partitionUpdateRows != null) {
+ this.partitionUpdateRows.putAll(partitionUpdateRows);
+ }
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java
index d64813afbc9..227184ba852 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java
@@ -210,6 +210,7 @@ public class StatisticsAutoCollector extends MasterDaemon {
.setRowCount(rowCount)
.setUpdateRows(tableStatsStatus == null ? 0 :
tableStatsStatus.updatedRows.get())
.setPriority(priority)
+ .setPartitionUpdateRows(tableStatsStatus == null ? null :
tableStatsStatus.partitionUpdateRows)
.build();
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsRepository.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsRepository.java
index a17b85002d2..dee9cb042fe 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsRepository.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsRepository.java
@@ -34,6 +34,7 @@ import org.apache.commons.text.StringSubstitutor;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
+import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
@@ -122,6 +123,9 @@ public class StatisticsRepository {
public static List<ResultRow> queryColumnStatisticsByPartitions(TableIf
table, Set<String> columnNames,
List<String> partitionNames) {
+ if (!table.isPartitionedTable()) {
+ return new ArrayList<>();
+ }
long ctlId = table.getDatabase().getCatalog().getId();
long dbId = table.getDatabase().getId();
Map<String, String> params = new HashMap<>();
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/TableStatsMeta.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/TableStatsMeta.java
index 3aa7decfed3..f9c633844ca 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/TableStatsMeta.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/TableStatsMeta.java
@@ -134,16 +134,22 @@ public class TableStatsMeta implements Writable {
for (Pair<String, String> colPair : analyzedJob.jobColumns) {
ColStatsMeta colStatsMeta = colToColStatsMeta.get(colPair);
if (colStatsMeta == null) {
- colToColStatsMeta.put(colPair, new ColStatsMeta(updatedTime,
analyzedJob.analysisMethod,
+ colToColStatsMeta.put(colPair, new
ColStatsMeta(analyzedJob.createTime, analyzedJob.analysisMethod,
analyzedJob.analysisType, analyzedJob.jobType, 0,
analyzedJob.rowCount,
- analyzedJob.updateRows));
+ analyzedJob.updateRows,
analyzedJob.partitionUpdateRows));
} else {
- colStatsMeta.updatedTime = updatedTime;
+ colStatsMeta.updatedTime = analyzedJob.startTime;
colStatsMeta.analysisType = analyzedJob.analysisType;
colStatsMeta.analysisMethod = analyzedJob.analysisMethod;
colStatsMeta.jobType = analyzedJob.jobType;
colStatsMeta.updatedRows = analyzedJob.updateRows;
colStatsMeta.rowCount = analyzedJob.rowCount;
+ if (colStatsMeta.partitionUpdateRows == null) {
+ colStatsMeta.partitionUpdateRows = new
ConcurrentHashMap<>();
+ } else {
+ colStatsMeta.partitionUpdateRows.clear();
+ }
+
colStatsMeta.partitionUpdateRows.putAll(analyzedJob.partitionUpdateRows);
}
}
jobType = analyzedJob.jobType;
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/statistics/util/StatisticsUtilTest.java
b/fe/fe-core/src/test/java/org/apache/doris/statistics/util/StatisticsUtilTest.java
index 275471a6698..49e7751f264 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/statistics/util/StatisticsUtilTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/statistics/util/StatisticsUtilTest.java
@@ -196,7 +196,7 @@ class StatisticsUtilTest {
new MockUp<TableStatsMeta>() {
@Mock
public ColStatsMeta findColumnStatsMeta(String indexName, String
colName) {
- return new ColStatsMeta(0, null, null, null, 0, 0, 0);
+ return new ColStatsMeta(0, null, null, null, 0, 0, 0, null);
}
};
@@ -244,7 +244,7 @@ class StatisticsUtilTest {
new MockUp<TableStatsMeta>() {
@Mock
public ColStatsMeta findColumnStatsMeta(String indexName, String
colName) {
- return new ColStatsMeta(0, null, null, null, 0, 100, 0);
+ return new ColStatsMeta(0, null, null, null, 0, 100, 0, null);
}
};
tableMeta.newPartitionLoaded.set(false);
@@ -254,7 +254,7 @@ class StatisticsUtilTest {
new MockUp<TableStatsMeta>() {
@Mock
public ColStatsMeta findColumnStatsMeta(String indexName, String
colName) {
- return new ColStatsMeta(0, null, null, null, 0, 0, 0);
+ return new ColStatsMeta(0, null, null, null, 0, 0, 0, null);
}
};
tableMeta.newPartitionLoaded.set(false);
@@ -270,7 +270,7 @@ class StatisticsUtilTest {
new MockUp<TableStatsMeta>() {
@Mock
public ColStatsMeta findColumnStatsMeta(String indexName, String
colName) {
- return new ColStatsMeta(0, null, null, null, 0, 500, 0);
+ return new ColStatsMeta(0, null, null, null, 0, 500, 0, null);
}
};
tableMeta.newPartitionLoaded.set(false);
@@ -286,7 +286,7 @@ class StatisticsUtilTest {
new MockUp<TableStatsMeta>() {
@Mock
public ColStatsMeta findColumnStatsMeta(String indexName, String
colName) {
- return new ColStatsMeta(0, null, null, null, 0, 100, 80);
+ return new ColStatsMeta(0, null, null, null, 0, 100, 80, null);
}
};
tableMeta.newPartitionLoaded.set(false);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
