This is an automated email from the ASF dual-hosted git repository.
starocean999 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 3466a0871bb [Enhancement] (nereids) delete translate paritionNamesIno
in ShowReplicaDistributionCommand/ShowDataSkewCommand (#51850)
3466a0871bb is described below
commit 3466a0871bbb4318aff15b2d4b8e64b9e4d589b3
Author: yaoxiao <[email protected]>
AuthorDate: Thu Jun 19 10:56:28 2025 +0800
[Enhancement] (nereids) delete translate paritionNamesIno in
ShowReplicaDistributionCommand/ShowDataSkewCommand (#51850)
---
.../org/apache/doris/catalog/MetadataViewer.java | 179 +++++++++++++++++++++
.../trees/plans/commands/ShowDataSkewCommand.java | 5 +-
.../commands/ShowReplicaDistributionCommand.java | 6 +-
3 files changed, 181 insertions(+), 9 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/MetadataViewer.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/MetadataViewer.java
index ac1dd054012..45dd1bb3a0e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/MetadataViewer.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/MetadataViewer.java
@@ -29,6 +29,7 @@ import org.apache.doris.common.FeConstants;
import org.apache.doris.nereids.trees.expressions.EqualTo;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.plans.commands.ShowReplicaStatusCommand;
+import org.apache.doris.nereids.trees.plans.commands.info.PartitionNamesInfo;
import org.apache.doris.resource.Tag;
import org.apache.doris.system.Backend;
import org.apache.doris.system.SystemInfoService;
@@ -300,6 +301,105 @@ public class MetadataViewer {
}
}
+ public static List<List<String>> getTabletDistribution(
+ String dbName, String tblName, PartitionNamesInfo
partitionNamesInfo)
+ throws DdlException {
+ DecimalFormat df = new DecimalFormat("00.00 %");
+
+ List<List<String>> result = Lists.newArrayList();
+
+ Env env = Env.getCurrentEnv();
+ SystemInfoService infoService = Env.getCurrentSystemInfo();
+
+ Database db = env.getInternalCatalog().getDbOrDdlException(dbName);
+ OlapTable olapTable = db.getOlapTableOrDdlException(tblName);
+ olapTable.readLock();
+ try {
+ List<Long> partitionIds = Lists.newArrayList();
+ if (partitionNamesInfo == null) {
+ for (Partition partition : olapTable.getPartitions()) {
+ partitionIds.add(partition.getId());
+ }
+ } else {
+ // check partition
+ for (String partName : partitionNamesInfo.getPartitionNames())
{
+ Partition partition = olapTable.getPartition(partName,
partitionNamesInfo.isTemp());
+ if (partition == null) {
+ throw new DdlException("Partition does not exist: " +
partName);
+ }
+ partitionIds.add(partition.getId());
+ }
+ }
+
+ // backend id -> replica count
+ Map<Long, Integer> countMap = Maps.newHashMap();
+ // backend id -> replica size
+ Map<Long, Long> sizeMap = Maps.newHashMap();
+ // init map
+ List<Long> beIds = infoService.getAllBackendIds(false);
+ for (long beId : beIds) {
+ countMap.put(beId, 0);
+ sizeMap.put(beId, 0L);
+ }
+
+ int totalReplicaNum = 0;
+ long totalReplicaSize = 0;
+ for (long partId : partitionIds) {
+ Partition partition = olapTable.getPartition(partId);
+ for (MaterializedIndex index :
partition.getMaterializedIndices(IndexExtState.VISIBLE)) {
+ for (Tablet tablet : index.getTablets()) {
+ for (Replica replica : tablet.getReplicas()) {
+ long beId = replica.getBackendIdWithoutException();
+ if (!countMap.containsKey(beId)) {
+ continue;
+ }
+ countMap.put(beId,
+ countMap.get(beId) + 1);
+ sizeMap.put(beId,
+ sizeMap.get(beId) + replica.getDataSize());
+ totalReplicaNum++;
+ totalReplicaSize += replica.getDataSize();
+ }
+ }
+ }
+ }
+
+ // graph
+ Collections.sort(beIds);
+ for (Long beId : beIds) {
+ List<String> row = Lists.newArrayList();
+ row.add(String.valueOf(beId));
+ row.add(String.valueOf(countMap.get(beId)));
+ row.add(String.valueOf(sizeMap.get(beId)));
+ row.add(graph(countMap.get(beId), totalReplicaNum));
+ row.add(totalReplicaNum == countMap.get(beId) ?
(totalReplicaNum == 0 ? "0.00%" : "100.00%")
+ : df.format((double) countMap.get(beId) /
totalReplicaNum));
+ row.add(graph(sizeMap.get(beId), totalReplicaSize));
+ row.add(totalReplicaSize == sizeMap.get(beId) ?
(totalReplicaSize == 0 ? "0.00%" : "100.00%")
+ : df.format((double) sizeMap.get(beId) /
totalReplicaSize));
+ if (Config.isNotCloudMode()) {
+ row.add("");
+ row.add("");
+ } else {
+ Backend be =
CloudEnv.getCurrentSystemInfo().getBackend(beId);
+ if (be != null) {
+ row.add(be.getTagMap().get(Tag.CLOUD_CLUSTER_NAME));
+ row.add(be.getTagMap().get(Tag.CLOUD_CLUSTER_ID));
+ } else {
+ row.add("not exist be");
+ row.add("not exist be");
+ }
+ }
+ result.add(row);
+ }
+
+ } finally {
+ olapTable.readUnlock();
+ }
+
+ return result;
+ }
+
public static List<List<String>> getTabletDistribution(
String dbName, String tblName, PartitionNames partitionNames)
throws DdlException {
@@ -408,6 +508,85 @@ public class MetadataViewer {
return sb.toString();
}
+ public static List<List<String>> getDataSkew(String dbName, String
tblName, PartitionNamesInfo partitionNamesInfo)
+ throws DdlException {
+ DecimalFormat df = new DecimalFormat("00.00 %");
+
+ List<List<String>> result = Lists.newArrayList();
+ Env env = Env.getCurrentEnv();
+
+ Database db = env.getInternalCatalog().getDbOrDdlException(dbName);
+ OlapTable olapTable = db.getOlapTableOrDdlException(tblName);
+
+ if (olapTable.getPartitionNames().isEmpty()) {
+ throw new DdlException("Can not find any partition from " + dbName
+ "." + tblName);
+ }
+
+ // patition -> isTmep
+ Map<String, Boolean> allPartionNames = new HashMap<>();
+ if (partitionNamesInfo == null) {
+ for (Partition p : olapTable.getPartitions()) {
+ allPartionNames.put(p.getName(), false);
+ }
+ for (Partition p : olapTable.getAllTempPartitions()) {
+ allPartionNames.put(p.getName(), true);
+ }
+ } else {
+ for (String name : partitionNamesInfo.getPartitionNames()) {
+ allPartionNames.put(name, partitionNamesInfo.isTemp());
+ }
+ }
+
+ olapTable.readLock();
+ try {
+ Partition partition = null;
+ // check partition
+ for (Map.Entry<String, Boolean> partName :
allPartionNames.entrySet()) {
+ partition = olapTable.getPartition(partName.getKey(),
partName.getValue());
+ if (partition == null) {
+ throw new DdlException("Partition does not exist: " +
partName);
+ }
+ DistributionInfo distributionInfo =
partition.getDistributionInfo();
+ List<Long> rowCountTabletInfos =
Lists.newArrayListWithCapacity(distributionInfo.getBucketNum());
+ List<Long> dataSizeTabletInfos =
Lists.newArrayListWithCapacity(distributionInfo.getBucketNum());
+ for (long i = 0; i < distributionInfo.getBucketNum(); i++) {
+ rowCountTabletInfos.add(0L);
+ dataSizeTabletInfos.add(0L);
+ }
+
+ long totalSize = 0;
+ for (MaterializedIndex mIndex :
partition.getMaterializedIndices(IndexExtState.VISIBLE)) {
+ List<Long> tabletIds = mIndex.getTabletIdsInOrder();
+ for (int i = 0; i < tabletIds.size(); i++) {
+ Tablet tablet = mIndex.getTablet(tabletIds.get(i));
+ long rowCount = tablet.getRowCount(true);
+ long dataSize = tablet.getDataSize(true);
+ rowCountTabletInfos.set(i, rowCountTabletInfos.get(i)
+ rowCount);
+ dataSizeTabletInfos.set(i, dataSizeTabletInfos.get(i)
+ dataSize);
+ totalSize += dataSize;
+ }
+ }
+
+ // graph
+ for (int i = 0; i < distributionInfo.getBucketNum(); i++) {
+ List<String> row = Lists.newArrayList();
+ row.add(partName.getKey());
+ row.add(String.valueOf(i));
+ row.add(rowCountTabletInfos.get(i).toString());
+ row.add(dataSizeTabletInfos.get(i).toString());
+ row.add(graph(dataSizeTabletInfos.get(i), totalSize));
+ row.add(totalSize == dataSizeTabletInfos.get(i) ?
(totalSize == 0L ? "0.00%" : "100.00%") :
+ df.format((double) dataSizeTabletInfos.get(i) /
totalSize));
+ result.add(row);
+ }
+ }
+ } finally {
+ olapTable.readUnlock();
+ }
+
+ return result;
+ }
+
public static List<List<String>> getDataSkew(String dbName, String
tblName, PartitionNames partitionNames)
throws DdlException {
DecimalFormat df = new DecimalFormat("00.00 %");
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowDataSkewCommand.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowDataSkewCommand.java
index 993f7db225e..4e56c9acd11 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowDataSkewCommand.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowDataSkewCommand.java
@@ -17,7 +17,6 @@
package org.apache.doris.nereids.trees.plans.commands;
-import org.apache.doris.analysis.PartitionNames;
import org.apache.doris.analysis.RedirectStatus;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.Env;
@@ -78,10 +77,8 @@ public class ShowDataSkewCommand extends ShowCommand {
tableRefInfo.analyze(ctx);
Util.prohibitExternalCatalog(tableRefInfo.getTableNameInfo().getCtl(),
this.getClass().getSimpleName());
- PartitionNames partitionNames =
(tableRefInfo.getPartitionNamesInfo() != null)
- ?
tableRefInfo.getPartitionNamesInfo().translateToLegacyPartitionNames() : null;
List<List<String>> results =
MetadataViewer.getDataSkew(tableRefInfo.getTableNameInfo().getDb(),
- tableRefInfo.getTableNameInfo().getTbl(), partitionNames);
+ tableRefInfo.getTableNameInfo().getTbl(),
tableRefInfo.getPartitionNamesInfo());
return new ShowResultSet(getMetaData(), results);
} catch (DdlException e) {
throw new AnalysisException(e.getMessage());
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowReplicaDistributionCommand.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowReplicaDistributionCommand.java
index 2a073b59fbc..531d1ce546b 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowReplicaDistributionCommand.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowReplicaDistributionCommand.java
@@ -17,7 +17,6 @@
package org.apache.doris.nereids.trees.plans.commands;
-import org.apache.doris.analysis.PartitionNames;
import org.apache.doris.analysis.RedirectStatus;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.Env;
@@ -77,11 +76,8 @@ public class ShowReplicaDistributionCommand extends
ShowCommand {
List<List<String>> results;
try {
- PartitionNames partitionNames =
(tableRefInfo.getPartitionNamesInfo() != null)
- ?
tableRefInfo.getPartitionNamesInfo().translateToLegacyPartitionNames() : null;
results =
MetadataViewer.getTabletDistribution(tableRefInfo.getTableNameInfo().getDb(),
- tableRefInfo.getTableNameInfo().getTbl(),
- partitionNames);
+ tableRefInfo.getTableNameInfo().getTbl(),
tableRefInfo.getPartitionNamesInfo());
} catch (DdlException e) {
throw new AnalysisException(e.getMessage());
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
