This is an automated email from the ASF dual-hosted git repository. morningman pushed a commit to branch dev-1.0.1 in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
commit 1f17a2b9ae7b0f5b6463f0967818042dddb8f955 Author: Mingyu Chen <[email protected]> AuthorDate: Tue Jun 14 15:34:16 2022 +0800 [fix] fix bug that show proc "/cluster_balance/history_tablets" return malformat error (#10073) --- .../common/proc/TabletSchedulerDetailProcDir.java | 11 ++-- .../org/apache/doris/planner/HashJoinNode.java | 22 ++++---- .../org/apache/doris/planner/OlapScanNode.java | 58 +++++++--------------- .../org/apache/doris/planner/PlanFragment.java | 5 +- .../org/apache/doris/planner/ColocatePlanTest.java | 2 +- .../doris/planner/DistributedPlannerTest.java | 8 +-- .../planner/MaterializedViewFunctionTest.java | 16 +++--- .../apache/doris/planner/ResourceTagQueryTest.java | 6 +-- .../doris/planner/TableFunctionPlanTest.java | 2 +- .../data/performance/redundant_conjuncts.out | 28 +++-------- 10 files changed, 63 insertions(+), 95 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/proc/TabletSchedulerDetailProcDir.java b/fe/fe-core/src/main/java/org/apache/doris/common/proc/TabletSchedulerDetailProcDir.java index 161cbc27c2..b050aa7db3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/proc/TabletSchedulerDetailProcDir.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/TabletSchedulerDetailProcDir.java @@ -34,12 +34,11 @@ import java.util.List; * show proc "/tablet_scheduler/history_tablets"; */ public class TabletSchedulerDetailProcDir implements ProcDirInterface { - public static final ImmutableList<String> TITLE_NAMES = new ImmutableList.Builder<String>() - .add("TabletId").add("Type").add("Medium").add("Status").add("State").add("OrigPrio").add("DynmPrio") - .add("SrcBe").add("SrcPath").add("DestBe").add("DestPath").add("Timeout") - .add("Create").add("LstSched").add("LstVisit").add("Finished").add("Rate").add("FailedSched") - .add("FailedRunning").add("LstAdjPrio").add("VisibleVer").add("VisibleVerHash") - .add("CmtVer").add("CmtVerHash").add("ErrMsg") + public static final ImmutableList<String> TITLE_NAMES = new ImmutableList.Builder<String>().add("TabletId") + .add("Type").add("Medium").add("Status").add("State").add("OrigPrio").add("DynmPrio").add("SrcBe") + .add("SrcPath").add("DestBe").add("DestPath").add("Timeout").add("Create").add("LstSched").add("LstVisit") + .add("Finished").add("Rate").add("FailedSched").add("FailedRunning").add("LstAdjPrio").add("VisibleVer") + .add("CmtVer").add("ErrMsg") .build(); private String type; diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/HashJoinNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/HashJoinNode.java index 5e6fa40fad..4f1411ce64 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/HashJoinNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/HashJoinNode.java @@ -682,23 +682,26 @@ public class HashJoinNode extends PlanNode { @Override public String getNodeExplainString(String detailPrefix, TExplainLevel detailLevel) { - String distrModeStr = - (distrMode != DistributionMode.NONE) ? (" (" + distrMode.toString() + ")") : ""; - StringBuilder output = new StringBuilder() - .append(detailPrefix).append("join op: ").append(joinOp.toString()).append(distrModeStr).append("\n"); + String distrModeStr = ""; + if (isColocate) { + distrModeStr = "COLOCATE[" + colocateReason + "]"; + } else { + distrModeStr = distrMode.toString(); + } + StringBuilder output = + new StringBuilder().append(detailPrefix).append("join op: ").append(joinOp.toString()).append("(") + .append(distrModeStr).append(")").append("[").append(colocateReason).append("]\n"); if (detailLevel == TExplainLevel.BRIEF) { return output.toString(); } - output.append(detailPrefix).append("hash predicates:\n") - .append(detailPrefix).append("colocate: ").append(isColocate).append(isColocate ? "" : ", reason: " + colocateReason).append("\n"); - for (BinaryPredicate eqJoinPredicate : eqJoinConjuncts) { output.append(detailPrefix).append("equal join conjunct: ").append(eqJoinPredicate.toSql()).append("\n"); } if (!otherJoinConjuncts.isEmpty()) { - output.append(detailPrefix).append("other join predicates: ").append(getExplainString(otherJoinConjuncts)).append("\n"); + output.append(detailPrefix).append("other join predicates: ").append(getExplainString(otherJoinConjuncts)) + .append("\n"); } if (!conjuncts.isEmpty()) { output.append(detailPrefix).append("other predicates: ").append(getExplainString(conjuncts)).append("\n"); @@ -707,8 +710,7 @@ public class HashJoinNode extends PlanNode { output.append(detailPrefix).append("runtime filters: "); output.append(getRuntimeFilterExplainString(true)); } - output.append(detailPrefix).append(String.format( - "cardinality=%s", cardinality)).append("\n"); + output.append(detailPrefix).append(String.format("cardinality=%s", cardinality)).append("\n"); // todo unify in plan node if (outputSlotIds != null) { output.append(detailPrefix).append("output slot ids: "); diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java index 7cc79ab534..6d33efbbeb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java @@ -687,63 +687,43 @@ public class OlapScanNode extends ScanNode { public String getNodeExplainString(String prefix, TExplainLevel detailLevel) { StringBuilder output = new StringBuilder(); - output.append(prefix).append("TABLE: ").append(olapTable.getName()).append("\n"); - + String indexName = olapTable.getIndexNameById(selectedIndexId); + output.append(prefix).append("TABLE: ").append(olapTable.getName()).append("(").append(indexName).append(")"); if (detailLevel == TExplainLevel.BRIEF) { return output.toString(); } + if (isPreAggregation) { + output.append(", PREAGGREGATION: ON"); + } else { + output.append(", PREAGGREGATION: OFF. Reason: ").append(reasonOfPreAggregation); + } + output.append("\n"); if (null != sortColumn) { output.append(prefix).append("SORT COLUMN: ").append(sortColumn).append("\n"); } - if (isPreAggregation) { - output.append(prefix).append("PREAGGREGATION: ON").append("\n"); - } else { - output.append(prefix).append("PREAGGREGATION: OFF. Reason: ").append(reasonOfPreAggregation).append("\n"); - } + if (!conjuncts.isEmpty()) { - output.append(prefix).append("PREDICATES: ").append( - getExplainString(conjuncts)).append("\n"); + output.append(prefix).append("PREDICATES: ").append(getExplainString(conjuncts)).append("\n"); } if (!runtimeFilters.isEmpty()) { output.append(prefix).append("runtime filters: "); output.append(getRuntimeFilterExplainString(false)); } - output.append(prefix).append(String.format( - "partitions=%s/%s", - selectedPartitionNum, - olapTable.getPartitions().size())); - - String indexName = olapTable.getIndexNameById(selectedIndexId); - output.append("\n").append(prefix).append(String.format("rollup: %s", indexName)); - - output.append("\n"); - - output.append(prefix).append(String.format( - "tabletRatio=%s/%s", selectedTabletsNum, totalTabletsNum)); - output.append("\n"); - - // We print up to 10 tablet, and we print "..." if the number is more than 10 - if (scanTabletIds.size() > 10) { - List<Long> firstTenTabletIds = scanTabletIds.subList(0, 10); - output.append(prefix).append(String.format("tabletList=%s ...", Joiner.on(",").join(firstTenTabletIds))); + output.append(prefix).append(String.format("partitions=%s/%s, tablets=%s/%s", selectedPartitionNum, + olapTable.getPartitions().size(), selectedTabletsNum, totalTabletsNum)); + // We print up to 3 tablet, and we print "..." if the number is more than 3 + if (scanTabletIds.size() > 3) { + List<Long> firstTenTabletIds = scanTabletIds.subList(0, 3); + output.append(String.format(", tabletList=%s ...", Joiner.on(",").join(firstTenTabletIds))); } else { - output.append(prefix).append(String.format("tabletList=%s", Joiner.on(",").join(scanTabletIds))); + output.append(String.format(", tabletList=%s", Joiner.on(",").join(scanTabletIds))); } - - output.append("\n"); - - output.append(prefix).append(String.format( - "cardinality=%s", cardinality)); - output.append("\n"); - - output.append(prefix).append(String.format( - "avgRowSize=%s", avgRowSize)); output.append("\n"); - output.append(prefix).append(String.format( - "numNodes=%s", numNodes)); + output.append(prefix).append(String.format("cardinality=%s", cardinality)) + .append(String.format(", avgRowSize=%s", avgRowSize)).append(String.format(", numNodes=%s", numNodes)); output.append("\n"); return output.toString(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/PlanFragment.java b/fe/fe-core/src/main/java/org/apache/doris/planner/PlanFragment.java index c5d01e6249..1847a6b857 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/PlanFragment.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/PlanFragment.java @@ -285,10 +285,9 @@ public class PlanFragment extends TreeNode<PlanFragment> { public String getExplainString(TExplainLevel explainLevel) { StringBuilder str = new StringBuilder(); Preconditions.checkState(dataPartition != null); - str.append(" OUTPUT EXPRS:"); if (CollectionUtils.isNotEmpty(outputExprs)) { - str.append(outputExprs.stream().map(Expr::toSql) - .collect(Collectors.joining(" | "))); + str.append(" OUTPUT EXPRS:"); + str.append(outputExprs.stream().map(Expr::toSql).collect(Collectors.joining(" | "))); } str.append("\n"); str.append(" PARTITION: " + dataPartition.getExplainString(explainLevel) + "\n"); diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/ColocatePlanTest.java b/fe/fe-core/src/test/java/org/apache/doris/planner/ColocatePlanTest.java index 716e3c68be..c944dd6a94 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/planner/ColocatePlanTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/planner/ColocatePlanTest.java @@ -40,7 +40,7 @@ import org.junit.BeforeClass; import org.junit.Test; public class ColocatePlanTest { - private static final String COLOCATE_ENABLE = "colocate: true"; + public static final String COLOCATE_ENABLE = "COLOCATE"; private static String runningDir = "fe/mocked/DemoTest/" + UUID.randomUUID().toString() + "/"; private static ConnectContext ctx; diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/DistributedPlannerTest.java b/fe/fe-core/src/test/java/org/apache/doris/planner/DistributedPlannerTest.java index 94b68b295d..ee5ad95e6a 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/planner/DistributedPlannerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/planner/DistributedPlannerTest.java @@ -137,7 +137,7 @@ public class DistributedPlannerTest { Planner planner = stmtExecutor.planner(); List<PlanFragment> fragments = planner.getFragments(); String plan = planner.getExplainString(fragments, new ExplainOptions(false, false)); - Assert.assertEquals(1, StringUtils.countMatches(plan, "INNER JOIN (BROADCAST)")); + Assert.assertEquals(1, StringUtils.countMatches(plan, "INNER JOIN(BROADCAST)")); sql = "explain select * from db1.tbl1 join [SHUFFLE] db1.tbl2 on tbl1.k1 = tbl2.k3"; stmtExecutor = new StmtExecutor(ctx, sql); @@ -145,7 +145,7 @@ public class DistributedPlannerTest { planner = stmtExecutor.planner(); fragments = planner.getFragments(); plan = planner.getExplainString(fragments, new ExplainOptions(false, false)); - Assert.assertEquals(1, StringUtils.countMatches(plan, "INNER JOIN (PARTITIONED)")); + Assert.assertEquals(1, StringUtils.countMatches(plan, "INNER JOIN(PARTITIONED)")); } @Test @@ -156,7 +156,7 @@ public class DistributedPlannerTest { Planner planner = stmtExecutor.planner(); List<PlanFragment> fragments = planner.getFragments(); String plan = planner.getExplainString(fragments, new ExplainOptions(false, false)); - Assert.assertEquals(1, StringUtils.countMatches(plan, "INNER JOIN (BROADCAST)")); + Assert.assertEquals(1, StringUtils.countMatches(plan, "INNER JOIN(BROADCAST)")); double originThreshold = ctx.getSessionVariable().autoBroadcastJoinThreshold; try { @@ -166,7 +166,7 @@ public class DistributedPlannerTest { planner = stmtExecutor.planner(); fragments = planner.getFragments(); plan = planner.getExplainString(fragments, new ExplainOptions(false, false)); - Assert.assertEquals(1, StringUtils.countMatches(plan, "INNER JOIN (PARTITIONED)")); + Assert.assertEquals(1, StringUtils.countMatches(plan, "INNER JOIN(PARTITIONED)")); } finally { ctx.getSessionVariable().autoBroadcastJoinThreshold = originThreshold; } diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/MaterializedViewFunctionTest.java b/fe/fe-core/src/test/java/org/apache/doris/planner/MaterializedViewFunctionTest.java index 1e34a47b84..68a4eedcd1 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/planner/MaterializedViewFunctionTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/planner/MaterializedViewFunctionTest.java @@ -34,21 +34,21 @@ import java.util.UUID; public class MaterializedViewFunctionTest { private static String baseDir = "fe"; - private static String runningDir = baseDir + "/mocked/MaterializedViewFunctionTest/" - + UUID.randomUUID().toString() + "/"; + private static String runningDir = + baseDir + "/mocked/MaterializedViewFunctionTest/" + UUID.randomUUID().toString() + "/"; private static final String EMPS_TABLE_NAME = "emps"; private static final String EMPS_MV_NAME = "emps_mv"; private static final String HR_DB_NAME = "db1"; - private static final String QUERY_USE_EMPS_MV = "rollup: " + EMPS_MV_NAME; - private static final String QUERY_USE_EMPS = "rollup: " + EMPS_TABLE_NAME; + private static final String QUERY_USE_EMPS_MV = "(" + EMPS_MV_NAME + ")"; + private static final String QUERY_USE_EMPS = "(" + EMPS_TABLE_NAME + ")"; private static final String DEPTS_TABLE_NAME = "depts"; private static final String DEPTS_MV_NAME = "depts_mv"; - private static final String QUERY_USE_DEPTS_MV = "rollup: " + DEPTS_MV_NAME; - private static final String QUERY_USE_DEPTS = "rollup: " + DEPTS_TABLE_NAME; + private static final String QUERY_USE_DEPTS_MV = "(" + DEPTS_MV_NAME + ")"; + private static final String QUERY_USE_DEPTS = "(" + DEPTS_TABLE_NAME + ")"; private static final String USER_TAG_TABLE_NAME = "user_tags"; private static final String USER_TAG_MV_NAME = "user_tags_mv"; - private static final String QUERY_USE_USER_TAG_MV = "rollup: " + USER_TAG_MV_NAME; - private static final String QUERY_USE_USER_TAG = "rollup: " + USER_TAG_TABLE_NAME; + private static final String QUERY_USE_USER_TAG_MV = "(" + USER_TAG_MV_NAME + ")"; + private static final String QUERY_USE_USER_TAG = "(" + USER_TAG_TABLE_NAME + ")"; private static final String TEST_TABLE_NAME = "test_tb"; private static DorisAssert dorisAssert; diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/ResourceTagQueryTest.java b/fe/fe-core/src/test/java/org/apache/doris/planner/ResourceTagQueryTest.java index e43c672c93..047e5a8258 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/planner/ResourceTagQueryTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/planner/ResourceTagQueryTest.java @@ -209,7 +209,7 @@ public class ResourceTagQueryTest { String queryStr = "explain select * from test.tbl1"; String explainString = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext, queryStr); System.out.println(explainString); - Assert.assertTrue(explainString.contains("tabletRatio=30/30")); + Assert.assertTrue(explainString.contains("tablets=30/30")); // set zone1 tag for root String setPropStr2 = "set property for 'root' 'resource_tags.location' = 'zone1';"; @@ -247,7 +247,7 @@ public class ResourceTagQueryTest { queryStr = "explain select * from test.tbl1"; explainString = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext, queryStr); System.out.println(explainString); - Assert.assertTrue(explainString.contains("tabletRatio=30/30")); + Assert.assertTrue(explainString.contains("tablets=30/30")); // for now, 3 backends with tag zone1, 2 with tag default, so table is not stable. ExceptionChecker.expectThrows(UserException.class, () -> tbl.checkReplicaAllocation()); @@ -270,7 +270,7 @@ public class ResourceTagQueryTest { queryStr = "explain select * from test.tbl1"; explainString = UtFrameUtils.getSQLPlanOrErrorMsg(connectContext, queryStr); System.out.println(explainString); - Assert.assertTrue(explainString.contains("tabletRatio=30/30")); + Assert.assertTrue(explainString.contains("tablets=30/30")); // set user exec mem limit String setExecMemLimitStr = "set property for 'root' 'exec_mem_limit' = '1000000';"; diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/TableFunctionPlanTest.java b/fe/fe-core/src/test/java/org/apache/doris/planner/TableFunctionPlanTest.java index 0480791baf..56d020c6c7 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/planner/TableFunctionPlanTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/planner/TableFunctionPlanTest.java @@ -497,7 +497,7 @@ public class TableFunctionPlanTest { public void testLateralViewWithWhere() throws Exception { String sql = "select k1,e1 from db1.table_for_view lateral view explode_split(k3,',') tmp as e1 where k1 in (select k2 from db1.table_for_view);"; String explainString = UtFrameUtils.getSQLPlanOrErrorMsg(ctx, sql, true); - Assert.assertTrue(explainString.contains("join op: LEFT SEMI JOIN (BROADCAST)")); + Assert.assertTrue(explainString.contains("join op: LEFT SEMI JOIN(BROADCAST)")); Assert.assertTrue(explainString.contains("equal join conjunct: `k1` = `k2`")); Assert.assertTrue(!explainString.contains("equal join conjunct: `k2` = `k2`")); } diff --git a/regression-test/data/performance/redundant_conjuncts.out b/regression-test/data/performance/redundant_conjuncts.out index ab2eadd2ea..af59089638 100644 --- a/regression-test/data/performance/redundant_conjuncts.out +++ b/regression-test/data/performance/redundant_conjuncts.out @@ -1,39 +1,27 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !redundant_conjuncts -- PLAN FRAGMENT 0 - OUTPUT EXPRS:`v1` + OUTPUT EXPRS:`v1` PARTITION: HASH_PARTITIONED: `default_cluster:regression_test`.`redundant_conjuncts`.`k1` VRESULT SINK 0:VOlapScanNode - TABLE: redundant_conjuncts - PREAGGREGATION: OFF. Reason: No AggregateInfo + TABLE: redundant_conjuncts(null), PREAGGREGATION: OFF. Reason: No AggregateInfo PREDICATES: `k1` = 1 - partitions=0/1 - rollup: null - tabletRatio=0/0 - tabletList= - cardinality=0 - avgRowSize=8.0 - numNodes=1 + partitions=0/1, tablets=0/0, tabletList= + cardinality=0, avgRowSize=8.0, numNodes=1 -- !redundant_conjuncts_gnerated_by_extract_common_filter -- PLAN FRAGMENT 0 - OUTPUT EXPRS:`v1` + OUTPUT EXPRS:`v1` PARTITION: HASH_PARTITIONED: `default_cluster:regression_test`.`redundant_conjuncts`.`k1` VRESULT SINK 0:VOlapScanNode - TABLE: redundant_conjuncts - PREAGGREGATION: OFF. Reason: No AggregateInfo + TABLE: redundant_conjuncts(null), PREAGGREGATION: OFF. Reason: No AggregateInfo PREDICATES: (`k1` = 1 OR `k1` = 2) - partitions=0/1 - rollup: null - tabletRatio=0/0 - tabletList= - cardinality=0 - avgRowSize=8.0 - numNodes=1 + partitions=0/1, tablets=0/0, tabletList= + cardinality=0, avgRowSize=8.0, numNodes=1 --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
