This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 32273a7a9b [improvement](backend)Optimized error messages for
insufficient replication (#19211)
32273a7a9b is described below
commit 32273a7a9b2714ae4949dd92a13cf4caec518527
Author: yongkang.zhong <[email protected]>
AuthorDate: Sun May 7 20:45:21 2023 +0800
[improvement](backend)Optimized error messages for insufficient replication
(#19211)
optimized the error message for creating insufficient table replications
---
.../org/apache/doris/system/SystemInfoService.java | 47 ++++++++++++++++------
.../org/apache/doris/catalog/CreateTableTest.java | 6 ++-
.../apache/doris/catalog/ModifyBackendTest.java | 6 ++-
3 files changed, 43 insertions(+), 16 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/system/SystemInfoService.java
b/fe/fe-core/src/main/java/org/apache/doris/system/SystemInfoService.java
index d2e8044964..76a664bb28 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/system/SystemInfoService.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/system/SystemInfoService.java
@@ -38,6 +38,7 @@ import org.apache.doris.system.Backend.BackendState;
import org.apache.doris.thrift.TStatusCode;
import org.apache.doris.thrift.TStorageMedium;
+import com.google.common.base.Joiner;
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
import com.google.common.collect.ImmutableMap;
@@ -969,26 +970,48 @@ public class SystemInfoService {
public Map<Tag, List<Long>> selectBackendIdsForReplicaCreation(
ReplicaAllocation replicaAlloc, String clusterName, TStorageMedium
storageMedium)
throws DdlException {
+ Map<Long, Backend> copiedBackends = Maps.newHashMap(idToBackendRef);
Map<Tag, List<Long>> chosenBackendIds = Maps.newHashMap();
Map<Tag, Short> allocMap = replicaAlloc.getAllocMap();
short totalReplicaNum = 0;
- for (Map.Entry<Tag, Short> entry : allocMap.entrySet()) {
- BeSelectionPolicy.Builder builder = new
BeSelectionPolicy.Builder().setCluster(clusterName)
-
.needScheduleAvailable().needCheckDiskUsage().addTags(Sets.newHashSet(entry.getKey()))
- .setStorageMedium(storageMedium);
- if (FeConstants.runningUnitTest ||
Config.allow_replica_on_same_host) {
- builder.allowOnSameHost();
+ int aliveBackendNum = (int)
copiedBackends.values().stream().filter(Backend::isAlive).count();
+ if (aliveBackendNum < replicaAlloc.getTotalReplicaNum()) {
+ throw new DdlException("replication num should be less than the
number of available backends. "
+ + "replication num is " + replicaAlloc.getTotalReplicaNum()
+ + ", available backend num is " + aliveBackendNum);
+ } else {
+ List<String> failedEntries = Lists.newArrayList();
+
+ for (Map.Entry<Tag, Short> entry : allocMap.entrySet()) {
+ BeSelectionPolicy.Builder builder = new
BeSelectionPolicy.Builder().setCluster(clusterName)
+
.needScheduleAvailable().needCheckDiskUsage().addTags(Sets.newHashSet(entry.getKey()))
+ .setStorageMedium(storageMedium);
+ if (FeConstants.runningUnitTest ||
Config.allow_replica_on_same_host) {
+ builder.allowOnSameHost();
+ }
+
+ BeSelectionPolicy policy = builder.build();
+ List<Long> beIds = selectBackendIdsByPolicy(policy,
entry.getValue());
+ if (beIds.isEmpty()) {
+ LOG.error("failed backend(s) for policy:" + policy);
+ String errorReplication = "replication tag: " +
entry.getKey()
+ + ", replication num: " + entry.getValue()
+ + ", storage medium: " + storageMedium;
+ failedEntries.add(errorReplication);
+ } else {
+ chosenBackendIds.put(entry.getKey(), beIds);
+ totalReplicaNum += beIds.size();
+ }
}
- BeSelectionPolicy policy = builder.build();
- List<Long> beIds = selectBackendIdsByPolicy(policy,
entry.getValue());
- if (beIds.isEmpty()) {
- throw new DdlException("Failed to find " + entry.getValue() +
" backend(s) for policy: " + policy);
+ if (!failedEntries.isEmpty()) {
+ String failedMsg = Joiner.on("\n").join(failedEntries);
+ throw new DdlException("Failed to find enough backend, please
check the replication num,"
+ + "replication tag and storage medium.\n" + "Create
failed replications:\n" + failedMsg);
}
- chosenBackendIds.put(entry.getKey(), beIds);
- totalReplicaNum += beIds.size();
}
+
Preconditions.checkState(totalReplicaNum ==
replicaAlloc.getTotalReplicaNum());
return chosenBackendIds;
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableTest.java
b/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableTest.java
index fef93b6682..5037327005 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableTest.java
@@ -271,7 +271,7 @@ public class CreateTableTest {
+ "properties('replication_num' = '1', 'short_key' =
'4');"));
ExceptionChecker
- .expectThrowsWithMsg(DdlException.class, "Failed to find 3
backend(s) for policy",
+ .expectThrowsWithMsg(DdlException.class, "replication num
should be less than the number of available backends. replication num is 3,
available backend num is 1",
() -> createTable("create table test.atbl5\n" + "(k1
int, k2 int, k3 int)\n"
+ "duplicate key(k1, k2, k3)\n" + "distributed
by hash(k1) buckets 1\n"
+ "properties('replication_num' = '3');"));
@@ -288,7 +288,9 @@ public class CreateTableTest {
ConfigBase.setMutableConfig("disable_storage_medium_check", "false");
ExceptionChecker
- .expectThrowsWithMsg(DdlException.class, " Failed to find 1
backend(s) for policy:",
+ .expectThrowsWithMsg(DdlException.class, "Failed to find
enough backend, please check the replication num,replication tag and storage
medium.\n"
+ + "Create failed replications:\n"
+ + "replication tag: {\"location\" :
\"default\"}, replication num: 1, storage medium: SSD",
() -> createTable("create table test.tb7(key1 int,
key2 varchar(10)) distributed by hash(key1) \n"
+ "buckets 1 properties('replication_num' =
'1', 'storage_medium' = 'ssd');"));
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/catalog/ModifyBackendTest.java
b/fe/fe-core/src/test/java/org/apache/doris/catalog/ModifyBackendTest.java
index 205bf84634..a457cf57ec 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/catalog/ModifyBackendTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/ModifyBackendTest.java
@@ -80,7 +80,9 @@ public class ModifyBackendTest {
String createStr = "create table test.tbl1(\n" + "k1 int\n" + ")
distributed by hash(k1)\n"
+ "buckets 3 properties(\n" + "\"replication_num\" = \"1\"\n"
+ ");";
CreateTableStmt createStmt = (CreateTableStmt)
UtFrameUtils.parseAndAnalyzeStmt(createStr, connectContext);
- ExceptionChecker.expectThrowsWithMsg(DdlException.class, "Failed to
find 1 backend(s) for policy:",
+ ExceptionChecker.expectThrowsWithMsg(DdlException.class, "Failed to
find enough backend, please check the replication num,replication tag and
storage medium.\n"
+ + "Create failed replications:\n"
+ + "replication tag: {\"location\" : \"default\"},
replication num: 1, storage medium: HDD",
() -> DdlExecutor.execute(Env.getCurrentEnv(), createStmt));
createStr = "create table test.tbl1(\n" + "k1 int\n" + ") distributed
by hash(k1)\n" + "buckets 3 properties(\n"
@@ -100,7 +102,7 @@ public class ModifyBackendTest {
+ ");";
CreateTableStmt createStmt3 = (CreateTableStmt)
UtFrameUtils.parseAndAnalyzeStmt(createStr, connectContext);
//partition create failed, because there is no BE with "default" tag
- ExceptionChecker.expectThrowsWithMsg(DdlException.class, "Failed to
find 3 backend(s) for policy",
+ ExceptionChecker.expectThrowsWithMsg(DdlException.class, "replication
num should be less than the number of available backends. replication num is 3,
available backend num is 1",
() -> DdlExecutor.execute(Env.getCurrentEnv(), createStmt3));
Database db =
Env.getCurrentInternalCatalog().getDbNullable("default_cluster:test");
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
