This is an automated email from the ASF dual-hosted git repository.
wangbo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/master by this push:
new b6abcbd Fix 5243 Skip repair replica not in colocate group. (#5250)
b6abcbd is described below
commit b6abcbdd35c4f2a223651601262490d639c24175
Author: Lijia Liu <[email protected]>
AuthorDate: Thu Feb 4 12:04:19 2021 +0800
Fix 5243 Skip repair replica not in colocate group. (#5250)
* Fix 5243 Skip repair replica not in colocate group.
* Remove useless parameter& Add UT
* Update fe/fe-core/src/main/java/org/apache/doris/catalog/Tablet.java
---
.../main/java/org/apache/doris/catalog/Tablet.java | 16 ++++---
.../apache/doris/clone/ColocateTableBalancer.java | 2 +-
.../org/apache/doris/clone/TabletScheduler.java | 1 -
.../java/org/apache/doris/catalog/TabletTest.java | 54 ++++++++++++++++++++++
.../doris/clone/ColocateTableBalancerTest.java | 8 ++--
5 files changed, 68 insertions(+), 13 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Tablet.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/Tablet.java
index 8c7b0ce..361d6ec 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Tablet.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Tablet.java
@@ -533,7 +533,7 @@ public class Tablet extends MetaObject implements Writable {
* tablet replicas: 1,2,4,5
*
* 2. Version incomplete:
- * backend matched, but some replica's version is incomplete
+ * backend matched, but some replica(in backends set)'s version is
incomplete
*
* 3. Redundant:
* backends set: 1,2,3
@@ -542,19 +542,21 @@ public class Tablet extends MetaObject implements
Writable {
* No need to check if backend is available. We consider all backends in
'backendsSet' are available,
* If not, unavailable backends will be relocated by CalocateTableBalancer
first.
*/
- public TabletStatus getColocateHealthStatus(long visibleVersion, long
visibleVersionHash,
- int replicationNum, Set<Long> backendsSet) {
+ public TabletStatus getColocateHealthStatus(long visibleVersion, int
replicationNum, Set<Long> backendsSet) {
// 1. check if replicas' backends are mismatch
Set<Long> replicaBackendIds = getBackendIds();
- for (Long backendId : backendsSet) {
- if (!replicaBackendIds.contains(backendId)) {
- return TabletStatus.COLOCATE_MISMATCH;
- }
+ if (!replicaBackendIds.containsAll(backendsSet)) {
+ return TabletStatus.COLOCATE_MISMATCH;
}
// 2. check version completeness
for (Replica replica : replicas) {
+ if (!backendsSet.contains(replica.getBackendId())) {
+ // We don't care about replicas that are not in backendsSet.
+ // eg: replicaBackendIds=(1,2,3,4); backendsSet=(1,2,3), then
replica 4 should be skipped here and then goto ```COLOCATE_REDUNDANT``` in step
3
+ continue;
+ }
if (replica.getLastFailedVersion() > 0 || replica.getVersion() <
visibleVersion) {
// this replica is alive but version incomplete
return TabletStatus.VERSION_INCOMPLETE;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/clone/ColocateTableBalancer.java
b/fe/fe-core/src/main/java/org/apache/doris/clone/ColocateTableBalancer.java
index e7af2d2..5f7956d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/clone/ColocateTableBalancer.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/clone/ColocateTableBalancer.java
@@ -213,7 +213,7 @@ public class ColocateTableBalancer extends MasterDaemon {
Set<Long> bucketsSeq =
backendBucketsSeq.get(idx);
Preconditions.checkState(bucketsSeq.size() ==
replicationNum, bucketsSeq.size() + " vs. " + replicationNum);
Tablet tablet = index.getTablet(tabletId);
- TabletStatus st =
tablet.getColocateHealthStatus(visibleVersion, visibleVersionHash,
replicationNum, bucketsSeq);
+ TabletStatus st =
tablet.getColocateHealthStatus(visibleVersion, replicationNum, bucketsSeq);
if (st != TabletStatus.HEALTHY) {
isGroupStable = false;
LOG.debug("get unhealthy tablet {} in
colocate table. status: {}", tablet.getId(), st);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/clone/TabletScheduler.java
b/fe/fe-core/src/main/java/org/apache/doris/clone/TabletScheduler.java
index 58453ea..0794296 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/clone/TabletScheduler.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/clone/TabletScheduler.java
@@ -496,7 +496,6 @@ public class TabletScheduler extends MasterDaemon {
Set<Long> backendsSet =
colocateTableIndex.getTabletBackendsByGroup(groupId, tabletOrderIdx);
TabletStatus st = tablet.getColocateHealthStatus(
partition.getVisibleVersion(),
- partition.getVisibleVersionHash(),
tbl.getPartitionInfo().getReplicationNum(partition.getId()),
backendsSet);
statusPair = Pair.create(st, Priority.HIGH);
diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/TabletTest.java
b/fe/fe-core/src/test/java/org/apache/doris/catalog/TabletTest.java
index 2e2285a..0507b96 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/catalog/TabletTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/TabletTest.java
@@ -19,10 +19,16 @@ package org.apache.doris.catalog;
import mockit.Expectations;
import mockit.Mocked;
+
import org.apache.doris.catalog.Replica.ReplicaState;
import org.apache.doris.common.FeConstants;
+import org.apache.doris.common.Pair;
+import org.apache.doris.system.Backend;
import org.apache.doris.thrift.TStorageMedium;
+import com.google.common.collect.Sets;
+
+import org.apache.arrow.flatbuf.Bool;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -32,6 +38,8 @@ import java.io.DataOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicInteger;
public class TabletTest {
@@ -154,4 +162,50 @@ public class TabletTest {
file.delete();
}
+ /**
+ * check the tablet's Tablet.TabletStatus, the right location is [1 2 3]
+ * @param backendId2ReplicaIsBad beId -> if replica is a bad replica
+ */
+ @SafeVarargs
+ private final void testTabletColocateHealthStatus0(Tablet.TabletStatus
exceptedTabletStatus, Pair<Long, Boolean>... backendId2ReplicaIsBad) {
+ Tablet tablet = new Tablet(1);
+ int replicaId = 1;
+ for (Pair<Long, Boolean> pair : backendId2ReplicaIsBad) {
+ long versionAndSuccessVersion = 100L;
+ long lastFailVersion = -1L;
+ if (pair.second) {
+ versionAndSuccessVersion = 99L;
+ lastFailVersion = 100L;
+ }
+ tablet.addReplica(new Replica(replicaId++, pair.first,
versionAndSuccessVersion, 0L, 0, 200000L, 3000L, ReplicaState.NORMAL,
lastFailVersion, 0, versionAndSuccessVersion, 0));
+ }
+ Assert.assertEquals(tablet.getColocateHealthStatus(100L, 3,
Sets.newHashSet(1L, 2L, 3L)), exceptedTabletStatus);
+ }
+
+ @Test
+ public void testTabletColocateHealthStatus() {
+ // [1 2 4]
+ testTabletColocateHealthStatus0(
+ Tablet.TabletStatus.COLOCATE_MISMATCH,
+ Pair.create(1L, false), Pair.create(2L, false),
Pair.create(4L, false)
+ );
+
+ // [1 2 3(bad)]
+ testTabletColocateHealthStatus0(
+ Tablet.TabletStatus.VERSION_INCOMPLETE,
+ Pair.create(1L, false), Pair.create(2L, false),
Pair.create(3L, true)
+ );
+
+ // 1 2 3 4(good)
+ testTabletColocateHealthStatus0(
+ Tablet.TabletStatus.COLOCATE_REDUNDANT,
+ Pair.create(1L, false), Pair.create(2L, false),
Pair.create(3L, false), Pair.create(4L, false)
+ );
+
+ // [1 2 3 4(bad)]
+ testTabletColocateHealthStatus0(
+ Tablet.TabletStatus.COLOCATE_REDUNDANT,
+ Pair.create(1L, false), Pair.create(2L, false),
Pair.create(3L, false), Pair.create(4L, true)
+ );
+ }
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/clone/ColocateTableBalancerTest.java
b/fe/fe-core/src/test/java/org/apache/doris/clone/ColocateTableBalancerTest.java
index ff4ddd8..0aae3d0 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/clone/ColocateTableBalancerTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/clone/ColocateTableBalancerTest.java
@@ -17,8 +17,10 @@
package org.apache.doris.clone;
-import com.google.common.collect.Sets;
import mockit.Delegate;
+import mockit.Expectations;
+import mockit.Mocked;
+
import org.apache.doris.catalog.ColocateGroupSchema;
import org.apache.doris.catalog.ColocateTableIndex;
import org.apache.doris.catalog.ColocateTableIndex.GroupId;
@@ -32,6 +34,7 @@ import org.apache.doris.system.SystemInfoService;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
import org.junit.Assert;
import org.junit.Before;
@@ -42,9 +45,6 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
-import mockit.Expectations;
-import mockit.Mocked;
-
public class ColocateTableBalancerTest {
private ColocateTableBalancer balancer =
ColocateTableBalancer.getInstance();
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
