This is an automated email from the ASF dual-hosted git repository.
hzlu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/helix.git
The following commit(s) were added to refs/heads/master by this push:
new 35ac0ff Enhance ZkHelixVerifier and its related subclass to take a
default wait period (#1328)
35ac0ff is described below
commit 35ac0ff58b14e6ecc51280f88ed7c82c126bd248
Author: kaisun2000 <[email protected]>
AuthorDate: Sat Sep 12 19:48:58 2020 -0700
Enhance ZkHelixVerifier and its related subclass to take a default wait
period (#1328)
We have random sleep() in the test before verifyByPolling().
This causes test failure in different environment. Here we enhance
ZKHelixVerifier
family with an option in builder to add a waiting period. To keep it
backward
compatible, without enable this waiting option, the code would behave
exactly the
same as before.
---
.../BestPossibleExternalViewVerifier.java | 11 +++++----
.../ClusterVerifiers/ClusterLiveNodesVerifier.java | 10 ++++----
.../StrictMatchExternalViewVerifier.java | 12 ++++++----
.../ClusterVerifiers/ZkHelixClusterVerifier.java | 27 +++++++++++++++++++++-
4 files changed, 46 insertions(+), 14 deletions(-)
diff --git
a/helix-core/src/main/java/org/apache/helix/tools/ClusterVerifiers/BestPossibleExternalViewVerifier.java
b/helix-core/src/main/java/org/apache/helix/tools/ClusterVerifiers/BestPossibleExternalViewVerifier.java
index 5e50b6f..744624d 100644
---
a/helix-core/src/main/java/org/apache/helix/tools/ClusterVerifiers/BestPossibleExternalViewVerifier.java
+++
b/helix-core/src/main/java/org/apache/helix/tools/ClusterVerifiers/BestPossibleExternalViewVerifier.java
@@ -98,7 +98,7 @@ public class BestPossibleExternalViewVerifier extends
ZkHelixClusterVerifier {
public BestPossibleExternalViewVerifier(RealmAwareZkClient zkClient, String
clusterName,
Set<String> resources, Map<String, Map<String, String>> errStates,
Set<String> expectLiveInstances) {
- super(zkClient, clusterName);
+ super(zkClient, clusterName, 0);
_errStates = errStates;
_resources = resources;
_expectLiveInstances = expectLiveInstances;
@@ -107,8 +107,8 @@ public class BestPossibleExternalViewVerifier extends
ZkHelixClusterVerifier {
private BestPossibleExternalViewVerifier(RealmAwareZkClient zkClient, String
clusterName,
Map<String, Map<String, String>> errStates, Set<String> resources,
- Set<String> expectLiveInstances) {
- super(zkClient, clusterName);
+ Set<String> expectLiveInstances, int waitPeriodTillVerify) {
+ super(zkClient, clusterName, waitPeriodTillVerify);
// Deep copy data from Builder
_errStates = new HashMap<>();
if (errStates != null) {
@@ -151,7 +151,7 @@ public class BestPossibleExternalViewVerifier extends
ZkHelixClusterVerifier {
return new BestPossibleExternalViewVerifier(
createZkClient(RealmAwareZkClient.RealmMode.SINGLE_REALM,
_realmAwareZkConnectionConfig,
_realmAwareZkClientConfig, _zkAddress), _clusterName,
_errStates, _resources,
- _expectLiveInstances);
+ _expectLiveInstances, _waitPeriodTillVerify);
}
public String getClusterName() {
@@ -202,6 +202,8 @@ public class BestPossibleExternalViewVerifier extends
ZkHelixClusterVerifier {
@Override
public boolean verifyByZkCallback(long timeout) {
+ waitTillVerify();
+
List<ClusterVerifyTrigger> triggers = new
ArrayList<ClusterVerifyTrigger>();
// setup triggers
@@ -319,6 +321,7 @@ public class BestPossibleExternalViewVerifier extends
ZkHelixClusterVerifier {
}
boolean result = verifyExternalView(extView, bpStateMap,
stateModelDef);
+
if (!result) {
if (LOG.isDebugEnabled()) {
LOG.debug("verifyExternalView fails for " + resourceName + "!
ExternalView: " + extView
diff --git
a/helix-core/src/main/java/org/apache/helix/tools/ClusterVerifiers/ClusterLiveNodesVerifier.java
b/helix-core/src/main/java/org/apache/helix/tools/ClusterVerifiers/ClusterLiveNodesVerifier.java
index 50ea8ea..8e28f3e 100644
---
a/helix-core/src/main/java/org/apache/helix/tools/ClusterVerifiers/ClusterLiveNodesVerifier.java
+++
b/helix-core/src/main/java/org/apache/helix/tools/ClusterVerifiers/ClusterLiveNodesVerifier.java
@@ -34,18 +34,20 @@ public class ClusterLiveNodesVerifier extends
ZkHelixClusterVerifier {
@Deprecated
public ClusterLiveNodesVerifier(RealmAwareZkClient zkclient, String
clusterName,
List<String> expectLiveNodes) {
- super(zkclient, clusterName);
+ super(zkclient, clusterName, 0);
_expectLiveNodes = new HashSet<>(expectLiveNodes);
}
private ClusterLiveNodesVerifier(RealmAwareZkClient zkClient, String
clusterName,
- Set<String> expectLiveNodes) {
- super(zkClient, clusterName);
+ Set<String> expectLiveNodes, int waitPeriodTillVerify) {
+ super(zkClient, clusterName, waitPeriodTillVerify);
_expectLiveNodes = expectLiveNodes == null ? new HashSet<>() : new
HashSet<>(expectLiveNodes);
}
@Override
public boolean verifyByZkCallback(long timeout) {
+ waitTillVerify();
+
List<ClusterVerifyTrigger> triggers = new
ArrayList<ClusterVerifyTrigger>();
triggers.add(new ClusterVerifyTrigger(_keyBuilder.liveInstances(), false,
true, true));
@@ -81,7 +83,7 @@ public class ClusterLiveNodesVerifier extends
ZkHelixClusterVerifier {
validate();
return new ClusterLiveNodesVerifier(
createZkClient(RealmAwareZkClient.RealmMode.SINGLE_REALM,
_realmAwareZkConnectionConfig,
- _realmAwareZkClientConfig, _zkAddress), _clusterName,
_expectLiveNodes);
+ _realmAwareZkClientConfig, _zkAddress), _clusterName,
_expectLiveNodes, _waitPeriodTillVerify);
}
}
}
diff --git
a/helix-core/src/main/java/org/apache/helix/tools/ClusterVerifiers/StrictMatchExternalViewVerifier.java
b/helix-core/src/main/java/org/apache/helix/tools/ClusterVerifiers/StrictMatchExternalViewVerifier.java
index 9f91160..1073dad 100644
---
a/helix-core/src/main/java/org/apache/helix/tools/ClusterVerifiers/StrictMatchExternalViewVerifier.java
+++
b/helix-core/src/main/java/org/apache/helix/tools/ClusterVerifiers/StrictMatchExternalViewVerifier.java
@@ -66,7 +66,7 @@ public class StrictMatchExternalViewVerifier extends
ZkHelixClusterVerifier {
@Deprecated
public StrictMatchExternalViewVerifier(RealmAwareZkClient zkClient, String
clusterName,
Set<String> resources, Set<String> expectLiveInstances) {
- this(zkClient, clusterName, resources, expectLiveInstances, false);
+ this(zkClient, clusterName, resources, expectLiveInstances, false, 0);
}
@Deprecated
@@ -79,8 +79,8 @@ public class StrictMatchExternalViewVerifier extends
ZkHelixClusterVerifier {
}
private StrictMatchExternalViewVerifier(RealmAwareZkClient zkClient, String
clusterName,
- Set<String> resources, Set<String> expectLiveInstances, boolean
isDeactivatedNodeAware) {
- super(zkClient, clusterName);
+ Set<String> resources, Set<String> expectLiveInstances, boolean
isDeactivatedNodeAware, int waitPeriodTillVerify) {
+ super(zkClient, clusterName, waitPeriodTillVerify);
_resources = resources == null ? new HashSet<>() : new
HashSet<>(resources);
_expectLiveInstances =
expectLiveInstances == null ? new HashSet<>() : new
HashSet<>(expectLiveInstances);
@@ -102,7 +102,7 @@ public class StrictMatchExternalViewVerifier extends
ZkHelixClusterVerifier {
if (_zkClient != null) {
return new StrictMatchExternalViewVerifier(_zkClient, _clusterName,
_resources,
- _expectLiveInstances, _isDeactivatedNodeAware);
+ _expectLiveInstances, _isDeactivatedNodeAware,
_waitPeriodTillVerify);
}
if (_realmAwareZkConnectionConfig == null || _realmAwareZkClientConfig
== null) {
@@ -115,7 +115,7 @@ public class StrictMatchExternalViewVerifier extends
ZkHelixClusterVerifier {
return new StrictMatchExternalViewVerifier(
createZkClient(RealmAwareZkClient.RealmMode.SINGLE_REALM,
_realmAwareZkConnectionConfig,
_realmAwareZkClientConfig, _zkAddress), _clusterName, _resources,
- _expectLiveInstances, _isDeactivatedNodeAware);
+ _expectLiveInstances, _isDeactivatedNodeAware,
_waitPeriodTillVerify);
}
public Builder(String clusterName) {
@@ -181,6 +181,8 @@ public class StrictMatchExternalViewVerifier extends
ZkHelixClusterVerifier {
@Override
public boolean verifyByZkCallback(long timeout) {
+ waitTillVerify();
+
List<ClusterVerifyTrigger> triggers = new
ArrayList<ClusterVerifyTrigger>();
// setup triggers
diff --git
a/helix-core/src/main/java/org/apache/helix/tools/ClusterVerifiers/ZkHelixClusterVerifier.java
b/helix-core/src/main/java/org/apache/helix/tools/ClusterVerifiers/ZkHelixClusterVerifier.java
index fe24b3b..734065f 100644
---
a/helix-core/src/main/java/org/apache/helix/tools/ClusterVerifiers/ZkHelixClusterVerifier.java
+++
b/helix-core/src/main/java/org/apache/helix/tools/ClusterVerifiers/ZkHelixClusterVerifier.java
@@ -58,6 +58,7 @@ public abstract class ZkHelixClusterVerifier
protected final HelixDataAccessor _accessor;
protected final PropertyKey.Builder _keyBuilder;
private CountDownLatch _countdown;
+ protected final int _waitPeriodTillVerify;
private ExecutorService _verifyTaskThreadPool =
Executors.newSingleThreadExecutor(r -> new Thread(r,
"ZkHelixClusterVerifier-verify_thread"));
@@ -93,7 +94,7 @@ public abstract class ZkHelixClusterVerifier
}
}
- protected ZkHelixClusterVerifier(RealmAwareZkClient zkClient, String
clusterName) {
+ protected ZkHelixClusterVerifier(RealmAwareZkClient zkClient, String
clusterName, int waitPeriodTillVerify) {
if (zkClient == null || clusterName == null) {
throw new IllegalArgumentException("requires zkClient|clusterName");
}
@@ -102,6 +103,7 @@ public abstract class ZkHelixClusterVerifier
_clusterName = clusterName;
_accessor = new ZKHelixDataAccessor(clusterName, new
ZkBaseDataAccessor<>(_zkClient));
_keyBuilder = _accessor.keyBuilder();
+ _waitPeriodTillVerify = waitPeriodTillVerify;
}
@Deprecated
@@ -136,6 +138,7 @@ public abstract class ZkHelixClusterVerifier
_clusterName = clusterName;
_accessor = new ZKHelixDataAccessor(clusterName, new
ZkBaseDataAccessor<>(_zkClient));
_keyBuilder = _accessor.keyBuilder();
+ _waitPeriodTillVerify = 0;
}
/**
@@ -178,6 +181,15 @@ public abstract class ZkHelixClusterVerifier
return verifyByZkCallback(DEFAULT_TIMEOUT);
}
+ protected void waitTillVerify() {
+ try {
+ if (_waitPeriodTillVerify != 0) {
+ Thread.sleep(_waitPeriodTillVerify);
+ }
+ } catch (InterruptedException e) {
+ LOG.error("cooldown in verifyByPolling interrupted");
+ }
+ }
/**
* Verify the cluster by periodically polling the cluster status and verify.
* The method will be blocked at most {@code timeout}.
@@ -187,6 +199,8 @@ public abstract class ZkHelixClusterVerifier
* @return
*/
public boolean verifyByPolling(long timeout, long period) {
+ waitTillVerify();
+
try {
long start = System.currentTimeMillis();
boolean success;
@@ -197,6 +211,7 @@ public abstract class ZkHelixClusterVerifier
}
TimeUnit.MILLISECONDS.sleep(period);
} while ((System.currentTimeMillis() - start) <= timeout);
+ LOG.error("verifier timeout out with timeout {}", timeout);
} catch (Exception e) {
LOG.error("Exception in verifier", e);
}
@@ -234,6 +249,9 @@ public abstract class ZkHelixClusterVerifier
if (!success) {
// make a final try if timeout
success = verifyState();
+ if (!success) {
+ LOG.error("verifyByCallback failed due to timeout {}", timeout);
+ }
}
}
} catch (Exception e) {
@@ -319,6 +337,8 @@ public abstract class ZkHelixClusterVerifier
}
protected abstract static class Builder<B extends Builder<B>> extends
GenericZkHelixApiBuilder<B> {
+ protected int _waitPeriodTillVerify;
+
public Builder() {
// Note: ZkHelixClusterVerifier is a single-realm API, so RealmMode is
assumed to be
// SINGLE-REALM
@@ -335,6 +355,11 @@ public abstract class ZkHelixClusterVerifier
return setZkAddress(zkAddress);
}
+ public B setWaitTillVerify(int waitPeriod) {
+ _waitPeriodTillVerify = waitPeriod;
+ return (B) this;
+ }
+
public String getClusterName() {
if (_realmAwareZkConnectionConfig != null && (
_realmAwareZkConnectionConfig.getZkRealmShardingKey() != null
