This is an automated email from the ASF dual-hosted git repository. xyuanlu pushed a commit to branch metaclient in repository https://gitbox.apache.org/repos/asf/helix.git
commit 80722de32249a344c1029db73de6edca6385f968 Author: xyuanlu <[email protected]> AuthorDate: Fri Mar 10 22:26:01 2023 -0800 Add retry policy class and config for operation retry for MetaClient (#2385) --- .../metaclient/constants/MetaClientConstants.java | 5 ++ .../metaclient/factories/MetaClientConfig.java | 71 ++++++++++++++++++---- .../helix/metaclient/impl/zk/ZkMetaClient.java | 5 +- .../impl/zk/factory/ZkMetaClientConfig.java | 18 +++--- .../policy/ExponentialBackoffReconnectPolicy.java | 53 ++++++++++++++++ .../MetaClientReconnectPolicy.java} | 27 ++++---- .../NoRetryReconnectPolicy.java} | 29 ++++----- 7 files changed, 155 insertions(+), 53 deletions(-) diff --git a/meta-client/src/main/java/org/apache/helix/metaclient/constants/MetaClientConstants.java b/meta-client/src/main/java/org/apache/helix/metaclient/constants/MetaClientConstants.java index e38b1f6a9..2c554189b 100644 --- a/meta-client/src/main/java/org/apache/helix/metaclient/constants/MetaClientConstants.java +++ b/meta-client/src/main/java/org/apache/helix/metaclient/constants/MetaClientConstants.java @@ -36,6 +36,11 @@ public final class MetaClientConstants { // new session will be established. public static final int DEFAULT_SESSION_TIMEOUT_MS = 30 * 1000; + // Max backoff window for exponential reconnect back off policy. by default is 30 seconds. + public static final long DEFAULT_MAX_EXP_BACKOFF_RETRY_INTERVAL_MS = 30 * 1000; + // Initial backoff window for exponential reconnect back off policy. by default is 500 ms. + public static final long DEFAULT_INIT_EXP_BACKOFF_RETRY_INTERVAL_MS = 500; + //public static final long DEFAULT_MAX_LINEAR_BACKOFF_RETRY_WINDOW_MS = 5*1000; } diff --git a/meta-client/src/main/java/org/apache/helix/metaclient/factories/MetaClientConfig.java b/meta-client/src/main/java/org/apache/helix/metaclient/factories/MetaClientConfig.java index 11f16e5b5..081727741 100644 --- a/meta-client/src/main/java/org/apache/helix/metaclient/factories/MetaClientConfig.java +++ b/meta-client/src/main/java/org/apache/helix/metaclient/factories/MetaClientConfig.java @@ -19,6 +19,8 @@ package org.apache.helix.metaclient.factories; * under the License. */ +import org.apache.helix.metaclient.policy.ExponentialBackoffReconnectPolicy; +import org.apache.helix.metaclient.policy.MetaClientReconnectPolicy; import org.apache.helix.metaclient.constants.MetaClientConstants; public class MetaClientConfig { @@ -32,10 +34,18 @@ public class MetaClientConfig { // Wait for init timeout time until connection is initiated private final long _connectionInitTimeoutInMillis; + // Operation failed because of connection lost will be auto retried if connection has recovered + // within timeout time. + private final long _operationRetryTimeoutInMillis; + // When a client becomes partitioned from the metadata service for more than session timeout, // new session will be established when reconnect. private final long _sessionTimeoutInMillis; + // Policy to define client re-establish connection behavior when the connection to underlying + // metadata store is expired. + private final MetaClientReconnectPolicy _metaClientReconnectPolicy; + private final boolean _enableAuth; private final StoreType _storeType; @@ -47,6 +57,10 @@ public class MetaClientConfig { return _connectionInitTimeoutInMillis; } + public long getOperationRetryTimeoutInMillis() { + return _operationRetryTimeoutInMillis; + } + public boolean isAuthEnabled() { return _enableAuth; } @@ -59,21 +73,22 @@ public class MetaClientConfig { return _sessionTimeoutInMillis; } + public MetaClientReconnectPolicy getMetaClientReconnectPolicy() { + return _metaClientReconnectPolicy; + } + // TODO: More options to add later // private boolean _autoReRegistWatcher; // re-register one time watcher when set to true // private boolean _resetWatchWhenReConnect; // re-register previous existing watcher when reconnect - // - // public enum RetryProtocol { - // NO_RETRY, EXP_BACK_OFF, CONST_RETRY_INTERVAL - // } - // private RetryProtocol _retryProtocol; - protected MetaClientConfig(String connectionAddress, long connectionInitTimeoutInMillis, - long sessionTimeoutInMillis, boolean enableAuth, StoreType storeType) { + long operationRetryTimeoutInMillis, long sessionTimeoutInMillis, + MetaClientReconnectPolicy metaClientReconnectPolicy, boolean enableAuth, StoreType storeType) { _connectionAddress = connectionAddress; _connectionInitTimeoutInMillis = connectionInitTimeoutInMillis; + _operationRetryTimeoutInMillis = operationRetryTimeoutInMillis; _sessionTimeoutInMillis = sessionTimeoutInMillis; + _metaClientReconnectPolicy = metaClientReconnectPolicy; _enableAuth = enableAuth; _storeType = storeType; } @@ -83,17 +98,16 @@ public class MetaClientConfig { protected long _connectionInitTimeoutInMillis; protected long _sessionTimeoutInMillis; - // protected long _operationRetryTimeout; - // protected RetryProtocol _retryProtocol; + protected long _operationRetryTimeout; protected boolean _enableAuth; protected StoreType _storeType; + protected MetaClientReconnectPolicy _metaClientReconnectPolicy; public MetaClientConfig build() { validate(); return new MetaClientConfig(_connectionAddress, _connectionInitTimeoutInMillis, - _sessionTimeoutInMillis, - _enableAuth, _storeType); + _operationRetryTimeout, _sessionTimeoutInMillis, _metaClientReconnectPolicy, _enableAuth, _storeType); } public MetaClientConfigBuilder() { @@ -114,7 +128,7 @@ public class MetaClientConfig { } /** - * Set timeout in mm for connection initialization timeout + * Set timeout in ms for connection initialization timeout * @param timeout * @return */ @@ -123,6 +137,27 @@ public class MetaClientConfig { return self(); } + /** + * Set timeout in ms for operation retry timeout + * @param timeout + * @return + */ + public B setOperationRetryTimeoutInMillis(long timeout) { + _operationRetryTimeout = timeout; + return self(); + } + + /** + * Set reconnect policy when connection is lost or expired. By default is + * ExponentialBackoffReconnectPolicy + * @param reconnectPolicy an instance of type MetaClientReconnectPolicy + * @return + */ + public B setMetaClientReconnectPolicy(MetaClientReconnectPolicy reconnectPolicy) { + _metaClientReconnectPolicy = reconnectPolicy; + return self(); + } + /** * Set timeout in mm for session timeout. When a client becomes partitioned from the metadata * service for more than session timeout, new session will be established. @@ -145,6 +180,18 @@ public class MetaClientConfig { } protected void validate() { + if (_metaClientReconnectPolicy == null) { + _metaClientReconnectPolicy = new ExponentialBackoffReconnectPolicy(); + } + + // check if reconnect policy and retry policy conflict. + if (_metaClientReconnectPolicy.getPolicyName() + == MetaClientReconnectPolicy.RetryPolicyName.NO_RETRY && _operationRetryTimeout > 0) { + throw new IllegalArgumentException( + "MetaClientConfig.Builder: Incompatible operationRetryTimeout with NO_RETRY ReconnectPolicy."); + } + // TODO: check operationRetryTimeout should be less than ReconnectPolicy timeout. + if (_storeType == null || _connectionAddress == null) { throw new IllegalArgumentException( "MetaClientConfig.Builder: store type or connection string is null"); diff --git a/meta-client/src/main/java/org/apache/helix/metaclient/impl/zk/ZkMetaClient.java b/meta-client/src/main/java/org/apache/helix/metaclient/impl/zk/ZkMetaClient.java index 7934fee8a..d3542e1fa 100644 --- a/meta-client/src/main/java/org/apache/helix/metaclient/impl/zk/ZkMetaClient.java +++ b/meta-client/src/main/java/org/apache/helix/metaclient/impl/zk/ZkMetaClient.java @@ -63,9 +63,12 @@ public class ZkMetaClient<T> implements MetaClientInterface<T>, AutoCloseable { public ZkMetaClient(ZkMetaClientConfig config) { _connectionTimeout = (int) config.getConnectionInitTimeoutInMillis(); + // TODO: Right new ZkClient reconnect using exp backoff with fixed max backoff interval. We should + // 1. Allow user to config max backoff interval (next PR) + // 2. Allow user to config reconnect policy (future PR) _zkClient = new ZkClient( new ZkConnection(config.getConnectionAddress(), (int) config.getSessionTimeoutInMillis()), - _connectionTimeout, -1 /*operationRetryTimeout*/, config.getZkSerializer(), + _connectionTimeout, config.getOperationRetryTimeoutInMillis(), config.getZkSerializer(), config.getMonitorType(), config.getMonitorKey(), config.getMonitorInstanceName(), config.getMonitorRootPathOnly(), false); } diff --git a/meta-client/src/main/java/org/apache/helix/metaclient/impl/zk/factory/ZkMetaClientConfig.java b/meta-client/src/main/java/org/apache/helix/metaclient/impl/zk/factory/ZkMetaClientConfig.java index d9292f846..3f21fa7df 100644 --- a/meta-client/src/main/java/org/apache/helix/metaclient/impl/zk/factory/ZkMetaClientConfig.java +++ b/meta-client/src/main/java/org/apache/helix/metaclient/impl/zk/factory/ZkMetaClientConfig.java @@ -19,6 +19,7 @@ package org.apache.helix.metaclient.impl.zk.factory; * under the License. */ +import org.apache.helix.metaclient.policy.MetaClientReconnectPolicy; import org.apache.helix.metaclient.factories.MetaClientConfig; import org.apache.helix.zookeeper.zkclient.serialize.BasicZkSerializer; import org.apache.helix.zookeeper.zkclient.serialize.PathBasedZkSerializer; @@ -60,17 +61,17 @@ public class ZkMetaClientConfig extends MetaClientConfig { } protected ZkMetaClientConfig(String connectionAddress, long connectionInitTimeoutInMillis, - long sessionTimeoutInMillis, boolean enableAuth, StoreType storeType, String monitorType, - String monitorKey, String monitorInstanceName, boolean monitorRootPathOnly, - PathBasedZkSerializer zkSerializer) { - super(connectionAddress, connectionInitTimeoutInMillis, sessionTimeoutInMillis, enableAuth, - storeType); + long operationRetryTimeoutInMillis, long sessionTimeoutInMillis, + MetaClientReconnectPolicy reconnectPolicy, boolean enableAuth, StoreType storeType, + String monitorType, String monitorKey, String monitorInstanceName, + boolean monitorRootPathOnly, PathBasedZkSerializer zkSerializer) { + super(connectionAddress, connectionInitTimeoutInMillis, operationRetryTimeoutInMillis, + sessionTimeoutInMillis, reconnectPolicy, enableAuth, storeType); _zkSerializer = zkSerializer; _monitorType = monitorType; _monitorKey = monitorKey; _monitorInstanceName = monitorInstanceName; _monitorRootPathOnly = monitorRootPathOnly; - } public static class ZkMetaClientConfigBuilder extends MetaClientConfig.MetaClientConfigBuilder<ZkMetaClientConfigBuilder> { @@ -133,8 +134,9 @@ public class ZkMetaClientConfig extends MetaClientConfig { _zkSerializer = new BasicZkSerializer(new SerializableSerializer()); } return new ZkMetaClientConfig(_connectionAddress, _connectionInitTimeoutInMillis, - _sessionTimeoutInMillis, _enableAuth, MetaClientConfig.StoreType.ZOOKEEPER, _monitorType, - _monitorKey, _monitorInstanceName, _monitorRootPathOnly, _zkSerializer); + _operationRetryTimeout, _sessionTimeoutInMillis, _metaClientReconnectPolicy, _enableAuth, + MetaClientConfig.StoreType.ZOOKEEPER, _monitorType, _monitorKey, _monitorInstanceName, + _monitorRootPathOnly, _zkSerializer); } @Override diff --git a/meta-client/src/main/java/org/apache/helix/metaclient/policy/ExponentialBackoffReconnectPolicy.java b/meta-client/src/main/java/org/apache/helix/metaclient/policy/ExponentialBackoffReconnectPolicy.java new file mode 100644 index 000000000..81e0c44f7 --- /dev/null +++ b/meta-client/src/main/java/org/apache/helix/metaclient/policy/ExponentialBackoffReconnectPolicy.java @@ -0,0 +1,53 @@ +package org.apache.helix.metaclient.policy; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.apache.helix.metaclient.policy.MetaClientReconnectPolicy; + +import static org.apache.helix.metaclient.constants.MetaClientConstants.DEFAULT_INIT_EXP_BACKOFF_RETRY_INTERVAL_MS; +import static org.apache.helix.metaclient.constants.MetaClientConstants.DEFAULT_MAX_EXP_BACKOFF_RETRY_INTERVAL_MS; + +/** + * Policy to define client re-establish connection behavior when connection to underlying metadata + * store is expired. + * Wait time before each backoff period will increase exponentially until a user defined max + * backoff interval. + */ +public class ExponentialBackoffReconnectPolicy implements MetaClientReconnectPolicy { + + private final long _maxBackOffInterval; + private final long _initBackoffInterval; + + @Override + public RetryPolicyName getPolicyName() { + return RetryPolicyName.EXP_BACKOFF; + } + + public ExponentialBackoffReconnectPolicy() { + _initBackoffInterval = DEFAULT_INIT_EXP_BACKOFF_RETRY_INTERVAL_MS; + _maxBackOffInterval = DEFAULT_MAX_EXP_BACKOFF_RETRY_INTERVAL_MS; + } + + public ExponentialBackoffReconnectPolicy(long maxBackOffInterval, long initBackoffInterval) { + _maxBackOffInterval = maxBackOffInterval; + _initBackoffInterval = initBackoffInterval; + + } +} diff --git a/meta-client/src/main/java/org/apache/helix/metaclient/constants/MetaClientConstants.java b/meta-client/src/main/java/org/apache/helix/metaclient/policy/MetaClientReconnectPolicy.java similarity index 55% copy from meta-client/src/main/java/org/apache/helix/metaclient/constants/MetaClientConstants.java copy to meta-client/src/main/java/org/apache/helix/metaclient/policy/MetaClientReconnectPolicy.java index e38b1f6a9..3d4cba3de 100644 --- a/meta-client/src/main/java/org/apache/helix/metaclient/constants/MetaClientConstants.java +++ b/meta-client/src/main/java/org/apache/helix/metaclient/policy/MetaClientReconnectPolicy.java @@ -1,4 +1,4 @@ -package org.apache.helix.metaclient.constants; +package org.apache.helix.metaclient.policy; /* * Licensed to the Apache Software Foundation (ASF) under one @@ -19,23 +19,20 @@ package org.apache.helix.metaclient.constants; * under the License. */ -public final class MetaClientConstants { +/** + * Policy to define client re-establish connection behavior when connection to underlying metadata + * store is expired. + */ - private MetaClientConstants(){ +public interface MetaClientReconnectPolicy { + enum RetryPolicyName { + NO_RETRY, + EXP_BACKOFF, + LINEAR_BACKOFF } - // Stop retrying when we reach timeout - //TODO The value should be the same as Helix default ZK retry time. Modify when change #2293 merged - public static final int DEFAULT_OPERATION_RETRY_TIMEOUT_MS = Integer.MAX_VALUE; - - // maxMsToWaitUntilConnected - public static final int DEFAULT_CONNECTION_INIT_TIMEOUT_MS = 60 * 1000; - - // When a client becomes partitioned from the metadata service for more than session timeout, - // new session will be established. - public static final int DEFAULT_SESSION_TIMEOUT_MS = 30 * 1000; - - + RetryPolicyName getPolicyName(); + // TODO: add reconnect timeout } diff --git a/meta-client/src/main/java/org/apache/helix/metaclient/constants/MetaClientConstants.java b/meta-client/src/main/java/org/apache/helix/metaclient/policy/NoRetryReconnectPolicy.java similarity index 55% copy from meta-client/src/main/java/org/apache/helix/metaclient/constants/MetaClientConstants.java copy to meta-client/src/main/java/org/apache/helix/metaclient/policy/NoRetryReconnectPolicy.java index e38b1f6a9..f81273b3a 100644 --- a/meta-client/src/main/java/org/apache/helix/metaclient/constants/MetaClientConstants.java +++ b/meta-client/src/main/java/org/apache/helix/metaclient/policy/NoRetryReconnectPolicy.java @@ -1,4 +1,4 @@ -package org.apache.helix.metaclient.constants; +package org.apache.helix.metaclient.policy; /* * Licensed to the Apache Software Foundation (ASF) under one @@ -19,23 +19,18 @@ package org.apache.helix.metaclient.constants; * under the License. */ -public final class MetaClientConstants { +import org.apache.helix.metaclient.policy.MetaClientReconnectPolicy; - private MetaClientConstants(){ +/** + * Policy to define client re-establish connection behavior when connection to underlying metadata + * store is expired. + * If this retry policy is passed to MetaClient, no auto retry connection will be issued when + * connection lost or expired. + */ +public class NoRetryReconnectPolicy implements MetaClientReconnectPolicy { + @Override + public RetryPolicyName getPolicyName() { + return RetryPolicyName.NO_RETRY; } - - // Stop retrying when we reach timeout - //TODO The value should be the same as Helix default ZK retry time. Modify when change #2293 merged - public static final int DEFAULT_OPERATION_RETRY_TIMEOUT_MS = Integer.MAX_VALUE; - - // maxMsToWaitUntilConnected - public static final int DEFAULT_CONNECTION_INIT_TIMEOUT_MS = 60 * 1000; - - // When a client becomes partitioned from the metadata service for more than session timeout, - // new session will be established. - public static final int DEFAULT_SESSION_TIMEOUT_MS = 30 * 1000; - - - }
