This is an automated email from the ASF dual-hosted git repository.
liubao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/servicecomb-java-chassis.git
The following commit(s) were added to refs/heads/master by this push:
new 6e8b55a72 [#4532] Fixed when the servicecomb engine storage pool is
faulty, the registration and configuration centers cannot perform HA switchover
problem (#4533)
6e8b55a72 is described below
commit 6e8b55a725b7715600a12a247092941228955644
Author: Alex <[email protected]>
AuthorDate: Fri Sep 27 15:47:22 2024 +0800
[#4532] Fixed when the servicecomb engine storage pool is faulty, the
registration and configuration centers cannot perform HA switchover problem
(#4533)
---
.../config/center/client/ConfigCenterClient.java | 22 ++++++
.../config/center/client/ConfigCenterManager.java | 17 +++++
.../center/client/ConfigCenterOperation.java | 8 ++
.../servicecomb/config/kie/client/KieClient.java | 16 ++++
.../config/kie/client/KieConfigManager.java | 22 ++++++
.../config/kie/client/KieConfigOperation.java | 8 ++
.../http/client/common/AbstractAddressManager.java | 85 ++++------------------
.../client/common/AbstractAddressManagerTest.java | 10 +--
.../service/center/client/ServiceCenterClient.java | 12 +++
.../center/client/ServiceCenterOperation.java | 8 ++
.../center/client/ServiceCenterRawClient.java | 26 +++++--
.../center/client/ServiceCenterRegistration.java | 11 +++
12 files changed, 162 insertions(+), 83 deletions(-)
diff --git
a/clients/config-center-client/src/main/java/org/apache/servicecomb/config/center/client/ConfigCenterClient.java
b/clients/config-center-client/src/main/java/org/apache/servicecomb/config/center/client/ConfigCenterClient.java
index cde37e920..4232c3db5 100644
---
a/clients/config-center-client/src/main/java/org/apache/servicecomb/config/center/client/ConfigCenterClient.java
+++
b/clients/config-center-client/src/main/java/org/apache/servicecomb/config/center/client/ConfigCenterClient.java
@@ -132,6 +132,28 @@ public class ConfigCenterClient implements
ConfigCenterOperation {
}
}
+ @Override
+ public void checkAddressAvailable(QueryConfigurationsRequest request, String
address) {
+ String dimensionsInfo = buildDimensionsInfo(request, true);
+ try {
+ String uri = address + "/configuration/items?dimensionsInfo="
+ + HttpUtils.encodeURLParam(dimensionsInfo) + "&revision=" +
request.getRevision();
+
+ Map<String, String> headers = new HashMap<>();
+ headers.put("x-environment", request.getEnvironment());
+ HttpRequest httpRequest = new HttpRequest(uri, headers, null,
+ HttpRequest.GET);
+
+ HttpResponse httpResponse = httpTransport.doRequest(httpRequest);
+ if (httpResponse.getStatusCode() == HttpStatus.SC_NOT_MODIFIED
+ || httpResponse.getStatusCode() == HttpStatus.SC_OK) {
+ addressManager.recoverIsolatedAddress(address);
+ }
+ } catch (Exception e) {
+ LOGGER.error("check config center isolation address {} available
error!", address);
+ }
+ }
+
private String buildDimensionsInfo(QueryConfigurationsRequest request,
boolean withVersion) {
String result =
request.getServiceName() + DEFAULT_APP_SEPARATOR
diff --git
a/clients/config-center-client/src/main/java/org/apache/servicecomb/config/center/client/ConfigCenterManager.java
b/clients/config-center-client/src/main/java/org/apache/servicecomb/config/center/client/ConfigCenterManager.java
index 6ba0212e6..45e2e608c 100644
---
a/clients/config-center-client/src/main/java/org/apache/servicecomb/config/center/client/ConfigCenterManager.java
+++
b/clients/config-center-client/src/main/java/org/apache/servicecomb/config/center/client/ConfigCenterManager.java
@@ -17,6 +17,7 @@
package org.apache.servicecomb.config.center.client;
+import java.util.List;
import java.util.Map;
import
org.apache.servicecomb.config.center.client.model.ConfigCenterConfiguration;
@@ -62,6 +63,7 @@ public class ConfigCenterManager extends AbstractTask {
public void startConfigCenterManager() {
this.startTask(new PollConfigurationTask(0));
+ this.startTask(new CheckConfigCenterAddressTask());
}
class PollConfigurationTask implements Task {
@@ -93,4 +95,19 @@ public class ConfigCenterManager extends AbstractTask {
}
}
}
+
+ class CheckConfigCenterAddressTask implements Task {
+ @Override
+ public void execute() {
+ List<String> isolationAddresses =
configCenterAddressManager.getIsolationAddresses();
+ if (isolationAddresses.isEmpty()) {
+ return;
+ }
+ for (String address : isolationAddresses) {
+ configCenterClient.checkAddressAvailable(queryConfigurationsRequest,
address);
+ }
+ startTask(new
BackOffSleepTask(configCenterConfiguration.getRefreshIntervalInMillis(),
+ new CheckConfigCenterAddressTask()));
+ }
+ }
}
diff --git
a/clients/config-center-client/src/main/java/org/apache/servicecomb/config/center/client/ConfigCenterOperation.java
b/clients/config-center-client/src/main/java/org/apache/servicecomb/config/center/client/ConfigCenterOperation.java
index d05967692..43b30c748 100644
---
a/clients/config-center-client/src/main/java/org/apache/servicecomb/config/center/client/ConfigCenterOperation.java
+++
b/clients/config-center-client/src/main/java/org/apache/servicecomb/config/center/client/ConfigCenterOperation.java
@@ -30,4 +30,12 @@ public interface ConfigCenterOperation {
* @throws OperationException If some problems happened to contact service
center or non http 200 returned.
*/
QueryConfigurationsResponse queryConfigurations(QueryConfigurationsRequest
request, String address);
+
+ /**
+ * Check config center isolation address available
+ *
+ * @param request queryConfigurationsRequest
+ * @param address isolation address
+ */
+ void checkAddressAvailable(QueryConfigurationsRequest request, String
address);
}
diff --git
a/clients/config-kie-client/src/main/java/org/apache/servicecomb/config/kie/client/KieClient.java
b/clients/config-kie-client/src/main/java/org/apache/servicecomb/config/kie/client/KieClient.java
index 50cf2b873..e85a7c658 100644
---
a/clients/config-kie-client/src/main/java/org/apache/servicecomb/config/kie/client/KieClient.java
+++
b/clients/config-kie-client/src/main/java/org/apache/servicecomb/config/kie/client/KieClient.java
@@ -19,6 +19,7 @@ package org.apache.servicecomb.config.kie.client;
import com.google.common.eventbus.EventBus;
+import java.io.IOException;
import java.io.StringReader;
import java.nio.charset.StandardCharsets;
import java.util.Collections;
@@ -113,6 +114,21 @@ public class KieClient implements KieConfigOperation {
}
}
+ @Override
+ public void checkAddressAvailable(ConfigurationsRequest request, String
address) {
+ String url = buildUrl(request, address);
+ HttpRequest httpRequest = new HttpRequest(url, null, null,
HttpRequest.GET);
+ try {
+ HttpResponse httpResponse = httpTransport.doRequest(httpRequest);
+ if (httpResponse.getStatusCode() == HttpStatus.SC_NOT_MODIFIED
+ || httpResponse.getStatusCode() == HttpStatus.SC_OK) {
+ addressManager.recoverIsolatedAddress(address);
+ }
+ } catch (IOException e) {
+ LOGGER.error("check kie config isolation address {} available error!",
address, e);
+ }
+ }
+
private String buildUrl(ConfigurationsRequest request, String
currentAddress) {
StringBuilder sb = new StringBuilder();
sb.append(currentAddress);
diff --git
a/clients/config-kie-client/src/main/java/org/apache/servicecomb/config/kie/client/KieConfigManager.java
b/clients/config-kie-client/src/main/java/org/apache/servicecomb/config/kie/client/KieConfigManager.java
index 3b3af5061..edd345f80 100644
---
a/clients/config-kie-client/src/main/java/org/apache/servicecomb/config/kie/client/KieConfigManager.java
+++
b/clients/config-kie-client/src/main/java/org/apache/servicecomb/config/kie/client/KieConfigManager.java
@@ -124,6 +124,7 @@ public class KieConfigManager extends AbstractTask {
public void startConfigKieManager() {
this.configurationsRequests.forEach((t) ->
this.startTask(new PollConfigurationTask(0, t)));
+ startTask(new CheckKieConfigAddressTask(configurationsRequests.get(0)));
}
class PollConfigurationTask implements Task {
@@ -160,4 +161,25 @@ public class KieConfigManager extends AbstractTask {
}
}
}
+
+ class CheckKieConfigAddressTask implements Task {
+ ConfigurationsRequest configurationsRequest;
+
+ public CheckKieConfigAddressTask(ConfigurationsRequest
configurationsRequest) {
+ this.configurationsRequest = configurationsRequest;
+ }
+
+ @Override
+ public void execute() {
+ List<String> isolationAddresses =
kieAddressManager.getIsolationAddresses();
+ if (isolationAddresses.isEmpty()) {
+ return;
+ }
+ for (String address : isolationAddresses) {
+ configKieClient.checkAddressAvailable(this.configurationsRequest,
address);
+ }
+ startTask(new
BackOffSleepTask(kieConfiguration.getRefreshIntervalInMillis(),
+ new CheckKieConfigAddressTask(this.configurationsRequest)));
+ }
+ }
}
diff --git
a/clients/config-kie-client/src/main/java/org/apache/servicecomb/config/kie/client/KieConfigOperation.java
b/clients/config-kie-client/src/main/java/org/apache/servicecomb/config/kie/client/KieConfigOperation.java
index 229439867..44f66137f 100644
---
a/clients/config-kie-client/src/main/java/org/apache/servicecomb/config/kie/client/KieConfigOperation.java
+++
b/clients/config-kie-client/src/main/java/org/apache/servicecomb/config/kie/client/KieConfigOperation.java
@@ -31,4 +31,12 @@ public interface KieConfigOperation {
* @throws OperationException If some problems happened to contact service
center or non http 200 returned.
*/
ConfigurationsResponse queryConfigurations(ConfigurationsRequest request,
String address);
+
+ /**
+ * Check kie isolation address available
+ *
+ * @param configurationsRequest configurationsRequest
+ * @param address isolation address
+ */
+ void checkAddressAvailable(ConfigurationsRequest configurationsRequest,
String address);
}
diff --git
a/clients/http-client-common/src/main/java/org/apache/servicecomb/http/client/common/AbstractAddressManager.java
b/clients/http-client-common/src/main/java/org/apache/servicecomb/http/client/common/AbstractAddressManager.java
index 533a6e530..99eae8427 100644
---
a/clients/http-client-common/src/main/java/org/apache/servicecomb/http/client/common/AbstractAddressManager.java
+++
b/clients/http-client-common/src/main/java/org/apache/servicecomb/http/client/common/AbstractAddressManager.java
@@ -17,21 +17,11 @@
package org.apache.servicecomb.http.client.common;
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.net.Socket;
-import java.net.URI;
-import java.net.URISyntaxException;
import java.util.ArrayList;
-import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Random;
-import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import org.apache.commons.lang3.StringUtils;
@@ -42,7 +32,6 @@ import org.slf4j.LoggerFactory;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.eventbus.EventBus;
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
public class AbstractAddressManager {
private static final Logger LOGGER =
LoggerFactory.getLogger(AbstractAddressManager.class);
@@ -53,8 +42,6 @@ public class AbstractAddressManager {
private static final String V3_PREFIX = "/v3/";
- private static final int DEFAULT_ADDRESS_CHECK_TIME = 30;
-
private static final int ISOLATION_THRESHOLD = 3;
private volatile List<String> addresses = new ArrayList<>();
@@ -68,9 +55,6 @@ public class AbstractAddressManager {
private String projectName;
- // all address list.
- private final Set<String> addressCategory = new HashSet<>();
-
// recording continuous times of failure of an address.
private final Map<String, Integer> addressFailureStatus = new
ConcurrentHashMap<>();
@@ -90,27 +74,22 @@ public class AbstractAddressManager {
private EventBus eventBus;
- private final ScheduledExecutorService executorService =
Executors.newScheduledThreadPool(1,
- new ThreadFactoryBuilder()
- .setNameFormat("check-available-address-%d")
- .build());
-
public AbstractAddressManager(List<String> addresses) {
this.projectName = DEFAULT_PROJECT;
this.addresses.addAll(addresses);
this.defaultAddress.addAll(addresses);
- this.addressCategory.addAll(addresses);
- this.index = !addresses.isEmpty() ? random.nextInt(addresses.size()) : 0;
- startCheck();
+ this.index = !addresses.isEmpty() ? getRandomIndex() : 0;
}
public AbstractAddressManager(String projectName, List<String> addresses) {
this.projectName = StringUtils.isEmpty(projectName) ? DEFAULT_PROJECT :
projectName;
this.addresses = this.transformAddress(addresses);
this.defaultAddress.addAll(addresses);
- this.addressCategory.addAll(this.addresses);
- this.index = !addresses.isEmpty() ? random.nextInt(addresses.size()) : 0;
- startCheck();
+ this.index = !addresses.isEmpty() ? getRandomIndex() : 0;
+ }
+
+ private int getRandomIndex() {
+ return random.nextInt(addresses.size());
}
public void refreshEndpoint(RefreshEndpointEvent event, String key) {
@@ -120,8 +99,6 @@ public class AbstractAddressManager {
availableZone =
event.getSameZone().stream().map(this::normalizeUri).collect(Collectors.toList());
availableRegion =
event.getSameRegion().stream().map(this::normalizeUri).collect(Collectors.toList());
- addressCategory.addAll(availableZone);
- addressCategory.addAll(availableRegion);
addressAutoRefreshed = true;
}
@@ -146,10 +123,6 @@ public class AbstractAddressManager {
return availableRegion;
}
- private void startCheck() {
- executorService.scheduleAtFixedRate(this::checkHistory, 0,
DEFAULT_ADDRESS_CHECK_TIME, TimeUnit.SECONDS);
- }
-
public String formatUrl(String url, boolean absoluteUrl, String address) {
return absoluteUrl ? address + url : formatAddress(address) + url;
}
@@ -188,7 +161,7 @@ public class AbstractAddressManager {
}
LOGGER.warn("all addresses are isolation, please check server status.");
// when all addresses are isolation, it will use all default address for
polling.
- return getCurrentAddress(new ArrayList<>(defaultAddress));
+ return getCurrentAddress(defaultAddress);
}
private String getAvailableZoneAddress() {
@@ -221,42 +194,7 @@ public class AbstractAddressManager {
return results;
}
- @VisibleForTesting
- protected void checkHistory() {
- addressCategory.forEach(address -> {
- if (telnetTest(address)) {
- // isolation addresses find address and restore it
- findAndRestoreAddress(address);
- } else {
- recordFailState(address);
- }
- });
- }
-
- protected boolean telnetTest(String address) {
- URI uri = parseIpPortFromURI(address);
- if (uri == null) {
- return false;
- }
- try (Socket s = new Socket()) {
- s.connect(new InetSocketAddress(uri.getHost(), uri.getPort()), 3000);
- return true;
- } catch (IOException e) {
- LOGGER.warn("ping endpoint {} failed, It will be quarantined again.",
address);
- return false;
- }
- }
-
- private URI parseIpPortFromURI(String address) {
- try {
- return new URI(address);
- } catch (URISyntaxException e) {
- LOGGER.error("parse address [{}] failed.", address, e);
- return null;
- }
- }
-
- protected void findAndRestoreAddress(String address) {
+ public void recoverIsolatedAddress(String address) {
recordSuccessState(address);
if (addressAutoRefreshed) {
if (isolationZoneAddress.remove(address)) {
@@ -326,4 +264,11 @@ public class AbstractAddressManager {
public void setEventBus(EventBus eventBus) {
this.eventBus = eventBus;
}
+
+ public List<String> getIsolationAddresses() {
+ List<String> isolationAddresses = new ArrayList<>(defaultIsolationAddress);
+ isolationAddresses.addAll(isolationZoneAddress);
+ isolationAddresses.addAll(isolationRegionAddress);
+ return isolationAddresses;
+ }
}
diff --git
a/clients/http-client-common/src/test/java/org/apache/servicecomb/http/client/common/AbstractAddressManagerTest.java
b/clients/http-client-common/src/test/java/org/apache/servicecomb/http/client/common/AbstractAddressManagerTest.java
index b60311f6d..83406ec74 100644
---
a/clients/http-client-common/src/test/java/org/apache/servicecomb/http/client/common/AbstractAddressManagerTest.java
+++
b/clients/http-client-common/src/test/java/org/apache/servicecomb/http/client/common/AbstractAddressManagerTest.java
@@ -88,12 +88,7 @@ public class AbstractAddressManagerTest {
zoneAndRegion.put("sameZone", addressAZ);
zoneAndRegion.put("sameRegion", addressRG);
RefreshEndpointEvent event = new RefreshEndpointEvent(zoneAndRegion,
"TEST");
- AbstractAddressManager addressManager = new
AbstractAddressManager(addresses) {
- @Override
- protected boolean telnetTest(String address) {
- return true;
- }
- };
+ AbstractAddressManager addressManager = new
AbstractAddressManager(addresses) {};
addressManager.refreshEndpoint(event, "TEST");
@@ -120,8 +115,7 @@ public class AbstractAddressManagerTest {
Assertions.assertEquals("http://127.0.0.4:30100";,
addressManager.address());
// test restore isolation
- addressManager.checkHistory();
- addressManager.findAndRestoreAddress("http://127.0.0.3:30100";);
+ addressManager.recoverIsolatedAddress("http://127.0.0.3:30100";);
Assertions.assertEquals("http://127.0.0.3:30100";,
addressManager.address());
Assertions.assertEquals("http://127.0.0.3:30100";,
addressManager.address());
}
diff --git
a/clients/service-center-client/src/main/java/org/apache/servicecomb/service/center/client/ServiceCenterClient.java
b/clients/service-center-client/src/main/java/org/apache/servicecomb/service/center/client/ServiceCenterClient.java
index f5cec283e..c99486151 100755
---
a/clients/service-center-client/src/main/java/org/apache/servicecomb/service/center/client/ServiceCenterClient.java
+++
b/clients/service-center-client/src/main/java/org/apache/servicecomb/service/center/client/ServiceCenterClient.java
@@ -553,4 +553,16 @@ public class ServiceCenterClient implements
ServiceCenterOperation {
"update service instance status fails", e);
}
}
+
+ @Override
+ public void checkIsolationAddressAvailable(String serviceId, String
instanceId) {
+ List<String> isolationAddresses = addressManager.getIsolationAddresses();
+ if (isolationAddresses.isEmpty()) {
+ return;
+ }
+ for (String address : isolationAddresses) {
+ httpClient.checkServiceCenterAddressAvailable("/registry/microservices/"
+ serviceId + "/instances/" + instanceId +
+ "/heartbeat", null, null, address);
+ }
+ }
}
diff --git
a/clients/service-center-client/src/main/java/org/apache/servicecomb/service/center/client/ServiceCenterOperation.java
b/clients/service-center-client/src/main/java/org/apache/servicecomb/service/center/client/ServiceCenterOperation.java
index 5521a7322..cd91b240b 100644
---
a/clients/service-center-client/src/main/java/org/apache/servicecomb/service/center/client/ServiceCenterOperation.java
+++
b/clients/service-center-client/src/main/java/org/apache/servicecomb/service/center/client/ServiceCenterOperation.java
@@ -193,4 +193,12 @@ public interface ServiceCenterOperation {
* @throws OperationException If some problems happened to contact service
center or non http 200 returned.
*/
boolean updateMicroserviceProperties(String microserviceId, Map<String,
String> serviceProperties);
+
+ /**
+ * Check serviceCenter isolation address available
+ *
+ * @param serviceId serviceId
+ * @param instanceId instanceId
+ */
+ void checkIsolationAddressAvailable(String serviceId, String instanceId);
}
diff --git
a/clients/service-center-client/src/main/java/org/apache/servicecomb/service/center/client/ServiceCenterRawClient.java
b/clients/service-center-client/src/main/java/org/apache/servicecomb/service/center/client/ServiceCenterRawClient.java
index 6c293004c..c58e5dc8b 100755
---
a/clients/service-center-client/src/main/java/org/apache/servicecomb/service/center/client/ServiceCenterRawClient.java
+++
b/clients/service-center-client/src/main/java/org/apache/servicecomb/service/center/client/ServiceCenterRawClient.java
@@ -71,11 +71,7 @@ public class ServiceCenterRawClient {
throws IOException {
String address = addressManager.address();
String formatUrl = addressManager.formatUrl(url, absoluteUrl, address);
- if (headers == null) {
- headers = new HashMap<>();
- }
- headers.put(HEADER_TENANT_NAME, tenantName);
- HttpRequest httpRequest = new HttpRequest(formatUrl, headers, content,
method);
+ HttpRequest httpRequest = buildHttpRequest(formatUrl, headers, content,
method);
try {
HttpResponse httpResponse = httpTransport.doRequest(httpRequest);
@@ -97,6 +93,26 @@ public class ServiceCenterRawClient {
}
}
+ public void checkServiceCenterAddressAvailable(String url, Map<String,
String> headers, String content,
+ String address) {
+ String formatUrl = addressManager.formatUrl(url, false, address);
+ HttpRequest httpRequest = buildHttpRequest(formatUrl, headers, content,
HttpRequest.PUT);
+ try {
+ httpTransport.doRequest(httpRequest);
+ addressManager.recoverIsolatedAddress(address);
+ } catch (IOException e) {
+ LOGGER.error("check service center isolation address {} available
error!", address, e);
+ }
+ }
+
+ private HttpRequest buildHttpRequest(String url, Map<String, String>
headers, String content, String method) {
+ if (headers == null) {
+ headers = new HashMap<>();
+ }
+ headers.put(HEADER_TENANT_NAME, tenantName);
+ return new HttpRequest(url, headers, content, method);
+ }
+
public static class Builder {
private String tenantName;
diff --git
a/clients/service-center-client/src/main/java/org/apache/servicecomb/service/center/client/ServiceCenterRegistration.java
b/clients/service-center-client/src/main/java/org/apache/servicecomb/service/center/client/ServiceCenterRegistration.java
index 555c739d4..f8ccd218f 100644
---
a/clients/service-center-client/src/main/java/org/apache/servicecomb/service/center/client/ServiceCenterRegistration.java
+++
b/clients/service-center-client/src/main/java/org/apache/servicecomb/service/center/client/ServiceCenterRegistration.java
@@ -258,6 +258,7 @@ public class ServiceCenterRegistration extends AbstractTask
{
microserviceInstance.getInstanceId());
eventBus.post(new MicroserviceInstanceRegistrationEvent(true,
microservice, microserviceInstance));
startTask(new SendHeartBeatTask(0));
+ startTask(new CheckServiceCenterAddressTask());
}
} catch (Exception e) {
LOGGER.error("register microservice instance failed, and will try
again.", e);
@@ -302,4 +303,14 @@ public class ServiceCenterRegistration extends
AbstractTask {
}
}
}
+
+ class CheckServiceCenterAddressTask implements Task {
+ @Override
+ public void execute() {
+
serviceCenterClient.checkIsolationAddressAvailable(microservice.getServiceId(),
+ microserviceInstance.getInstanceId());
+ startTask(new BackOffSleepTask(Math.max(heartBeatInterval,
heartBeatRequestTimeout),
+ new CheckServiceCenterAddressTask()));
+ }
+ }
}
