This is an automated email from the ASF dual-hosted git repository.
dockerzhang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
The following commit(s) were added to refs/heads/master by this push:
new 9d745b8449 [INLONG-8393][DataProxy] Optimize the HeartbeatManager
class (#9515)
9d745b8449 is described below
commit 9d745b8449b8f57573668d7c332a71179027be20
Author: Goson Zhang <[email protected]>
AuthorDate: Sat Dec 23 09:59:09 2023 +0800
[INLONG-8393][DataProxy] Optimize the HeartbeatManager class (#9515)
---
.../inlong/dataproxy/config/ConfigManager.java | 12 +++--
.../config/holder/SourceReportConfigHolder.java | 51 +++++--------------
.../dataproxy/config/holder/SourceReportInfo.java | 57 ----------------------
.../dataproxy/heartbeat/HeartbeatManager.java | 15 ++----
.../apache/inlong/dataproxy/source/BaseSource.java | 11 ++++-
.../inlong/dataproxy/source/SimpleHttpSource.java | 7 +--
.../inlong/dataproxy/source/SimpleTcpSource.java | 7 +--
.../inlong/dataproxy/source/SimpleUdpSource.java | 7 +--
.../inlong/dataproxy/source/SourceConstants.java | 2 +
.../config/holder/TestSourceReportHolder.java | 30 ++++++------
10 files changed, 63 insertions(+), 136 deletions(-)
diff --git
a/inlong-dataproxy/dataproxy-source/src/main/java/org/apache/inlong/dataproxy/config/ConfigManager.java
b/inlong-dataproxy/dataproxy-source/src/main/java/org/apache/inlong/dataproxy/config/ConfigManager.java
index 647a41936e..d6a8205460 100644
---
a/inlong-dataproxy/dataproxy-source/src/main/java/org/apache/inlong/dataproxy/config/ConfigManager.java
+++
b/inlong-dataproxy/dataproxy-source/src/main/java/org/apache/inlong/dataproxy/config/ConfigManager.java
@@ -17,6 +17,7 @@
package org.apache.inlong.dataproxy.config;
+import org.apache.inlong.common.heartbeat.AddressInfo;
import org.apache.inlong.common.pojo.dataproxy.DataProxyCluster;
import org.apache.inlong.common.pojo.dataproxy.DataProxyConfigRequest;
import org.apache.inlong.common.pojo.dataproxy.DataProxyConfigResponse;
@@ -25,7 +26,6 @@ import
org.apache.inlong.dataproxy.config.holder.ConfigUpdateCallback;
import org.apache.inlong.dataproxy.config.holder.GroupIdNumConfigHolder;
import org.apache.inlong.dataproxy.config.holder.MetaConfigHolder;
import org.apache.inlong.dataproxy.config.holder.SourceReportConfigHolder;
-import org.apache.inlong.dataproxy.config.holder.SourceReportInfo;
import org.apache.inlong.dataproxy.config.holder.WeightConfigHolder;
import org.apache.inlong.dataproxy.config.holder.WhiteListConfigHolder;
import org.apache.inlong.dataproxy.config.pojo.CacheClusterConfig;
@@ -47,6 +47,7 @@ import org.slf4j.LoggerFactory;
import java.security.SecureRandom;
import java.util.ArrayList;
import java.util.List;
+import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;
@@ -204,12 +205,13 @@ public class ConfigManager {
|| whitelistConfigHolder.isIllegalIP(strRemoteIP);
}
- public void addSourceReportInfo(String sourceIp, String sourcePort, String
protocolType) {
- sourceReportConfigHolder.addSourceInfo(sourceIp, sourcePort,
protocolType);
+ public void addSourceReportInfo(String sourceIp,
+ String sourcePort, String rptSrcType, String protocolType) {
+ sourceReportConfigHolder.addSourceInfo(sourceIp, sourcePort,
rptSrcType, protocolType);
}
- public SourceReportInfo getSourceReportInfo() {
- return sourceReportConfigHolder.getSourceReportInfo();
+ public Map<String, AddressInfo> getSrcAddressInfos() {
+ return sourceReportConfigHolder.getSrcAddressInfos();
}
public boolean isMqClusterReady() {
diff --git
a/inlong-dataproxy/dataproxy-source/src/main/java/org/apache/inlong/dataproxy/config/holder/SourceReportConfigHolder.java
b/inlong-dataproxy/dataproxy-source/src/main/java/org/apache/inlong/dataproxy/config/holder/SourceReportConfigHolder.java
index a332150804..cdcee01f00 100644
---
a/inlong-dataproxy/dataproxy-source/src/main/java/org/apache/inlong/dataproxy/config/holder/SourceReportConfigHolder.java
+++
b/inlong-dataproxy/dataproxy-source/src/main/java/org/apache/inlong/dataproxy/config/holder/SourceReportConfigHolder.java
@@ -17,14 +17,14 @@
package org.apache.inlong.dataproxy.config.holder;
+import org.apache.inlong.common.heartbeat.AddressInfo;
+
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.HashMap;
import java.util.Map;
-import java.util.concurrent.locks.ReadWriteLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
/**
*
@@ -36,57 +36,28 @@ public class SourceReportConfigHolder {
public static final Logger LOG =
LoggerFactory.getLogger(SourceReportConfigHolder.class);
- private final ReadWriteLock readWriteLock = new ReentrantReadWriteLock();
- private final Map<String, SourceReportInfo> sourceConfMap = new
HashMap<>();
- // Begin: this part can be optimized later
- // after optimizing the implementation of the heartbeat reporting interface
- // between Manager and DataProxy, the report is directly configured
according to
- // the SourceReportInfo of each source, instead of splicing report items
separately.
- private String ipSet = "";
- private String portSet = "";
- private String protocolTypeSet = "";
- // end
+ private final Map<String, AddressInfo> srcAddressMap = new HashMap<>();
public SourceReportConfigHolder() {
}
- public void addSourceInfo(String sourceIp, String sourcePort, String
protocolType) {
+ public void addSourceInfo(String sourceIp,
+ String sourcePort, String rptSrcType, String protocolType) {
if (StringUtils.isEmpty(sourceIp)
|| StringUtils.isEmpty(sourcePort)
+ || StringUtils.isEmpty(rptSrcType)
|| StringUtils.isEmpty(protocolType)) {
- LOG.warn("[Source Report Holder] found empty parameter!, add
values is {}, {}, {}",
- sourceIp, sourcePort, protocolType);
+ LOG.warn("[Source Report Holder] found empty parameter!, add
values is {}, {}, {}, {}",
+ sourceIp, sourcePort, rptSrcType, protocolType);
return;
}
String recordKey = sourceIp + "#" + sourcePort + "#" + protocolType;
- SourceReportInfo sourceReportInfo =
- new SourceReportInfo(sourceIp, sourcePort, protocolType);
- try {
- readWriteLock.writeLock().lock();
- if (sourceConfMap.putIfAbsent(recordKey, sourceReportInfo) ==
null) {
- if (ipSet.isEmpty()) {
- ipSet = sourceIp;
- portSet = sourcePort;
- protocolTypeSet = protocolType;
- } else {
- ipSet += "," + sourceIp;
- portSet += "," + sourcePort;
- protocolTypeSet += "," + protocolType;
- }
- }
- } finally {
- readWriteLock.writeLock().unlock();
- }
+ srcAddressMap.put(recordKey, new AddressInfo(sourceIp, sourcePort,
rptSrcType, protocolType));
}
- public SourceReportInfo getSourceReportInfo() {
- try {
- readWriteLock.readLock().lock();
- return new SourceReportInfo(ipSet, portSet, protocolTypeSet);
- } finally {
- readWriteLock.readLock().unlock();
- }
+ public Map<String, AddressInfo> getSrcAddressInfos() {
+ return this.srcAddressMap;
}
}
diff --git
a/inlong-dataproxy/dataproxy-source/src/main/java/org/apache/inlong/dataproxy/config/holder/SourceReportInfo.java
b/inlong-dataproxy/dataproxy-source/src/main/java/org/apache/inlong/dataproxy/config/holder/SourceReportInfo.java
deleted file mode 100644
index 61cdc549ce..0000000000
---
a/inlong-dataproxy/dataproxy-source/src/main/java/org/apache/inlong/dataproxy/config/holder/SourceReportInfo.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.inlong.dataproxy.config.holder;
-
-public class SourceReportInfo {
-
- private String ip;
-
- private String port;
-
- private String protocolType;
-
- public SourceReportInfo(String ip, String port, String protocolType) {
- this.ip = ip;
- this.port = port;
- this.protocolType = protocolType;
- }
-
- public String getIp() {
- return ip;
- }
-
- public void setIp(String ip) {
- this.ip = ip;
- }
-
- public String getPort() {
- return port;
- }
-
- public void setPort(String port) {
- this.port = port;
- }
-
- public String getProtocolType() {
- return protocolType;
- }
-
- public void setProtocolType(String protocolType) {
- this.protocolType = protocolType;
- }
-}
diff --git
a/inlong-dataproxy/dataproxy-source/src/main/java/org/apache/inlong/dataproxy/heartbeat/HeartbeatManager.java
b/inlong-dataproxy/dataproxy-source/src/main/java/org/apache/inlong/dataproxy/heartbeat/HeartbeatManager.java
index c1a9bc664c..fdc3bc304d 100644
---
a/inlong-dataproxy/dataproxy-source/src/main/java/org/apache/inlong/dataproxy/heartbeat/HeartbeatManager.java
+++
b/inlong-dataproxy/dataproxy-source/src/main/java/org/apache/inlong/dataproxy/heartbeat/HeartbeatManager.java
@@ -20,12 +20,12 @@ package org.apache.inlong.dataproxy.heartbeat;
import org.apache.inlong.common.enums.ComponentTypeEnum;
import org.apache.inlong.common.enums.NodeSrvStatus;
import org.apache.inlong.common.heartbeat.AbstractHeartbeatManager;
+import org.apache.inlong.common.heartbeat.AddressInfo;
import org.apache.inlong.common.heartbeat.GroupHeartbeat;
import org.apache.inlong.common.heartbeat.HeartbeatMsg;
import org.apache.inlong.common.heartbeat.StreamHeartbeat;
import org.apache.inlong.dataproxy.config.CommonConfigHolder;
import org.apache.inlong.dataproxy.config.ConfigManager;
-import org.apache.inlong.dataproxy.config.holder.SourceReportInfo;
import org.apache.inlong.dataproxy.consts.ConfigConstants;
import org.apache.inlong.dataproxy.utils.HttpUtils;
@@ -43,6 +43,7 @@ import org.apache.http.util.EntityUtils;
import java.util.ArrayList;
import java.util.List;
+import java.util.Map;
import java.util.Map.Entry;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;
@@ -132,16 +133,14 @@ public class HeartbeatManager implements
AbstractHeartbeatManager {
private HeartbeatMsg buildHeartbeat() {
ConfigManager configManager = ConfigManager.getInstance();
HeartbeatMsg heartbeatMsg = new HeartbeatMsg();
- SourceReportInfo reportInfo = configManager.getSourceReportInfo();
- if (!validReportInfo(reportInfo)) {
+ Map<String, AddressInfo> reportInfoMap =
configManager.getSrcAddressInfos();
+ if (reportInfoMap.isEmpty()) {
return null;
}
+ heartbeatMsg.setAddressInfos(new ArrayList<>(reportInfoMap.values()));
heartbeatMsg.setNodeSrvStatus(ConfigManager.getInstance().isMqClusterReady()
? NodeSrvStatus.OK
: NodeSrvStatus.SERVICE_UNREADY);
- heartbeatMsg.setIp(reportInfo.getIp());
- heartbeatMsg.setPort(reportInfo.getPort());
- heartbeatMsg.setProtocolType(reportInfo.getProtocolType());
heartbeatMsg.setComponentType(ComponentTypeEnum.DataProxy.getType());
heartbeatMsg.setReportTime(System.currentTimeMillis());
heartbeatMsg.setLoad(0xffff);
@@ -183,8 +182,4 @@ public class HeartbeatManager implements
AbstractHeartbeatManager {
heartbeatMsg.setStreamHeartbeats(streamHeartbeats);
return heartbeatMsg;
}
-
- private boolean validReportInfo(SourceReportInfo reportInfo) {
- return StringUtils.isNotBlank(reportInfo.getIp()) &&
StringUtils.isNotBlank(reportInfo.getPort());
- }
}
diff --git
a/inlong-dataproxy/dataproxy-source/src/main/java/org/apache/inlong/dataproxy/source/BaseSource.java
b/inlong-dataproxy/dataproxy-source/src/main/java/org/apache/inlong/dataproxy/source/BaseSource.java
index 07ed1efe01..c317bba799 100644
---
a/inlong-dataproxy/dataproxy-source/src/main/java/org/apache/inlong/dataproxy/source/BaseSource.java
+++
b/inlong-dataproxy/dataproxy-source/src/main/java/org/apache/inlong/dataproxy/source/BaseSource.java
@@ -17,6 +17,7 @@
package org.apache.inlong.dataproxy.source;
+import org.apache.inlong.common.heartbeat.ReportResourceType;
import org.apache.inlong.common.metric.MetricRegister;
import org.apache.inlong.dataproxy.admin.ProxyServiceMBean;
import org.apache.inlong.dataproxy.channel.FailoverChannelProcessor;
@@ -86,6 +87,8 @@ public abstract class BaseSource
// source serviced port
protected int srcPort;
protected String strPort;
+ // report source name
+ protected String rptSrcType;
// message factory name
protected String msgFactoryName;
// message handler name
@@ -143,8 +146,14 @@ public abstract class BaseSource
this.srcHost = getHostIp(context);
this.srcPort = getHostPort(context);
this.strPort = String.valueOf(this.srcPort);
+ // get source logic type
+ String tmpVal = context.getString(
+ SourceConstants.SRCCXT_LOGIC_TYPE_NAME,
ReportResourceType.INLONG);
+ Preconditions.checkArgument(StringUtils.isNotBlank(tmpVal),
+ SourceConstants.SRCCXT_LOGIC_TYPE_NAME + " config is blank");
+ this.rptSrcType = tmpVal.trim().toUpperCase();
// get message factory
- String tmpVal =
context.getString(SourceConstants.SRCCXT_MSG_FACTORY_NAME,
+ tmpVal = context.getString(SourceConstants.SRCCXT_MSG_FACTORY_NAME,
ServerMessageFactory.class.getName()).trim();
Preconditions.checkArgument(StringUtils.isNotBlank(tmpVal),
SourceConstants.SRCCXT_MSG_FACTORY_NAME + " config is blank");
diff --git
a/inlong-dataproxy/dataproxy-source/src/main/java/org/apache/inlong/dataproxy/source/SimpleHttpSource.java
b/inlong-dataproxy/dataproxy-source/src/main/java/org/apache/inlong/dataproxy/source/SimpleHttpSource.java
index ce62d7ab24..64375733fd 100644
---
a/inlong-dataproxy/dataproxy-source/src/main/java/org/apache/inlong/dataproxy/source/SimpleHttpSource.java
+++
b/inlong-dataproxy/dataproxy-source/src/main/java/org/apache/inlong/dataproxy/source/SimpleHttpSource.java
@@ -100,9 +100,10 @@ public class SimpleHttpSource extends BaseSource
implements Configurable {
this.getCachedSrcName(), srcHost, srcPort, e);
System.exit(-1);
}
- ConfigManager.getInstance().addSourceReportInfo(
- srcHost, String.valueOf(srcPort),
getProtocolName().toUpperCase());
- logger.info("Source {} started at ({}:{})!", this.getCachedSrcName(),
srcHost, srcPort);
+ ConfigManager.getInstance().addSourceReportInfo(srcHost,
+ String.valueOf(srcPort), rptSrcType,
getProtocolName().toUpperCase());
+ logger.info("Source {} started at ({}:{}), rptSrcType={}!",
+ this.getCachedSrcName(), srcHost, srcPort, rptSrcType);
}
@Override
diff --git
a/inlong-dataproxy/dataproxy-source/src/main/java/org/apache/inlong/dataproxy/source/SimpleTcpSource.java
b/inlong-dataproxy/dataproxy-source/src/main/java/org/apache/inlong/dataproxy/source/SimpleTcpSource.java
index 9306128fa2..5455cdddb4 100644
---
a/inlong-dataproxy/dataproxy-source/src/main/java/org/apache/inlong/dataproxy/source/SimpleTcpSource.java
+++
b/inlong-dataproxy/dataproxy-source/src/main/java/org/apache/inlong/dataproxy/source/SimpleTcpSource.java
@@ -109,9 +109,10 @@ public class SimpleTcpSource extends BaseSource implements
Configurable {
this.getCachedSrcName(), srcHost, srcPort, e);
System.exit(-1);
}
- ConfigManager.getInstance().addSourceReportInfo(
- srcHost, String.valueOf(srcPort),
getProtocolName().toUpperCase());
- logger.info("Source {} started at ({}:{})!", this.getCachedSrcName(),
srcHost, srcPort);
+ ConfigManager.getInstance().addSourceReportInfo(srcHost,
+ String.valueOf(srcPort), rptSrcType,
getProtocolName().toUpperCase());
+ logger.info("Source {} started at ({}:{}), rptSrcType={}!",
+ this.getCachedSrcName(), srcHost, srcPort, rptSrcType);
}
@Override
diff --git
a/inlong-dataproxy/dataproxy-source/src/main/java/org/apache/inlong/dataproxy/source/SimpleUdpSource.java
b/inlong-dataproxy/dataproxy-source/src/main/java/org/apache/inlong/dataproxy/source/SimpleUdpSource.java
index 8b7df49122..fef78bdd1b 100644
---
a/inlong-dataproxy/dataproxy-source/src/main/java/org/apache/inlong/dataproxy/source/SimpleUdpSource.java
+++
b/inlong-dataproxy/dataproxy-source/src/main/java/org/apache/inlong/dataproxy/source/SimpleUdpSource.java
@@ -72,9 +72,10 @@ public class SimpleUdpSource extends BaseSource implements
Configurable {
this.getCachedSrcName(), srcHost, srcPort, e);
System.exit(-1);
}
- ConfigManager.getInstance().addSourceReportInfo(
- srcHost, String.valueOf(srcPort),
getProtocolName().toUpperCase());
- logger.info("Source {} started at ({}:{})!", this.getCachedSrcName(),
srcHost, srcPort);
+ ConfigManager.getInstance().addSourceReportInfo(srcHost,
+ String.valueOf(srcPort), rptSrcType,
getProtocolName().toUpperCase());
+ logger.info("Source {} started at ({}:{}), rptSrcType={}!",
+ this.getCachedSrcName(), srcHost, srcPort, rptSrcType);
}
@Override
diff --git
a/inlong-dataproxy/dataproxy-source/src/main/java/org/apache/inlong/dataproxy/source/SourceConstants.java
b/inlong-dataproxy/dataproxy-source/src/main/java/org/apache/inlong/dataproxy/source/SourceConstants.java
index 67758db01c..9529aa38ff 100644
---
a/inlong-dataproxy/dataproxy-source/src/main/java/org/apache/inlong/dataproxy/source/SourceConstants.java
+++
b/inlong-dataproxy/dataproxy-source/src/main/java/org/apache/inlong/dataproxy/source/SourceConstants.java
@@ -29,6 +29,8 @@ public class SourceConstants {
public static final String SRCCXT_CONFIG_PORT = "port";
// system env source port
public static final String SYSENV_HOST_PORT = "inlongHostPort";
+ // source logic type name
+ public static final String SRCCXT_LOGIC_TYPE_NAME = "logic-type-name";
// message factory name
public static final String SRCCXT_MSG_FACTORY_NAME = "msg-factory-name";
// message handler name
diff --git
a/inlong-dataproxy/dataproxy-source/src/test/java/org/apache/inlong/dataproxy/config/holder/TestSourceReportHolder.java
b/inlong-dataproxy/dataproxy-source/src/test/java/org/apache/inlong/dataproxy/config/holder/TestSourceReportHolder.java
index 705a034444..b28cb96468 100644
---
a/inlong-dataproxy/dataproxy-source/src/test/java/org/apache/inlong/dataproxy/config/holder/TestSourceReportHolder.java
+++
b/inlong-dataproxy/dataproxy-source/src/test/java/org/apache/inlong/dataproxy/config/holder/TestSourceReportHolder.java
@@ -17,11 +17,14 @@
package org.apache.inlong.dataproxy.config.holder;
+import org.apache.inlong.common.heartbeat.AddressInfo;
import org.apache.inlong.dataproxy.config.ConfigManager;
import org.junit.Assert;
import org.junit.Test;
+import java.util.Map;
+
/**
* Test for {@link SourceReportConfigHolder}
*/
@@ -30,21 +33,20 @@ public class TestSourceReportHolder {
@Test
public void testCase() {
// first get
- SourceReportInfo reportInfo =
ConfigManager.getInstance().getSourceReportInfo();
- Assert.assertEquals(reportInfo.getIp(), "");
- Assert.assertEquals(reportInfo.getPort(), "");
- Assert.assertEquals(reportInfo.getProtocolType(), "");
+ Map<String, AddressInfo> srcAddressInfoMap =
ConfigManager.getInstance().getSrcAddressInfos();
+ Assert.assertTrue(srcAddressInfoMap.isEmpty());
// add
- ConfigManager.getInstance().addSourceReportInfo("0.0.0.0", "46801",
"UDP");
- ConfigManager.getInstance().addSourceReportInfo("0.0.0.0", "46801",
"TCP");
- ConfigManager.getInstance().addSourceReportInfo("127.0.0.1", "46802",
"TCP");
- ConfigManager.getInstance().addSourceReportInfo("127.0.0.1", "46803",
"HTTP");
- ConfigManager.getInstance().addSourceReportInfo("0.0.0.0", "46801",
"TCP");
- ConfigManager.getInstance().addSourceReportInfo("127.0.0.1", "46803",
"HTTP");
+ ConfigManager.getInstance().addSourceReportInfo("0.0.0.0", "46801",
"INLONG", "UDP");
+ ConfigManager.getInstance().addSourceReportInfo("0.0.0.0", "46801",
"INLONG", "TCP");
+ ConfigManager.getInstance().addSourceReportInfo("127.0.0.1", "46802",
"TEST", "TCP");
+ ConfigManager.getInstance().addSourceReportInfo("127.0.0.1", "46803",
"TEST", "HTTP");
+ ConfigManager.getInstance().addSourceReportInfo("0.0.0.0", "46801",
"TEST", "TCP");
+ ConfigManager.getInstance().addSourceReportInfo("127.0.0.1", "46803",
"TEST", "HTTP");
// get and check
- reportInfo = ConfigManager.getInstance().getSourceReportInfo();
- Assert.assertEquals(reportInfo.getIp(),
"0.0.0.0,0.0.0.0,127.0.0.1,127.0.0.1");
- Assert.assertEquals(reportInfo.getPort(), "46801,46801,46802,46803");
- Assert.assertEquals(reportInfo.getProtocolType(), "UDP,TCP,TCP,HTTP");
+ srcAddressInfoMap = ConfigManager.getInstance().getSrcAddressInfos();
+ Assert.assertEquals(srcAddressInfoMap.size(), 4);
+ String recordKey = "127.0.0.1" + "#" + "46803" + "#" + "HTTP";
+ AddressInfo addressInfo = srcAddressInfoMap.get(recordKey);
+ Assert.assertEquals(addressInfo.getReportSourceType(), "TEST");
}
}
