This is an automated email from the ASF dual-hosted git repository.
wenweihuang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
The following commit(s) were added to refs/heads/master by this push:
new c6892951ed [INLONG-10319][Agent] Get audit Id from audit sdk (#10322)
c6892951ed is described below
commit c6892951ed37c9153d1cdc7f4106cd961155d659
Author: justinwwhuang <[email protected]>
AuthorDate: Thu May 30 18:10:03 2024 +0800
[INLONG-10319][Agent] Get audit Id from audit sdk (#10322)
* [INLONG-10319][Agent] Get audit Id from audit sdk
* [INLONG-10319][Agent] Get audit Id from audit sdk
* [INLONG-10319][Agent] Get audit Id from audit sdk
---
.../inlong/agent/metrics/audit/AuditUtils.java | 49 ++++----
.../inlong/agent/metrics/TestAuditUtils.java | 139 +++++++++++++++++++++
.../inlong/agent/plugin/task/AbstractTask.java | 1 +
.../apache/inlong/agent/plugin/task/KafkaTask.java | 1 -
.../inlong/agent/plugin/task/MongoDBTask.java | 1 -
.../inlong/agent/plugin/task/PulsarTask.java | 1 -
.../java/org/apache/inlong/audit/MetricIdEnum.java | 72 +++++++++++
7 files changed, 236 insertions(+), 28 deletions(-)
diff --git
a/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/metrics/audit/AuditUtils.java
b/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/metrics/audit/AuditUtils.java
index 7b5edb5a0b..2ed9097775 100644
---
a/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/metrics/audit/AuditUtils.java
+++
b/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/metrics/audit/AuditUtils.java
@@ -42,31 +42,30 @@ public class AuditUtils {
public static final String AUDIT_DEFAULT_FILE_PATH = "/data/inlong/audit/";
public static final String AUDIT_KEY_MAX_CACHE_ROWS = "audit.maxCacheRows";
public static final int AUDIT_DEFAULT_MAX_CACHE_ROWS = 2000000;
- public static final int AUDIT_ID_AGENT_READ_SUCCESS = 3;
- public static final int AUDIT_ID_AGENT_SEND_SUCCESS = 4;
- public static final int AUDIT_ID_AGENT_READ_FAILED = 10003;
- public static final int AUDIT_ID_AGENT_SEND_FAILED = 10004;
- public static final int AUDIT_ID_AGENT_READ_SUCCESS_REAL_TIME = 30001;
- public static final int AUDIT_ID_AGENT_SEND_SUCCESS_REAL_TIME = 30002;
- public static final int AUDIT_ID_AGENT_ADD_INSTANCE_MEM = 30003;
- public static final int AUDIT_ID_AGENT_DEL_INSTANCE_MEM = 30004;
- public static final int AUDIT_ID_AGENT_ADD_INSTANCE_DB = 30005;
- public static final int AUDIT_ID_AGENT_DEL_INSTANCE_DB = 30006;
- public static final int AUDIT_ID_AGENT_TASK_MGR_HEARTBEAT = 30007;
- public static final int AUDIT_ID_AGENT_TASK_HEARTBEAT = 30008;
- public static final int AUDIT_ID_AGENT_INSTANCE_MGR_HEARTBEAT = 30009;
- public static final int AUDIT_ID_AGENT_INSTANCE_HEARTBEAT = 30010;
- public static final int AUDIT_ID_AGENT_SEND_FAILED_REAL_TIME = 30011;
- public static final int AUDIT_ID_AGENT_READ_FAILED_REAL_TIME = 30012;
- public static final int AUDIT_ID_AGENT_ADD_INSTANCE_MEM_FAILED = 30013;
- public static final int AUDIT_ID_AGENT_DEL_INSTANCE_MEM_UNUSUAL = 30014;
- public static final int AUDIT_ID_AGENT_TRY_SEND = 30020;
- public static final int AUDIT_ID_AGENT_TRY_SEND_REAL_TIME = 30021;
- public static final int AUDIT_ID_AGENT_SEND_EXCEPTION = 30022;
- public static final int AUDIT_ID_AGENT_SEND_EXCEPTION_REAL_TIME = 30023;
- public static final int AUDIT_ID_AGENT_RESEND = 30024;
- public static final int AUDIT_ID_AGENT_RESEND_REAL_TIME = 30025;
-
+ public static int AUDIT_ID_AGENT_READ_SUCCESS = 3;
+ public static int AUDIT_ID_AGENT_SEND_SUCCESS = 4;
+ public static int AUDIT_ID_AGENT_READ_FAILED = 524291;
+ public static int AUDIT_ID_AGENT_SEND_FAILED = 524292;
+ public static int AUDIT_ID_AGENT_RESEND = 65540;
+ public static int AUDIT_ID_AGENT_READ_SUCCESS_REAL_TIME = 1073741825;
+ public static int AUDIT_ID_AGENT_SEND_SUCCESS_REAL_TIME = 1073741826;
+ public static int AUDIT_ID_AGENT_READ_FAILED_REAL_TIME = 1073741827;
+ public static int AUDIT_ID_AGENT_SEND_FAILED_REAL_TIME = 1073741828;
+ public static int AUDIT_ID_AGENT_TRY_SEND_REAL_TIME = 1073741829;
+ public static int AUDIT_ID_AGENT_SEND_EXCEPTION_REAL_TIME = 1073741830;
+ public static int AUDIT_ID_AGENT_RESEND_REAL_TIME = 1073741831;
+ public static int AUDIT_ID_AGENT_SEND_EXCEPTION = 1073741832;
+ public static int AUDIT_ID_AGENT_TRY_SEND = 1073741833;
+ public static int AUDIT_ID_AGENT_ADD_INSTANCE_DB = 1073741834;
+ public static int AUDIT_ID_AGENT_DEL_INSTANCE_DB = 1073741835;
+ public static int AUDIT_ID_AGENT_ADD_INSTANCE_MEM = 1073741836;
+ public static int AUDIT_ID_AGENT_DEL_INSTANCE_MEM = 1073741837;
+ public static int AUDIT_ID_AGENT_TASK_MGR_HEARTBEAT = 1073741838;
+ public static int AUDIT_ID_AGENT_TASK_HEARTBEAT = 1073741839;
+ public static int AUDIT_ID_AGENT_INSTANCE_MGR_HEARTBEAT = 1073741840;
+ public static int AUDIT_ID_AGENT_INSTANCE_HEARTBEAT = 1073741841;
+ public static int AUDIT_ID_AGENT_ADD_INSTANCE_MEM_FAILED = 1073741842;
+ public static int AUDIT_ID_AGENT_DEL_INSTANCE_MEM_UNUSUAL = 1073741843;
private static boolean IS_AUDIT = true;
/**
diff --git
a/inlong-agent/agent-common/src/test/java/org/apache/inlong/agent/metrics/TestAuditUtils.java
b/inlong-agent/agent-common/src/test/java/org/apache/inlong/agent/metrics/TestAuditUtils.java
new file mode 100755
index 0000000000..e34bea6af3
--- /dev/null
+++
b/inlong-agent/agent-common/src/test/java/org/apache/inlong/agent/metrics/TestAuditUtils.java
@@ -0,0 +1,139 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.agent.metrics;
+
+import org.apache.inlong.agent.metrics.audit.AuditUtils;
+import org.apache.inlong.audit.AuditIdEnum;
+import org.apache.inlong.audit.AuditOperator;
+import org.apache.inlong.audit.MetricIdEnum;
+
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class TestAuditUtils {
+
+ private static final Logger LOGGER =
LoggerFactory.getLogger(TestAuditUtils.class);
+
+ @BeforeClass
+ public static void setup() {
+
+ }
+
+ @Test
+ public void testAuditId() {
+ int expected = AuditUtils.AUDIT_ID_AGENT_READ_SUCCESS;
+ int real =
AuditOperator.getInstance().buildSuccessfulAuditId(AuditIdEnum.AGENT_INPUT);
+ Assert.assertTrue(real + " != " + expected, expected == real);
+
+ expected = AuditUtils.AUDIT_ID_AGENT_SEND_SUCCESS;
+ real =
AuditOperator.getInstance().buildSuccessfulAuditId(AuditIdEnum.AGENT_OUTPUT);
+ Assert.assertTrue(real + " != " + expected, expected == real);
+
+ expected = AuditUtils.AUDIT_ID_AGENT_READ_FAILED;
+ real =
AuditOperator.getInstance().buildFailedAuditId(AuditIdEnum.AGENT_INPUT);
+ Assert.assertTrue(real + " != " + expected, expected == real);
+
+ expected = AuditUtils.AUDIT_ID_AGENT_SEND_FAILED;
+ real =
AuditOperator.getInstance().buildFailedAuditId(AuditIdEnum.AGENT_OUTPUT);
+ Assert.assertTrue(real + " != " + expected, expected == real);
+
+ expected = AuditUtils.AUDIT_ID_AGENT_RESEND;
+ real =
AuditOperator.getInstance().buildRetryAuditId(AuditIdEnum.AGENT_OUTPUT);
+ Assert.assertTrue(real + " != " + expected, expected == real);
+
+ expected = AuditUtils.AUDIT_ID_AGENT_READ_SUCCESS_REAL_TIME;
+ real = MetricIdEnum.AGENT_READ_SUCCESS_REAL_TIME.getValue();
+ Assert.assertTrue(real + " != " + expected, expected == real);
+
+ expected = AuditUtils.AUDIT_ID_AGENT_SEND_SUCCESS_REAL_TIME;
+ real = MetricIdEnum.AGENT_SEND_SUCCESS_REAL_TIME.getValue();
+ Assert.assertTrue(real + " != " + expected, expected == real);
+
+ expected = AuditUtils.AUDIT_ID_AGENT_READ_FAILED_REAL_TIME;
+ real = MetricIdEnum.AGENT_READ_FAILED_REAL_TIME.getValue();
+ Assert.assertTrue(real + " != " + expected, expected == real);
+
+ expected = AuditUtils.AUDIT_ID_AGENT_SEND_FAILED_REAL_TIME;
+ real = MetricIdEnum.AGENT_SEND_FAILED_REAL_TIME.getValue();
+ Assert.assertTrue(real + " != " + expected, expected == real);
+
+ expected = AuditUtils.AUDIT_ID_AGENT_TRY_SEND_REAL_TIME;
+ real = MetricIdEnum.AGENT_TRY_SEND_REAL_TIME.getValue();
+ Assert.assertTrue(real + " != " + expected, expected == real);
+
+ expected = AuditUtils.AUDIT_ID_AGENT_SEND_EXCEPTION_REAL_TIME;
+ real = MetricIdEnum.AGENT_SEND_EXCEPTION_REAL_TIME.getValue();
+ Assert.assertTrue(real + " != " + expected, expected == real);
+
+ expected = AuditUtils.AUDIT_ID_AGENT_RESEND_REAL_TIME;
+ real = MetricIdEnum.AGENT_RESEND_REAL_TIME.getValue();
+ Assert.assertTrue(real + " != " + expected, expected == real);
+
+ expected = AuditUtils.AUDIT_ID_AGENT_SEND_EXCEPTION;
+ real = MetricIdEnum.AGENT_SEND_EXCEPTION.getValue();
+ Assert.assertTrue(real + " != " + expected, expected == real);
+
+ expected = AuditUtils.AUDIT_ID_AGENT_TRY_SEND;
+ real = MetricIdEnum.AGENT_TRY_SEND.getValue();
+ Assert.assertTrue(real + " != " + expected, expected == real);
+
+ expected = AuditUtils.AUDIT_ID_AGENT_ADD_INSTANCE_DB;
+ real = MetricIdEnum.AGENT_ADD_INSTANCE_DB.getValue();
+ Assert.assertTrue(real + " != " + expected, expected == real);
+
+ expected = AuditUtils.AUDIT_ID_AGENT_DEL_INSTANCE_DB;
+ real = MetricIdEnum.AGENT_DEL_INSTANCE_DB.getValue();
+ Assert.assertTrue(real + " != " + expected, expected == real);
+
+ expected = AuditUtils.AUDIT_ID_AGENT_ADD_INSTANCE_MEM;
+ real = MetricIdEnum.AGENT_ADD_INSTANCE_MEM.getValue();
+ Assert.assertTrue(real + " != " + expected, expected == real);
+
+ expected = AuditUtils.AUDIT_ID_AGENT_DEL_INSTANCE_MEM;
+ real = MetricIdEnum.AGENT_DEL_INSTANCE_MEM.getValue();
+ Assert.assertTrue(real + " != " + expected, expected == real);
+
+ expected = AuditUtils.AUDIT_ID_AGENT_TASK_MGR_HEARTBEAT;
+ real = MetricIdEnum.AGENT_TASK_MGR_HEARTBEAT.getValue();
+ Assert.assertTrue(real + " != " + expected, expected == real);
+
+ expected = AuditUtils.AUDIT_ID_AGENT_TASK_HEARTBEAT;
+ real = MetricIdEnum.AGENT_TASK_HEARTBEAT.getValue();
+ Assert.assertTrue(real + " != " + expected, expected == real);
+
+ expected = AuditUtils.AUDIT_ID_AGENT_INSTANCE_MGR_HEARTBEAT;
+ real = MetricIdEnum.AGENT_INSTANCE_MGR_HEARTBEAT.getValue();
+ Assert.assertTrue(real + " != " + expected, expected == real);
+
+ expected = AuditUtils.AUDIT_ID_AGENT_INSTANCE_HEARTBEAT;
+ real = MetricIdEnum.AGENT_INSTANCE_HEARTBEAT.getValue();
+ Assert.assertTrue(real + " != " + expected, expected == real);
+
+ expected = AuditUtils.AUDIT_ID_AGENT_ADD_INSTANCE_MEM_FAILED;
+ real = MetricIdEnum.AGENT_ADD_INSTANCE_MEM_FAILED.getValue();
+ Assert.assertTrue(real + " != " + expected, expected == real);
+
+ expected = AuditUtils.AUDIT_ID_AGENT_DEL_INSTANCE_MEM_UNUSUAL;
+ real = MetricIdEnum.AGENT_DEL_INSTANCE_MEM_UNUSUAL.getValue();
+ Assert.assertTrue(real + " != " + expected, expected == real);
+ }
+
+}
diff --git
a/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/task/AbstractTask.java
b/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/task/AbstractTask.java
index acf8287817..2de853b596 100644
---
a/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/task/AbstractTask.java
+++
b/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/task/AbstractTask.java
@@ -42,6 +42,7 @@ public abstract class AbstractTask extends Task {
private static final Logger LOGGER =
LoggerFactory.getLogger(AbstractTask.class);
public static final int CORE_THREAD_SLEEP_TIME = 1000;
public static final int CORE_THREAD_PRINT_TIME = 10000;
+ protected static final int DEFAULT_INSTANCE_LIMIT = 1;
protected TaskProfile taskProfile;
protected Db basicDb;
protected TaskManager taskManager;
diff --git
a/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/task/KafkaTask.java
b/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/task/KafkaTask.java
index e7058c9929..86b06079ab 100644
---
a/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/task/KafkaTask.java
+++
b/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/task/KafkaTask.java
@@ -36,7 +36,6 @@ public class KafkaTask extends AbstractTask {
private static final Logger LOGGER =
LoggerFactory.getLogger(KafkaTask.class);
public static final String DEFAULT_KAFKA_INSTANCE =
"org.apache.inlong.agent.plugin.instance.KafkaInstance";
- public static final int DEFAULT_INSTANCE_LIMIT = 1;
private boolean isAdded = false;
private String topic;
private final DateTimeFormatter dateTimeFormatter =
DateTimeFormatter.ofPattern("yyyyMMddHH");
diff --git
a/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/task/MongoDBTask.java
b/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/task/MongoDBTask.java
index 01f97029b2..4a3ef07e26 100644
---
a/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/task/MongoDBTask.java
+++
b/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/task/MongoDBTask.java
@@ -35,7 +35,6 @@ public class MongoDBTask extends AbstractTask {
private static final Logger LOGGER =
LoggerFactory.getLogger(MongoDBTask.class);
public static final String DEFAULT_MONGODB_INSTANCE =
"org.apache.inlong.agent.plugin.instance.MongoDBInstance";
- public static final int DEFAULT_INSTANCE_LIMIT = 1;
private boolean isAdded = false;
private String collection;
private final DateTimeFormatter dateTimeFormatter =
DateTimeFormatter.ofPattern("yyyyMMddHH");
diff --git
a/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/task/PulsarTask.java
b/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/task/PulsarTask.java
index 62b32dcbca..4586da7bb1 100644
---
a/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/task/PulsarTask.java
+++
b/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/task/PulsarTask.java
@@ -38,7 +38,6 @@ public class PulsarTask extends AbstractTask {
private static final Logger LOGGER =
LoggerFactory.getLogger(PulsarTask.class);
public static final String DEFAULT_PULSAR_INSTANCE =
"org.apache.inlong.agent.plugin.instance.PulsarInstance";
- public static final int DEFAULT_INSTANCE_LIMIT = 1;
private boolean isAdded = false;
private String tenant;
private String namespace;
diff --git
a/inlong-audit/audit-sdk/src/main/java/org/apache/inlong/audit/MetricIdEnum.java
b/inlong-audit/audit-sdk/src/main/java/org/apache/inlong/audit/MetricIdEnum.java
new file mode 100644
index 0000000000..aa8c535a8b
--- /dev/null
+++
b/inlong-audit/audit-sdk/src/main/java/org/apache/inlong/audit/MetricIdEnum.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.audit;
+
+import org.apache.inlong.audit.util.AuditManagerUtils;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Metric item management, each module is assigned two baseline audit item
IDs, namely receiving and sending.
+ */
+public enum MetricIdEnum {
+
+ AGENT_READ_SUCCESS_REAL_TIME(1, "Agent 读取成功-监控", "Agent Read
Success-Metric"),
+ AGENT_SEND_SUCCESS_REAL_TIME(2, "Agent 发送成功-监控", "Agent Send
Success-Metric"),
+ AGENT_READ_FAILED_REAL_TIME(3, "Agent 读取失败-监控", "Agent Read
Failed-Metric"),
+ AGENT_SEND_FAILED_REAL_TIME(4, "Agent 发送失败-监控", "Agent Send
Failed-Metric"),
+ AGENT_TRY_SEND_REAL_TIME(5, "Agent 尝试发送-监控", "Agent Try Send-Metric"),
+ AGENT_SEND_EXCEPTION_REAL_TIME(6, "Agent 发送异常-监控", "Agent Send
Exception-Metric"),
+ AGENT_RESEND_REAL_TIME(7, "Agent 重发-监控", "Agent Resend-Metric"),
+ AGENT_SEND_EXCEPTION(8, "Agent 发送异常", "Agent Send Exception"),
+ AGENT_TRY_SEND(9, "Agent 尝试发送", "Agent Try Send"),
+ AGENT_ADD_INSTANCE_DB(10, "Agent 增加实例-DB", "Agent Add Instance-DB"),
+ AGENT_DEL_INSTANCE_DB(11, "Agent 删除实例-DB", "Agent Delete Instance-DB"),
+ AGENT_ADD_INSTANCE_MEM(12, "Agent 增加实例-内存", "Agent Add Instance-Mem"),
+ AGENT_DEL_INSTANCE_MEM(13, "Agent 删除实例-内存", "Agent Delete Instance-Mem"),
+ AGENT_TASK_MGR_HEARTBEAT(14, "Agent 任务管理器心跳", "Agent Task Manager
Heartbeat"),
+ AGENT_TASK_HEARTBEAT(15, "Agent 任务心跳", "Agent Task Heartbeat"),
+ AGENT_INSTANCE_MGR_HEARTBEAT(16, "Agent 实例管理器心跳", "Agent Instance Manager
Heartbeat"),
+ AGENT_INSTANCE_HEARTBEAT(17, "Agent 实例心跳", "Agent Instance Heartbeat"),
+ AGENT_ADD_INSTANCE_MEM_FAILED(18, "Agent 增加实例失败-内存", "Agent add Instance
Failed-Mem"),
+ AGENT_DEL_INSTANCE_MEM_UNUSUAL(19, "Agent 删除实例异常-内存", "Agent Delete
Instance Unusual-Mem");
+
+ private static final Logger LOGGER =
LoggerFactory.getLogger(MetricIdEnum.class);
+ private final int metricId;
+ private final String chineseDescription;
+ private final String englishDescription;
+
+ MetricIdEnum(int metricId, String chineseDescription, String
englishDescription) {
+ this.metricId = metricId;
+ this.chineseDescription = chineseDescription;
+ this.englishDescription = englishDescription;
+ }
+
+ public int getValue() {
+ return metricId + AuditManagerUtils.getStartAuditIdForMetric();
+ }
+
+ public String getChineseDescription() {
+ return chineseDescription;
+ }
+
+ public String getEnglishDescription() {
+ return englishDescription;
+ }
+}
