This is an automated email from the ASF dual-hosted git repository.
wenweihuang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
The following commit(s) were added to refs/heads/master by this push:
new 229174675b [INLONG-10298][Agent] Delete useless code (#10299)
229174675b is described below
commit 229174675bf9b687f1fdd2605e670b20b6272944
Author: justinwwhuang <[email protected]>
AuthorDate: Wed May 29 20:17:13 2024 +0800
[INLONG-10298][Agent] Delete useless code (#10299)
---
.../java/org/apache/inlong/agent/db/CommandDb.java | 61 ------------
.../main/java/org/apache/inlong/agent/db/Db.java | 75 +--------------
.../org/apache/inlong/agent/db/RocksDbImp.java | 104 +--------------------
.../org/apache/inlong/agent/db/TestRocksDbImp.java | 20 +---
.../apache/inlong/agent/core/task/TaskManager.java | 4 +
.../inlong/agent/plugin/task/TestLogFileTask.java | 5 +-
6 files changed, 10 insertions(+), 259 deletions(-)
diff --git
a/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/db/CommandDb.java
b/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/db/CommandDb.java
deleted file mode 100644
index 393b4eda8e..0000000000
---
a/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/db/CommandDb.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.inlong.agent.db;
-
-import org.apache.inlong.common.constant.Constants;
-import org.apache.inlong.common.db.CommandEntity;
-
-import java.util.List;
-
-/**
- * Command for database
- */
-public class CommandDb {
-
- private final Db db;
-
- public CommandDb(Db db) {
- this.db = db;
- }
-
- /**
- * store manager command to db
- */
- public void storeCommand(CommandEntity commandEntity) {
- db.putCommand(commandEntity);
- }
-
- /**
- * get those commands not ack to manager
- */
- public List<CommandEntity> getUnackedCommands() {
- return db.searchCommands(false);
- }
-
- /**
- * save special command result for trigger (retry\makeup\check)
- */
- public void saveSpecialCmds(Integer id, Integer taskId, boolean success) {
- CommandEntity entity = new CommandEntity();
- entity.setId(String.valueOf(id));
- entity.setTaskId(taskId);
- entity.setAcked(false);
- entity.setCommandResult(success ? Constants.RESULT_SUCCESS :
Constants.RESULT_FAIL);
- storeCommand(entity);
- }
-}
diff --git
a/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/db/Db.java
b/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/db/Db.java
index b1a0e897c9..8043c911fc 100644
--- a/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/db/Db.java
+++ b/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/db/Db.java
@@ -17,10 +17,6 @@
package org.apache.inlong.agent.db;
-import org.apache.inlong.common.db.CommandEntity;
-
-import javax.management.openmbean.KeyAlreadyExistsException;
-
import java.io.Closeable;
import java.util.List;
@@ -29,35 +25,14 @@ import java.util.List;
*/
public interface Db extends Closeable {
- abstract KeyValueEntity get(String key);
-
- /**
- * get command by command id
- */
- CommandEntity getCommand(String commandId);
-
- /**
- * put command entity in db
- */
- CommandEntity putCommand(CommandEntity entity);
-
- /**
- * store keyValue, if key has exists, throw exception.
- *
- * @param entity key/value
- * @throws NullPointerException key should not be null
- * @throws KeyAlreadyExistsException key already exists
- */
- void set(KeyValueEntity entity);
+ KeyValueEntity get(String key);
/**
* store keyValue, if key has exists, overwrite it.
*
* @param entity key/value
- * @return null or old value which is overwritten.
- * @throws NullPointerException key should not be null.
*/
- KeyValueEntity put(KeyValueEntity entity);
+ void put(KeyValueEntity entity);
/**
* remove keyValue by key.
@@ -68,52 +43,6 @@ public interface Db extends Closeable {
*/
KeyValueEntity remove(String key);
- /**
- * search keyValue list by search key.
- *
- * @param searchKey search keys.
- * @return key/value list
- * @throws NullPointerException search key should not be null.
- */
- List<KeyValueEntity> search(StateSearchKey searchKey);
-
- /**
- * search keyValue list by search key.
- *
- * @param searchKeys search keys.
- * @return key/value list
- * @throws NullPointerException search key should not be null.
- */
- List<KeyValueEntity> search(List<StateSearchKey> searchKeys);
-
- /**
- * search keyValue list by search key.
- *
- * @param searchKey search keys.
- * @param keyPrefix key prefix.
- * @return key/value list
- * @throws NullPointerException search key should not be null.
- */
- List<KeyValueEntity> searchWithKeyPrefix(StateSearchKey searchKey, String
keyPrefix);
-
- /**
- * search commands using ack status
- */
- List<CommandEntity> searchCommands(boolean isAcked);
-
- /**
- * search one keyValue by search key
- *
- * @param searchKey search key
- * @return null or keyValue
- */
- KeyValueEntity searchOne(StateSearchKey searchKey);
-
- /**
- * search one keyValue by fileName
- */
- KeyValueEntity searchOne(String fileName);
-
/**
* find all by prefix key.
*
diff --git
a/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/db/RocksDbImp.java
b/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/db/RocksDbImp.java
index eac8479aa4..02edcc6165 100755
---
a/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/db/RocksDbImp.java
+++
b/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/db/RocksDbImp.java
@@ -19,7 +19,6 @@ package org.apache.inlong.agent.db;
import org.apache.inlong.agent.conf.AgentConfiguration;
import org.apache.inlong.agent.constant.AgentConstants;
-import org.apache.inlong.common.db.CommandEntity;
import com.google.gson.Gson;
import org.rocksdb.AbstractImmutableNativeReference;
@@ -40,7 +39,6 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
-import java.util.Objects;
import java.util.concurrent.ConcurrentHashMap;
import java.util.stream.Collectors;
@@ -159,30 +157,13 @@ public class RocksDbImp implements Db {
}
@Override
- public CommandEntity getCommand(String commandId) {
- return null;
- }
-
- @Override
- public CommandEntity putCommand(CommandEntity entity) {
- return null;
- }
-
- @Override
- public void set(KeyValueEntity entity) {
- requireNonNull(entity);
- put(entity);
- }
-
- @Override
- public KeyValueEntity put(KeyValueEntity entity) {
+ public void put(KeyValueEntity entity) {
requireNonNull(entity);
try {
db.put(columnHandlesMap.get(defaultFamilyName),
entity.getKey().getBytes(), GSON.toJson(entity).getBytes());
} catch (Exception e) {
throw new RuntimeException("put value to rocks db error", e);
}
- return entity;
}
@Override
@@ -201,89 +182,6 @@ public class RocksDbImp implements Db {
}
}
- @Override
- public List<KeyValueEntity> searchWithKeyPrefix(StateSearchKey searchKey,
String keyPrefix) {
- List<KeyValueEntity> results = new LinkedList<>();
- try (final RocksIterator it =
db.newIterator(columnHandlesMap.get(defaultFamilyName))) {
- it.seekToFirst();
- while (it.isValid()) {
- KeyValueEntity keyValue = GSON.fromJson(new
String(it.value()), KeyValueEntity.class);
- if (keyValue.getStateSearchKey().equals(searchKey) &&
keyValue.getKey().startsWith(keyPrefix)) {
- results.add(keyValue);
- }
- it.next();
- }
- }
- return results;
- }
-
- @Override
- public List<CommandEntity> searchCommands(boolean isAcked) {
- return null;
- }
-
- @Override
- public List<KeyValueEntity> search(StateSearchKey searchKey) {
- List<KeyValueEntity> results = new LinkedList<>();
- try (final RocksIterator it =
db.newIterator(columnHandlesMap.get(defaultFamilyName))) {
- it.seekToFirst();
- while (it.isValid()) {
- KeyValueEntity keyValue = GSON.fromJson(new
String(it.value()), KeyValueEntity.class);
- if (keyValue.getStateSearchKey().equals(searchKey)) {
- results.add(keyValue);
- }
- it.next();
- }
- }
- return results;
- }
-
- @Override
- public List<KeyValueEntity> search(List<StateSearchKey> searchKeys) {
- List<KeyValueEntity> results = new LinkedList<>();
- try (final RocksIterator it =
db.newIterator(columnHandlesMap.get(defaultFamilyName))) {
- it.seekToFirst();
- while (it.isValid()) {
- KeyValueEntity keyValue = GSON.fromJson(new
String(it.value()), KeyValueEntity.class);
- if (Objects.nonNull(keyValue) &&
searchKeys.contains(keyValue.getStateSearchKey())) {
- results.add(keyValue);
- }
- it.next();
- }
- }
- return results;
- }
-
- @Override
- public KeyValueEntity searchOne(StateSearchKey searchKey) {
- try (final RocksIterator it =
db.newIterator(columnHandlesMap.get(defaultFamilyName))) {
- it.seekToFirst();
- while (it.isValid()) {
- KeyValueEntity keyValue = GSON.fromJson(new
String(it.value()), KeyValueEntity.class);
- if (keyValue.getStateSearchKey().equals(searchKey)) {
- return keyValue;
- }
- it.next();
- }
- }
- return null;
- }
-
- @Override
- public KeyValueEntity searchOne(String fileName) {
- try (final RocksIterator it =
db.newIterator(columnHandlesMap.get(defaultFamilyName))) {
- it.seekToFirst();
- while (it.isValid()) {
- KeyValueEntity keyValue = GSON.fromJson(new
String(it.value()), KeyValueEntity.class);
- if (keyValue.getFileName().equals(fileName)) {
- return keyValue;
- }
- it.next();
- }
- }
- return null;
- }
-
@Override
public List<KeyValueEntity> findAll(String prefix) {
List<KeyValueEntity> results = new LinkedList<>();
diff --git
a/inlong-agent/agent-common/src/test/java/org/apache/inlong/agent/db/TestRocksDbImp.java
b/inlong-agent/agent-common/src/test/java/org/apache/inlong/agent/db/TestRocksDbImp.java
index eccb9c298e..3e51c4a891 100644
---
a/inlong-agent/agent-common/src/test/java/org/apache/inlong/agent/db/TestRocksDbImp.java
+++
b/inlong-agent/agent-common/src/test/java/org/apache/inlong/agent/db/TestRocksDbImp.java
@@ -25,7 +25,6 @@ import org.junit.BeforeClass;
import org.junit.Test;
import java.io.IOException;
-import java.util.List;
public class TestRocksDbImp {
@@ -61,15 +60,6 @@ public class TestRocksDbImp {
entity.setStateSearchKey(keys);
entity1.setStateSearchKey(keys);
- db.set(entity);
- db.set(entity1);
-
- List<KeyValueEntity> entityList = db.search(keys);
- for (KeyValueEntity keyValueEntity : entityList) {
- Assert.assertEquals(StateSearchKey.SUCCESS,
keyValueEntity.getStateSearchKey());
- }
- Assert.assertEquals(2, entityList.size());
-
entity.setJsonValue("testC");
db.put(entity);
KeyValueEntity newEntity = db.get("test1");
@@ -81,15 +71,7 @@ public class TestRocksDbImp {
KeyValueEntity entity = new KeyValueEntity("searchKey1",
"searchResult1", "test");
db.put(entity);
db.remove("searchKey1");
- KeyValueEntity entityResult = db.searchOne(StateSearchKey.ACCEPTED);
+ KeyValueEntity entityResult = db.get("searchKey1");
Assert.assertNull(entityResult);
}
-
- @Test
- public void testFileNameSearch() {
- KeyValueEntity entity = new KeyValueEntity("searchKey1",
"searchResult1", "test");
- db.put(entity);
- KeyValueEntity entityResult = db.searchOne(StateSearchKey.ACCEPTED);
- Assert.assertEquals("searchKey1", entityResult.getKey());
- }
}
diff --git
a/inlong-agent/agent-core/src/main/java/org/apache/inlong/agent/core/task/TaskManager.java
b/inlong-agent/agent-core/src/main/java/org/apache/inlong/agent/core/task/TaskManager.java
index 7fded1fcbd..58bd274c8b 100644
---
a/inlong-agent/agent-core/src/main/java/org/apache/inlong/agent/core/task/TaskManager.java
+++
b/inlong-agent/agent-core/src/main/java/org/apache/inlong/agent/core/task/TaskManager.java
@@ -149,6 +149,10 @@ public class TaskManager extends AbstractDaemon {
return taskDb;
}
+ public Db getInstanceBasicDb() {
+ return instanceBasicDb;
+ }
+
/**
* init db by class name
*
diff --git
a/inlong-agent/agent-plugins/src/test/java/org/apache/inlong/agent/plugin/task/TestLogFileTask.java
b/inlong-agent/agent-plugins/src/test/java/org/apache/inlong/agent/plugin/task/TestLogFileTask.java
index 37fcb158a6..b538c69f61 100644
---
a/inlong-agent/agent-plugins/src/test/java/org/apache/inlong/agent/plugin/task/TestLogFileTask.java
+++
b/inlong-agent/agent-plugins/src/test/java/org/apache/inlong/agent/plugin/task/TestLogFileTask.java
@@ -21,7 +21,6 @@ import org.apache.inlong.agent.common.AgentThreadFactory;
import org.apache.inlong.agent.conf.TaskProfile;
import org.apache.inlong.agent.constant.TaskConstants;
import org.apache.inlong.agent.core.task.TaskManager;
-import org.apache.inlong.agent.db.Db;
import org.apache.inlong.agent.plugin.AgentBaseTestsHelper;
import org.apache.inlong.agent.plugin.task.file.LogFileTask;
import org.apache.inlong.common.enums.TaskStateEnum;
@@ -74,7 +73,6 @@ public class TestLogFileTask {
@BeforeClass
public static void setup() {
helper = new
AgentBaseTestsHelper(TestLogFileTask.class.getName()).setupAgentHome();
- Db basicDb = TaskManager.initDb("/localdb");
resourceName =
LOADER.getResource("testScan/20230928_1/test_1.txt").getPath();
tempResourceName = LOADER.getResource("testScan/temp.txt").getPath();
File f = new File(tempResourceName);
@@ -97,7 +95,8 @@ public class TestLogFileTask {
return null;
}).when(task, "addToEvenMap", Mockito.anyString(),
Mockito.anyString());
Assert.assertTrue(task.isProfileValid(taskProfile));
- task.init(manager, taskProfile, basicDb);
+ manager.getTaskDb().storeTask(taskProfile);
+ task.init(manager, taskProfile, manager.getInstanceBasicDb());
EXECUTOR_SERVICE.submit(task);
} catch (Exception e) {
LOGGER.error("source init error {}", e);
