This is an automated email from the ASF dual-hosted git repository.
zhangxiaowei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozhera.git
The following commit(s) were added to refs/heads/master by this push:
new 35dd2b82 fix: fix log file path judgment logic and optimize related
functions (#599)
35dd2b82 is described below
commit 35dd2b824936ff5ccf6b724ffb83d1fdb41bc3f2
Author: wtt <[email protected]>
AuthorDate: Tue Jul 8 14:18:51 2025 +0800
fix: fix log file path judgment logic and optimize related functions (#599)
* fix:fix log file path judgment logic and optimize related functions
* fix: modify-mysql-driver-dependency-configuration
---
ozhera-app/app-service/pom.xml | 7 ++++
.../log/agent/channel/ChannelServiceFactory.java | 8 +++-
.../log/agent/channel/ChannelServiceImpl.java | 5 +--
ozhera-log/log-manager/pom.xml | 43 +++++++++++-----------
.../log/manager/model/vo/LogContextQuery.java | 1 +
.../manager/service/impl/EsDataServiceImpl.java | 2 +-
.../manager/service/impl/LogQueryServiceImpl.java | 14 +++----
.../ozhera/log/stream/plugin/es/EsPlugin.java | 4 ++
8 files changed, 49 insertions(+), 35 deletions(-)
diff --git a/ozhera-app/app-service/pom.xml b/ozhera-app/app-service/pom.xml
index dc2921a2..9319cdc0 100644
--- a/ozhera-app/app-service/pom.xml
+++ b/ozhera-app/app-service/pom.xml
@@ -203,6 +203,13 @@
</includes>
<targetPath>META-INF/</targetPath>
</resource>
+ <resource>
+ <directory>src/main/resources</directory>
+ <filtering>false</filtering>
+ <includes>
+ <include>**/*.xml</include>
+ </includes>
+ </resource>
</resources>
</build>
diff --git
a/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/channel/ChannelServiceFactory.java
b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/channel/ChannelServiceFactory.java
index a3feea86..f49e9876 100644
---
a/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/channel/ChannelServiceFactory.java
+++
b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/channel/ChannelServiceFactory.java
@@ -51,7 +51,7 @@ public class ChannelServiceFactory {
private final String memoryBasePath;
private static final Pattern regexCharsPattern =
Pattern.compile("[*+?^${}()\\[\\]\\\\]");
- private List<String> multiSpecialFileSuffix;
+ private static List<String> multiSpecialFileSuffix;
public ChannelServiceFactory(AgentMemoryService agentMemoryService, String
memoryBasePath) {
this.agentMemoryService = agentMemoryService;
@@ -62,6 +62,10 @@ public class ChannelServiceFactory {
}
}
+ public static boolean isSpecialFilePath(String logPattern) {
+ return CollectionUtils.isNotEmpty(multiSpecialFileSuffix) &&
logPattern.contains("*") &&
multiSpecialFileSuffix.stream().anyMatch(logPattern::endsWith);
+ }
+
public ChannelService createChannelService(ChannelDefine channelDefine,
MsgExporter exporter,
FilterChain filterChain) {
if (channelDefine == null || channelDefine.getInput() == null) {
@@ -72,7 +76,7 @@ public class ChannelServiceFactory {
String logType = input.getType();
String logPattern = input.getLogPattern();
- if (CollectionUtils.isNotEmpty(multiSpecialFileSuffix) &&
multiSpecialFileSuffix.stream().anyMatch(logPattern::endsWith)) {
+ if (isSpecialFilePath(logPattern)) {
return createStandardChannelService(exporter, channelDefine,
filterChain);
}
diff --git
a/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/channel/ChannelServiceImpl.java
b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/channel/ChannelServiceImpl.java
index 6a284fee..7e4ef178 100644
---
a/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/channel/ChannelServiceImpl.java
+++
b/ozhera-log/log-agent/src/main/java/org/apache/ozhera/log/agent/channel/ChannelServiceImpl.java
@@ -257,16 +257,13 @@ public class ChannelServiceImpl extends
AbstractChannelService {
* 2.logSplitExpress:/home/work/log/log-agent/(server.log.*|error.log.*)
realFilePaths:
["/home/work/log/log-agent/server.log","/home/work/log/log-agent/server.log"]
* 2.logSplitExpress:/home/work/log/(log-agent|log-stream)/server.log.*
realFilePaths:
["/home/work/log/log-agent/server.log","/home/work/log/log-stream/server.log"]
* The real file does not exist, it should also listen
- *
- * @param logSplitExpress
- * @param realFilePaths
*/
private void logMonitorPathDisassembled(String logSplitExpress,
List<String> realFilePaths, String configPath) {
List<String> cleanedPathList = Lists.newArrayList();
if (StringUtils.isNotBlank(logSplitExpress)) {
PathUtils.dismantlingStrWithSymbol(logSplitExpress,
cleanedPathList);
}
- if (LogTypeEnum.OPENTELEMETRY == logTypeEnum ||
realFilePaths.isEmpty()) {
+ if (LogTypeEnum.OPENTELEMETRY == logTypeEnum ||
realFilePaths.isEmpty() || ChannelServiceFactory.isSpecialFilePath(configPath))
{
opentelemetryMonitor(configPath);
return;
}
diff --git a/ozhera-log/log-manager/pom.xml b/ozhera-log/log-manager/pom.xml
index 8be0cf3e..b02a601e 100644
--- a/ozhera-log/log-manager/pom.xml
+++ b/ozhera-log/log-manager/pom.xml
@@ -41,6 +41,18 @@ http://www.apache.org/licenses/LICENSE-2.0
<artifactId>spring-context-support</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.yaml</groupId>
+ <artifactId>snakeyaml</artifactId>
+ <version>1.26</version>
+ </dependency>
+
+ <dependency>
+ <groupId>run.mone</groupId>
+ <artifactId>excel</artifactId>
+ <version>1.5.0-jdk21</version>
+ </dependency>
+
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
@@ -94,16 +106,16 @@ http://www.apache.org/licenses/LICENSE-2.0
<version>2.2.6-SNAPSHOT</version>
</dependency>
- <dependency>
- <groupId>run.mone</groupId>
- <artifactId>excel</artifactId>
- <exclusions>
- <exclusion>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
+<!-- <dependency>-->
+<!-- <groupId>run.mone</groupId>-->
+<!-- <artifactId>excel</artifactId>-->
+<!-- <exclusions>-->
+<!-- <exclusion>-->
+<!-- <groupId>log4j</groupId>-->
+<!-- <artifactId>log4j</artifactId>-->
+<!-- </exclusion>-->
+<!-- </exclusions>-->
+<!-- </dependency>-->
<dependency>
<groupId>run.mone</groupId>
<artifactId>docean-plugin-dubbo</artifactId>
@@ -280,17 +292,6 @@ http://www.apache.org/licenses/LICENSE-2.0
<artifactId>hera-trace</artifactId>
</dependency>
- <dependency>
- <groupId>org.apache.ozhera</groupId>
- <artifactId>app-api</artifactId>
- <exclusions>
- <exclusion>
- <groupId>org.projectlombok</groupId>
- <artifactId>lombok</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
<dependency>
<groupId>run.mone</groupId>
<artifactId>hive</artifactId>
diff --git
a/ozhera-log/log-manager/src/main/java/org/apache/ozhera/log/manager/model/vo/LogContextQuery.java
b/ozhera-log/log-manager/src/main/java/org/apache/ozhera/log/manager/model/vo/LogContextQuery.java
index 60f12b8a..ef643b50 100644
---
a/ozhera-log/log-manager/src/main/java/org/apache/ozhera/log/manager/model/vo/LogContextQuery.java
+++
b/ozhera-log/log-manager/src/main/java/org/apache/ozhera/log/manager/model/vo/LogContextQuery.java
@@ -24,6 +24,7 @@ import java.io.Serializable;
@Data
public class LogContextQuery implements Serializable {
private String logstore;
+ private Long logStoreId;
private String ip;
private String fileName;
private Long lineNumber;
diff --git
a/ozhera-log/log-manager/src/main/java/org/apache/ozhera/log/manager/service/impl/EsDataServiceImpl.java
b/ozhera-log/log-manager/src/main/java/org/apache/ozhera/log/manager/service/impl/EsDataServiceImpl.java
index 5575d1c7..8f1a00e1 100644
---
a/ozhera-log/log-manager/src/main/java/org/apache/ozhera/log/manager/service/impl/EsDataServiceImpl.java
+++
b/ozhera-log/log-manager/src/main/java/org/apache/ozhera/log/manager/service/impl/EsDataServiceImpl.java
@@ -637,7 +637,7 @@ public class EsDataServiceImpl implements EsDataService,
LogDataService, EsDataB
if (searchLog.isLegalParam(logContextQuery) == false) {
return Result.failParam("Required parameters are missing");
}
- MilogLogStoreDO milogLogstoreDO =
logstoreDao.getByName(logContextQuery.getLogstore());
+ MilogLogStoreDO milogLogstoreDO =
logstoreDao.queryById(logContextQuery.getLogStoreId());
if (milogLogstoreDO.getEsClusterId() == null ||
StringUtils.isEmpty(milogLogstoreDO.getEsIndex())) {
return Result.failParam("store Configuration exceptions");
}
diff --git
a/ozhera-log/log-manager/src/main/java/org/apache/ozhera/log/manager/service/impl/LogQueryServiceImpl.java
b/ozhera-log/log-manager/src/main/java/org/apache/ozhera/log/manager/service/impl/LogQueryServiceImpl.java
index 292ebd19..fc42f05e 100644
---
a/ozhera-log/log-manager/src/main/java/org/apache/ozhera/log/manager/service/impl/LogQueryServiceImpl.java
+++
b/ozhera-log/log-manager/src/main/java/org/apache/ozhera/log/manager/service/impl/LogQueryServiceImpl.java
@@ -23,6 +23,10 @@ import cn.hutool.core.date.StopWatch;
import com.google.common.collect.Lists;
import com.google.gson.Gson;
import com.xiaomi.mone.es.EsClient;
+import com.xiaomi.youpin.docean.anno.Service;
+import com.xiaomi.youpin.docean.common.StringUtils;
+import com.xiaomi.youpin.docean.plugin.es.EsService;
+import lombok.extern.slf4j.Slf4j;
import org.apache.ozhera.log.api.model.dto.TraceLogDTO;
import org.apache.ozhera.log.api.model.vo.TraceLogQuery;
import org.apache.ozhera.log.api.service.LogDataService;
@@ -45,10 +49,6 @@ import org.apache.ozhera.log.manager.service.LogQueryService;
import
org.apache.ozhera.log.manager.service.extension.common.CommonExtensionService;
import
org.apache.ozhera.log.manager.service.extension.common.CommonExtensionServiceFactory;
import org.apache.ozhera.log.parse.LogParser;
-import com.xiaomi.youpin.docean.anno.Service;
-import com.xiaomi.youpin.docean.common.StringUtils;
-import com.xiaomi.youpin.docean.plugin.es.EsService;
-import lombok.extern.slf4j.Slf4j;
import org.apache.poi.hssf.usermodel.HSSFWorkbook;
import org.elasticsearch.ElasticsearchStatusException;
import org.elasticsearch.action.search.SearchRequest;
@@ -128,7 +128,7 @@ public class LogQueryServiceImpl implements
LogQueryService, LogDataService, EsD
StopWatch stopWatch = new StopWatch("HERA-LOG-QUERY");
try {
stopWatch.start("before-query");
- MilogLogStoreDO logStore =
logstoreDao.getByName(logQuery.getLogstore());
+ MilogLogStoreDO logStore =
logstoreDao.queryById(logQuery.getStoreId());
if (logStore == null) {
log.warn("[EsDataService.logQuery] not find logstore:[{}]",
logQuery.getLogstore());
return Result.failParam("Not found [" + logQuery.getLogstore()
+ "]The corresponding data");
@@ -223,7 +223,7 @@ public class LogQueryServiceImpl implements
LogQueryService, LogDataService, EsD
try {
EsStatisticResult result = new EsStatisticResult();
result.setName(constractEsStatisticRet(logQuery));
- MilogLogStoreDO logStore =
logstoreDao.getByName(logQuery.getLogstore());
+ MilogLogStoreDO logStore =
logstoreDao.queryById(logQuery.getStoreId());
if (logStore == null) {
return new Result<>(CommonError.UnknownError.getCode(), "not
found logstore", null);
}
@@ -337,7 +337,7 @@ public class LogQueryServiceImpl implements
LogQueryService, LogDataService, EsD
if (searchLog.isLegalParam(logContextQuery) == false) {
return Result.failParam("Required parameters are missing");
}
- MilogLogStoreDO logStore =
logstoreDao.getByName(logContextQuery.getLogstore());
+ MilogLogStoreDO logStore =
logstoreDao.queryById(logContextQuery.getLogStoreId());
if (logStore.getEsClusterId() == null ||
StringUtils.isEmpty(logStore.getEsIndex())) {
return Result.failParam("Store configuration exception");
}
diff --git
a/ozhera-log/log-stream/src/main/java/org/apache/ozhera/log/stream/plugin/es/EsPlugin.java
b/ozhera-log/log-stream/src/main/java/org/apache/ozhera/log/stream/plugin/es/EsPlugin.java
index 7c878beb..f33fc498 100644
---
a/ozhera-log/log-stream/src/main/java/org/apache/ozhera/log/stream/plugin/es/EsPlugin.java
+++
b/ozhera-log/log-stream/src/main/java/org/apache/ozhera/log/stream/plugin/es/EsPlugin.java
@@ -222,6 +222,10 @@ public class EsPlugin {
}
return;
}
+ if (failure.getMessage().contains("I/O reactor status:
STOPPED")) {
+ esServiceMap.remove(cacheKey(esInfo));
+ esProcessorMap.remove(cacheKey(esInfo));
+ }
log.error(String.format("fail send %s message to es,desc:%s,es
addr:%s", request.numberOfActions(), request.getDescription(),
esInfo.getAddr()), new RuntimeException(failure));
Class clazz = failure.getClass();
log.error("Bulk [{}] finished with [{}] requests of error:{},
{}, {}:-[{}]", executionId
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
