This is an automated email from the ASF dual-hosted git repository.
leonbao pushed a commit to branch 2.0.2-prepare
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git
The following commit(s) were added to refs/heads/2.0.2-prepare by this push:
new 32fb3f8 [Bug] [dolphinscheduler-server] memory leak of logger #7661
(#7665)
32fb3f8 is described below
commit 32fb3f84df18fbd686b9c4522f91ca1898bf6a7a
Author: zwZjut <[email protected]>
AuthorDate: Tue Dec 28 18:57:44 2021 +0800
[Bug] [dolphinscheduler-server] memory leak of logger #7661 (#7665)
* [Feature][dolphinscheduler-api] parse traceId in http header for Cross
system delivery to #7237 (#7238)
* to #7237
* rerun test
Co-authored-by: honghuo.zw <[email protected]>
* chery-pick 05aef27 and handle conflicts
* to #7065: fix ExecutorService and schedulerService (#7072)
Co-authored-by: honghuo.zw <[email protected]>
* [Feature][dolphinscheduler-api] access control of taskDefinition and
taskInstance in project to #7081 (#7082)
* to #7081
* fix #7081
* to #7081
Co-authored-by: honghuo.zw <[email protected]>
* chery-pick 8ebe060 and handle conflicts
* cherry-pick 1f18444 and handle conflicts
* fix #6807: dolphinscheduler.zookeeper.env_vars - >
dolphinscheduler.registry.env_vars (#6808)
Co-authored-by: honghuo.zw <[email protected]>
Co-authored-by: Kirs <[email protected]>
* add default constructor (#6780)
Co-authored-by: honghuo.zw <[email protected]>
* to #7108 (#7109)
* add conf && pick #7562 to worker
* to #7661
* to #7661
* to #7661:fix ut
Co-authored-by: honghuo.zw <[email protected]>
Co-authored-by: Kirs <[email protected]>
---
.../dolphinscheduler/logback/logback-master.xml | 4 +--
.../dolphinscheduler/logback/logback-worker.xml | 4 +--
.../dolphinscheduler/common/utils/LoggerUtils.java | 10 ++----
.../server/log/TaskLogDiscriminator.java | 37 ++++++++++++++++------
.../dolphinscheduler/server/log/TaskLogFilter.java | 17 ++++++----
.../server/master/MasterServer.java | 3 --
.../master/runner/task/BaseTaskProcessor.java | 3 +-
.../master/runner/task/ConditionTaskProcessor.java | 11 ++++---
.../server/worker/runner/TaskExecuteThread.java | 32 +++++++++++++------
.../server/log/TaskLogDiscriminatorTest.java | 17 ++++++----
.../server/log/TaskLogFilterTest.java | 18 ++++++-----
.../worker/processor/TaskExecuteProcessorTest.java | 8 -----
.../dolphinscheduler/spi/task/TaskConstants.java | 24 ++++++++++----
.../plugin/task/api/AbstractCommandExecutor.java | 5 +--
.../plugin/task/api/AbstractTaskExecutor.java | 4 +--
.../plugin/task/datax/DataxTask.java | 3 --
.../plugin/task/http/HttpTask.java | 4 ---
.../plugin/task/procedure/ProcedureTask.java | 4 ---
.../dolphinscheduler/plugin/task/sql/SqlTask.java | 3 --
19 files changed, 120 insertions(+), 91 deletions(-)
diff --git a/docker/build/conf/dolphinscheduler/logback/logback-master.xml
b/docker/build/conf/dolphinscheduler/logback/logback-master.xml
index f0d2c81..0d61968 100644
--- a/docker/build/conf/dolphinscheduler/logback/logback-master.xml
+++ b/docker/build/conf/dolphinscheduler/logback/logback-master.xml
@@ -45,7 +45,7 @@
<file>${log.base}/${taskAppId}.log</file>
<encoder>
<pattern>
- [%level] %date{yyyy-MM-dd HH:mm:ss.SSS}
%logger{96}:[%line] - %messsage%n
+ [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} [%thread] -
%messsage%n
</pattern>
<charset>UTF-8</charset>
</encoder>
@@ -66,7 +66,7 @@
</rollingPolicy>
<encoder>
<pattern>
- [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] -
%msg%n
+ [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} [%thread]
%logger{96}:[%line] - %msg%n
</pattern>
<charset>UTF-8</charset>
</encoder>
diff --git a/docker/build/conf/dolphinscheduler/logback/logback-worker.xml
b/docker/build/conf/dolphinscheduler/logback/logback-worker.xml
index 7127219..4aca8b8 100644
--- a/docker/build/conf/dolphinscheduler/logback/logback-worker.xml
+++ b/docker/build/conf/dolphinscheduler/logback/logback-worker.xml
@@ -46,7 +46,7 @@
<file>${log.base}/${taskAppId}.log</file>
<encoder>
<pattern>
- [%level] %date{yyyy-MM-dd HH:mm:ss.SSS}
%logger{96}:[%line] - %messsage%n
+ [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} [%thread] -
%messsage%n
</pattern>
<charset>UTF-8</charset>
</encoder>
@@ -66,7 +66,7 @@
</rollingPolicy>
<encoder>
<pattern>
- [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] -
%messsage%n
+ [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} [%thread]
%logger{96}:[%line] - %messsage%n
</pattern>
<charset>UTF-8</charset>
</encoder>
diff --git
a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/LoggerUtils.java
b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/LoggerUtils.java
index 1b25298..0b40980 100644
---
a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/LoggerUtils.java
+++
b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/LoggerUtils.java
@@ -18,8 +18,6 @@
package org.apache.dolphinscheduler.common.utils;
import org.apache.dolphinscheduler.common.Constants;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import java.io.BufferedReader;
import java.io.FileInputStream;
@@ -30,6 +28,9 @@ import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
/**
* logger utils
*/
@@ -54,11 +55,6 @@ public class LoggerUtils {
/**
* Task Logger Thread's name
*/
- public static final String TASK_LOGGER_THREAD_NAME = "TaskLogInfo";
-
- /**
- * Task Logger Thread's name
- */
public static final String TASK_APPID_LOG_FORMAT = "[taskAppId=";
/**
diff --git
a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/TaskLogDiscriminator.java
b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/TaskLogDiscriminator.java
index 029242f..524f8be 100644
---
a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/TaskLogDiscriminator.java
+++
b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/TaskLogDiscriminator.java
@@ -16,16 +16,23 @@
*/
package org.apache.dolphinscheduler.server.log;
-import ch.qos.logback.classic.spi.ILoggingEvent;
-import ch.qos.logback.core.sift.AbstractDiscriminator;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.utils.LoggerUtils;
+import org.apache.dolphinscheduler.spi.task.TaskConstants;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import ch.qos.logback.classic.spi.ILoggingEvent;
+import ch.qos.logback.core.sift.AbstractDiscriminator;
/**
* Task Log Discriminator
*/
public class TaskLogDiscriminator extends AbstractDiscriminator<ILoggingEvent>
{
+ private static Logger logger =
LoggerFactory.getLogger(TaskLogDiscriminator.class);
+
/**
* key
*/
@@ -42,15 +49,25 @@ public class TaskLogDiscriminator extends
AbstractDiscriminator<ILoggingEvent> {
*/
@Override
public String getDiscriminatingValue(ILoggingEvent event) {
- String loggerName = event.getLoggerName()
- .split(Constants.EQUAL_SIGN)[1];
- String prefix = LoggerUtils.TASK_LOGGER_INFO_PREFIX + "-";
- if (loggerName.startsWith(prefix)) {
- return loggerName.substring(prefix.length(),
- loggerName.length() - 1).replace("-","/");
- } else {
- return "unknown_task";
+ String key = "unknown_task";
+
+ logger.debug("task log discriminator start, key is:{}, thread
name:{},loggerName:{}", key, event.getThreadName(), event.getLoggerName());
+
+ if
(event.getLoggerName().startsWith(TaskConstants.TASK_LOG_LOGGER_NAME)) {
+ String threadName = event.getThreadName();
+ if (threadName.endsWith(TaskConstants.GET_OUTPUT_LOG_SERVICE)) {
+ threadName = threadName.substring(0, threadName.length() -
TaskConstants.GET_OUTPUT_LOG_SERVICE.length());
+ }
+ String part1 = threadName
+ .split(Constants.EQUAL_SIGN)[1];
+ String prefix = LoggerUtils.TASK_LOGGER_INFO_PREFIX + "-";
+ if (part1.startsWith(prefix)) {
+ key = part1.substring(prefix.length(),
+ part1.length() - 1).replace("-", "/");
+ }
}
+ logger.debug("task log discriminator end, key is:{}, thread
name:{},loggerName:{}", key, event.getThreadName(), event.getLoggerName());
+ return key;
}
@Override
diff --git
a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/TaskLogFilter.java
b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/TaskLogFilter.java
index 9c47fb9..dfe95c1 100644
---
a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/TaskLogFilter.java
+++
b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/TaskLogFilter.java
@@ -14,11 +14,13 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.dolphinscheduler.server.log;
-import static
org.apache.dolphinscheduler.common.utils.LoggerUtils.TASK_APPID_LOG_FORMAT;
+import org.apache.dolphinscheduler.spi.task.TaskConstants;
-import org.apache.dolphinscheduler.common.utils.LoggerUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import ch.qos.logback.classic.Level;
import ch.qos.logback.classic.spi.ILoggingEvent;
@@ -30,6 +32,7 @@ import ch.qos.logback.core.spi.FilterReply;
*/
public class TaskLogFilter extends Filter<ILoggingEvent> {
+ private static Logger logger =
LoggerFactory.getLogger(TaskLogFilter.class);
/**
* level
*/
@@ -46,11 +49,13 @@ public class TaskLogFilter extends Filter<ILoggingEvent> {
*/
@Override
public FilterReply decide(ILoggingEvent event) {
- if
(event.getThreadName().startsWith(LoggerUtils.TASK_LOGGER_THREAD_NAME)
- || event.getLoggerName().startsWith(" - " +
TASK_APPID_LOG_FORMAT)
+ FilterReply filterReply = FilterReply.DENY;
+ if
((event.getThreadName().startsWith(TaskConstants.TASK_LOGGER_THREAD_NAME)
+ &&
event.getLoggerName().startsWith(TaskConstants.TASK_LOG_LOGGER_NAME))
|| event.getLevel().isGreaterOrEqual(level)) {
- return FilterReply.ACCEPT;
+ filterReply = FilterReply.ACCEPT;
}
- return FilterReply.DENY;
+ logger.debug("task log filter, thread
name:{},loggerName:{},filterReply:{}", event.getThreadName(),
event.getLoggerName(), filterReply.name());
+ return filterReply;
}
}
\ No newline at end of file
diff --git
a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java
b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java
index 576fadd..a5d7cf4 100644
---
a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java
+++
b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java
@@ -41,8 +41,6 @@ import
org.apache.dolphinscheduler.service.quartz.QuartzExecutors;
import java.util.concurrent.ConcurrentHashMap;
-import javax.annotation.PostConstruct;
-
import org.quartz.SchedulerException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -69,7 +67,6 @@ import
org.springframework.transaction.annotation.EnableTransactionManagement;
})
@EnableTransactionManagement
public class MasterServer implements IStoppable {
-
/**
* logger of MasterServer
*/
diff --git
a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/BaseTaskProcessor.java
b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/BaseTaskProcessor.java
index fb14d96..5532477 100644
---
a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/BaseTaskProcessor.java
+++
b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/BaseTaskProcessor.java
@@ -43,6 +43,7 @@ import
org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
import org.apache.dolphinscheduler.service.process.ProcessService;
import org.apache.dolphinscheduler.service.queue.entity.TaskExecutionContext;
import org.apache.dolphinscheduler.spi.enums.ResourceType;
+import org.apache.dolphinscheduler.spi.task.TaskConstants;
import org.apache.dolphinscheduler.spi.task.request.DataxTaskExecutionContext;
import
org.apache.dolphinscheduler.spi.task.request.ProcedureTaskExecutionContext;
import org.apache.dolphinscheduler.spi.task.request.SQLTaskExecutionContext;
@@ -67,7 +68,7 @@ import com.google.common.base.Strings;
public abstract class BaseTaskProcessor implements ITaskProcessor {
- protected Logger logger = LoggerFactory.getLogger(getClass());
+ protected final Logger logger =
LoggerFactory.getLogger(String.format(TaskConstants.TASK_LOG_LOGGER_NAME,
getClass()));
protected boolean killed = false;
diff --git
a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/ConditionTaskProcessor.java
b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/ConditionTaskProcessor.java
index b4a6c73..7c593b0 100644
---
a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/ConditionTaskProcessor.java
+++
b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/ConditionTaskProcessor.java
@@ -34,6 +34,7 @@ import org.apache.dolphinscheduler.dao.entity.TaskInstance;
import org.apache.dolphinscheduler.server.master.config.MasterConfig;
import org.apache.dolphinscheduler.server.utils.LogUtils;
import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
+import org.apache.dolphinscheduler.spi.task.TaskConstants;
import java.util.ArrayList;
import java.util.Date;
@@ -41,6 +42,7 @@ import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
+import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
@@ -48,6 +50,8 @@ import org.slf4j.LoggerFactory;
*/
public class ConditionTaskProcessor extends BaseTaskProcessor {
+ protected static final Logger logger =
LoggerFactory.getLogger(TaskConstants.TASK_LOG_LOGGER_NAME);
+
/**
* dependent parameters
*/
@@ -81,13 +85,12 @@ public class ConditionTaskProcessor extends
BaseTaskProcessor {
taskInstance.getTaskCode(),
taskInstance.getTaskDefinitionVersion()
);
- logger =
LoggerFactory.getLogger(LoggerUtils.buildTaskId(LoggerUtils.TASK_LOGGER_INFO_PREFIX,
+ String threadLoggerInfoName =
LoggerUtils.buildTaskId(LoggerUtils.TASK_LOGGER_INFO_PREFIX,
processInstance.getProcessDefinitionCode(),
processInstance.getProcessDefinitionVersion(),
taskInstance.getProcessInstanceId(),
- taskInstance.getId()));
- String threadLoggerInfoName =
String.format(Constants.TASK_LOG_INFO_FORMAT,
processService.formatTaskAppId(this.taskInstance));
- Thread.currentThread().setName(threadLoggerInfoName);
+ taskInstance.getId());
+
Thread.currentThread().setName(String.format(TaskConstants.TASK_LOGGER_THREAD_NAME_FORMAT,threadLoggerInfoName));
initTaskParameters();
logger.info("dependent task start");
return true;
diff --git
a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java
b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java
index b633a54..0b18dcf 100644
---
a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java
+++
b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/TaskExecuteThread.java
@@ -17,16 +17,18 @@
package org.apache.dolphinscheduler.server.worker.runner;
-import com.github.rholder.retry.RetryException;
-import org.apache.commons.collections.MapUtils;
-import org.apache.commons.lang.StringUtils;
import org.apache.dolphinscheduler.common.Constants;
-import org.apache.dolphinscheduler.common.enums.CommandType;
import org.apache.dolphinscheduler.common.enums.Event;
import org.apache.dolphinscheduler.common.enums.ExecutionStatus;
import org.apache.dolphinscheduler.common.enums.TaskType;
import org.apache.dolphinscheduler.common.process.Property;
-import org.apache.dolphinscheduler.common.utils.*;
+import org.apache.dolphinscheduler.common.utils.CommonUtils;
+import org.apache.dolphinscheduler.common.utils.DateUtils;
+import org.apache.dolphinscheduler.common.utils.HadoopUtils;
+import org.apache.dolphinscheduler.common.utils.JSONUtils;
+import org.apache.dolphinscheduler.common.utils.LoggerUtils;
+import org.apache.dolphinscheduler.common.utils.OSUtils;
+import org.apache.dolphinscheduler.common.utils.RetryerUtils;
import org.apache.dolphinscheduler.remote.command.Command;
import org.apache.dolphinscheduler.remote.command.TaskExecuteAckCommand;
import org.apache.dolphinscheduler.remote.command.TaskExecuteResponseCommand;
@@ -39,20 +41,29 @@ import
org.apache.dolphinscheduler.service.queue.entity.TaskExecutionContext;
import org.apache.dolphinscheduler.spi.task.AbstractTask;
import org.apache.dolphinscheduler.spi.task.TaskAlertInfo;
import org.apache.dolphinscheduler.spi.task.TaskChannel;
+import org.apache.dolphinscheduler.spi.task.TaskConstants;
import org.apache.dolphinscheduler.spi.task.TaskExecutionContextCacheManager;
import org.apache.dolphinscheduler.spi.task.request.TaskRequest;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+
+import org.apache.commons.collections.MapUtils;
+import org.apache.commons.lang.StringUtils;
import java.io.File;
import java.io.IOException;
-import java.util.*;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
import java.util.concurrent.Delayed;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
-import static java.util.Calendar.DAY_OF_MONTH;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.github.rholder.retry.RetryException;
/**
* task scheduler thread
@@ -162,6 +173,9 @@ public class TaskExecuteThread implements Runnable, Delayed
{
taskExecutionContext.getTaskInstanceId());
taskRequest.setTaskLogName(taskLogName);
+ // set the name of the current thread
+
Thread.currentThread().setName(String.format(TaskConstants.TASK_LOGGER_THREAD_NAME_FORMAT,taskLogName));
+
task = taskChannel.createTask(taskRequest);
// task init
diff --git
a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/log/TaskLogDiscriminatorTest.java
b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/log/TaskLogDiscriminatorTest.java
index 1908475..13232c3 100644
---
a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/log/TaskLogDiscriminatorTest.java
+++
b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/log/TaskLogDiscriminatorTest.java
@@ -16,16 +16,19 @@
*/
package org.apache.dolphinscheduler.server.log;
-import ch.qos.logback.classic.Level;
-import ch.qos.logback.classic.spi.ILoggingEvent;
-import ch.qos.logback.classic.spi.IThrowableProxy;
-import ch.qos.logback.classic.spi.LoggerContextVO;
+import org.apache.dolphinscheduler.spi.task.TaskConstants;
+
+import java.util.Map;
+
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.slf4j.Marker;
-import java.util.Map;
+import ch.qos.logback.classic.Level;
+import ch.qos.logback.classic.spi.ILoggingEvent;
+import ch.qos.logback.classic.spi.IThrowableProxy;
+import ch.qos.logback.classic.spi.LoggerContextVO;
public class TaskLogDiscriminatorTest {
@@ -48,7 +51,7 @@ public class TaskLogDiscriminatorTest {
String result = taskLogDiscriminator.getDiscriminatingValue(new
ILoggingEvent() {
@Override
public String getThreadName() {
- return null;
+ return
String.format(TaskConstants.TASK_LOGGER_THREAD_NAME_FORMAT,"-[taskAppId=TASK-1-1-1");
}
@Override
@@ -73,7 +76,7 @@ public class TaskLogDiscriminatorTest {
@Override
public String getLoggerName() {
- return "[taskAppId=TASK-1-1-1";
+ return TaskConstants.TASK_LOG_LOGGER_NAME;
}
@Override
diff --git
a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/log/TaskLogFilterTest.java
b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/log/TaskLogFilterTest.java
index d8abb48..78ebae6 100644
---
a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/log/TaskLogFilterTest.java
+++
b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/log/TaskLogFilterTest.java
@@ -16,17 +16,19 @@
*/
package org.apache.dolphinscheduler.server.log;
+import org.apache.dolphinscheduler.spi.task.TaskConstants;
+
+import java.util.Map;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Marker;
+
import ch.qos.logback.classic.Level;
import ch.qos.logback.classic.spi.ILoggingEvent;
import ch.qos.logback.classic.spi.IThrowableProxy;
import ch.qos.logback.classic.spi.LoggerContextVO;
import ch.qos.logback.core.spi.FilterReply;
-import org.apache.dolphinscheduler.common.utils.LoggerUtils;
-import org.junit.Assert;
-import org.junit.Test;
-import org.slf4j.Marker;
-
-import java.util.Map;
public class TaskLogFilterTest {
@@ -39,7 +41,7 @@ public class TaskLogFilterTest {
FilterReply filterReply = taskLogFilter.decide(new ILoggingEvent() {
@Override
public String getThreadName() {
- return LoggerUtils.TASK_LOGGER_THREAD_NAME;
+ return TaskConstants.TASK_LOGGER_THREAD_NAME;
}
@Override
@@ -64,7 +66,7 @@ public class TaskLogFilterTest {
@Override
public String getLoggerName() {
- return null;
+ return TaskConstants.TASK_LOG_LOGGER_NAME;
}
@Override
diff --git
a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessorTest.java
b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessorTest.java
index daee652..559bd6d 100644
---
a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessorTest.java
+++
b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessorTest.java
@@ -22,7 +22,6 @@ import org.apache.dolphinscheduler.common.enums.TaskType;
import org.apache.dolphinscheduler.common.thread.ThreadUtils;
import org.apache.dolphinscheduler.common.utils.FileUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
-import org.apache.dolphinscheduler.common.utils.LoggerUtils;
import org.apache.dolphinscheduler.remote.command.Command;
import org.apache.dolphinscheduler.remote.command.CommandType;
import org.apache.dolphinscheduler.remote.command.TaskExecuteAckCommand;
@@ -49,7 +48,6 @@ import org.powermock.api.mockito.PowerMockito;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
/**
* test task execute processor
@@ -107,12 +105,6 @@ public class TaskExecuteProcessorTest {
PowerMockito.when(SpringApplicationContext.getBean(WorkerConfig.class))
.thenReturn(workerConfig);
- Logger taskLogger =
LoggerFactory.getLogger(LoggerUtils.buildTaskId(LoggerUtils.TASK_LOGGER_INFO_PREFIX,
- taskExecutionContext.getProcessDefineCode(),
- taskExecutionContext.getProcessDefineVersion(),
- taskExecutionContext.getProcessInstanceId(),
- taskExecutionContext.getTaskInstanceId()));
-
workerManager = PowerMockito.mock(WorkerManagerThread.class);
PowerMockito.when(workerManager.offer(new
TaskExecuteThread(taskExecutionContext, taskCallbackService,
alertClientService))).thenReturn(Boolean.TRUE);
diff --git
a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/TaskConstants.java
b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/TaskConstants.java
index 5ec6342..ed1f823 100644
---
a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/TaskConstants.java
+++
b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/task/TaskConstants.java
@@ -124,9 +124,24 @@ public class TaskConstants {
public static final String RWXR_XR_X = "rwxr-xr-x";
/**
- * task log info format
+ * Task Logger Thread's name
+ */
+ public static final String TASK_LOGGER_THREAD_NAME = "TaskLogInfo";
+
+ /**
+ * task logger thread name format
+ */
+ public static final String TASK_LOGGER_THREAD_NAME_FORMAT =
TASK_LOGGER_THREAD_NAME + "-%s";
+
+ /**
+ * task log logger name
*/
- public static final String TASK_LOG_INFO_FORMAT = "TaskLogInfo-%s";
+ public static final String TASK_LOG_LOGGER_NAME = "TaskLogLogger";
+
+ /**
+ * task log logger name format
+ */
+ public static final String TASK_LOG_LOGGER_NAME_FORMAT =
TASK_LOG_LOGGER_NAME + "-%s";
/**
* date format of yyyyMMdd
@@ -320,10 +335,7 @@ public class TaskConstants {
*/
public static final String HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE =
"hadoop.security.authentication.startup.state";
- /**
- * Task Logger Thread's name
- */
- public static final String TASK_LOGGER_THREAD_NAME = "TaskLogInfo";
+ public static final String GET_OUTPUT_LOG_SERVICE = "-getOutputLogService";
/**
* hdfs/s3 configuration
diff --git
a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/AbstractCommandExecutor.java
b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/AbstractCommandExecutor.java
index 418bf1a..805d035 100644
---
a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/AbstractCommandExecutor.java
+++
b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/AbstractCommandExecutor.java
@@ -307,8 +307,9 @@ public abstract class AbstractCommandExecutor {
* @param process process
*/
private void parseProcessOutput(Process process) {
- String threadLoggerInfoName =
String.format(TaskConstants.TASK_LOGGER_THREAD_NAME + "-%s",
taskRequest.getTaskAppId());
- ExecutorService getOutputLogService =
newDaemonSingleThreadExecutor(threadLoggerInfoName + "-" +
"getOutputLogService");
+ String threadLoggerInfoName =
String.format(TaskConstants.TASK_LOGGER_THREAD_NAME_FORMAT,
+ taskRequest.getTaskLogName() +
TaskConstants.GET_OUTPUT_LOG_SERVICE);
+ ExecutorService getOutputLogService =
newDaemonSingleThreadExecutor(threadLoggerInfoName);
getOutputLogService.submit(() -> {
try (BufferedReader inReader = new BufferedReader(new
InputStreamReader(process.getInputStream()))) {
String line;
diff --git
a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/AbstractTaskExecutor.java
b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/AbstractTaskExecutor.java
index e572418..5b5661c 100644
---
a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/AbstractTaskExecutor.java
+++
b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/AbstractTaskExecutor.java
@@ -18,6 +18,7 @@
package org.apache.dolphinscheduler.plugin.task.api;
import org.apache.dolphinscheduler.spi.task.AbstractTask;
+import org.apache.dolphinscheduler.spi.task.TaskConstants;
import org.apache.dolphinscheduler.spi.task.request.TaskRequest;
import java.util.StringJoiner;
@@ -32,7 +33,7 @@ public abstract class AbstractTaskExecutor extends
AbstractTask {
public static final Marker FINALIZE_SESSION_MARKER =
MarkerFactory.getMarker("FINALIZE_SESSION");
- protected Logger logger;
+ protected final Logger logger =
LoggerFactory.getLogger(String.format(TaskConstants.TASK_LOG_LOGGER_NAME_FORMAT,
getClass()));
/**
* constructor
@@ -41,7 +42,6 @@ public abstract class AbstractTaskExecutor extends
AbstractTask {
*/
protected AbstractTaskExecutor(TaskRequest taskRequest) {
super(taskRequest);
- logger = LoggerFactory.getLogger(taskRequest.getTaskLogName());
}
/**
diff --git
a/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxTask.java
b/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxTask.java
index 3fbabe5..216ad74 100644
---
a/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxTask.java
+++
b/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/main/java/org/apache/dolphinscheduler/plugin/task/datax/DataxTask.java
@@ -145,9 +145,6 @@ public class DataxTask extends AbstractTaskExecutor {
@Override
public void handle() throws Exception {
try {
- // set the name of the current thread
- String threadLoggerInfoName = String.format("TaskLogInfo-%s",
taskExecutionContext.getTaskAppId());
- Thread.currentThread().setName(threadLoggerInfoName);
// replace placeholder,and combine local and global parameters
Map<String, Property> paramsMap =
ParamUtils.convert(taskExecutionContext, getParameters());
diff --git
a/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpTask.java
b/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpTask.java
index f57238d..04978f0 100644
---
a/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpTask.java
+++
b/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpTask.java
@@ -18,7 +18,6 @@
package org.apache.dolphinscheduler.plugin.task.http;
import static
org.apache.dolphinscheduler.plugin.task.http.HttpTaskConstants.APPLICATION_JSON;
-import static
org.apache.dolphinscheduler.spi.task.TaskConstants.TASK_LOG_INFO_FORMAT;
import org.apache.dolphinscheduler.plugin.task.api.AbstractTaskExecutor;
import org.apache.dolphinscheduler.plugin.task.util.MapUtils;
@@ -92,9 +91,6 @@ public class HttpTask extends AbstractTaskExecutor {
@Override
public void handle() throws Exception {
- String threadLoggerInfoName = String.format(TASK_LOG_INFO_FORMAT,
taskExecutionContext.getTaskAppId());
- Thread.currentThread().setName(threadLoggerInfoName);
-
long startTime = System.currentTimeMillis();
String formatTimeStamp = DateUtils.formatTimeStamp(startTime);
String statusCode = null;
diff --git
a/dolphinscheduler-task-plugin/dolphinscheduler-task-procedure/src/main/java/org/apache/dolphinscheduler/plugin/task/procedure/ProcedureTask.java
b/dolphinscheduler-task-plugin/dolphinscheduler-task-procedure/src/main/java/org/apache/dolphinscheduler/plugin/task/procedure/ProcedureTask.java
index 58d597f..202b75d 100644
---
a/dolphinscheduler-task-plugin/dolphinscheduler-task-procedure/src/main/java/org/apache/dolphinscheduler/plugin/task/procedure/ProcedureTask.java
+++
b/dolphinscheduler-task-plugin/dolphinscheduler-task-procedure/src/main/java/org/apache/dolphinscheduler/plugin/task/procedure/ProcedureTask.java
@@ -19,7 +19,6 @@ package org.apache.dolphinscheduler.plugin.task.procedure;
import static
org.apache.dolphinscheduler.spi.task.TaskConstants.EXIT_CODE_FAILURE;
import static
org.apache.dolphinscheduler.spi.task.TaskConstants.EXIT_CODE_SUCCESS;
-import static
org.apache.dolphinscheduler.spi.task.TaskConstants.TASK_LOG_INFO_FORMAT;
import
org.apache.dolphinscheduler.plugin.datasource.api.plugin.DataSourceClientProvider;
import org.apache.dolphinscheduler.plugin.datasource.api.utils.DatasourceUtil;
@@ -85,9 +84,6 @@ public class ProcedureTask extends AbstractTaskExecutor {
@Override
public void handle() throws Exception {
- // set the name of the current thread
- String threadLoggerInfoName = String.format(TASK_LOG_INFO_FORMAT,
taskExecutionContext.getTaskAppId());
- Thread.currentThread().setName(threadLoggerInfoName);
logger.info("procedure type : {}, datasource : {}, method : {} ,
localParams : {}",
procedureParameters.getType(),
diff --git
a/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTask.java
b/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTask.java
index 1ec5a2c..d8bae7a 100644
---
a/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTask.java
+++
b/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTask.java
@@ -114,9 +114,6 @@ public class SqlTask extends AbstractTaskExecutor {
@Override
public void handle() throws Exception {
- // set the name of the current thread
- String threadLoggerInfoName =
String.format(TaskConstants.TASK_LOG_INFO_FORMAT,
taskExecutionContext.getTaskAppId());
- Thread.currentThread().setName(threadLoggerInfoName);
logger.info("Full sql parameters: {}", sqlParameters);
logger.info("sql type : {}, datasource : {}, sql : {} , localParams :
{},udfs : {},showType : {},connParams : {},varPool : {} ,query max result limit
{}",
