This is an automated email from the ASF dual-hosted git repository.
lidongdai pushed a commit to branch refactor-worker
in repository https://gitbox.apache.org/repos/asf/incubator-dolphinscheduler.git
The following commit(s) were added to refs/heads/refactor-worker by this push:
new f551913 split logback config into seperate child module logback
(#2120)
f551913 is described below
commit f55191324994d505f52c4b2aaaa6e30679828131
Author: dailidong <[email protected]>
AuthorDate: Mon Mar 9 19:03:51 2020 +0800
split logback config into seperate child module logback (#2120)
* update logback
* update log
---
.../conf/dolphinscheduler/conf/worker_logback.xml | 6 +-
.../src/main/resources/logback-alert.xml | 52 +++++++
.../src/main/resources/logback-api.xml | 51 +++----
.../src/main/resources/logback.xml | 169 ---------------------
.../server}/log/MasterLogFilter.java | 2 +-
.../server}/log/SensitiveDataConverter.java | 2 +-
.../server}/log/TaskLogDiscriminator.java | 2 +-
.../server}/log/TaskLogFilter.java | 2 +-
.../server}/log/WorkerLogFilter.java | 5 +-
.../worker/processor/TaskExecuteProcessor.java | 3 +-
.../src/main/resources/logback-master.xml | 49 ++----
.../src/main/resources/logback-worker.xml | 28 ++--
.../server}/log/MasterLogFilterTest.java | 2 +-
.../server}/log/SensitiveDataConverterTest.java | 2 +-
.../server}/log/TaskLogDiscriminatorTest.java | 4 +-
.../server}/log/TaskLogFilterTest.java | 3 +-
.../server}/log/WorkerLogFilterTest.java | 2 +-
script/dolphinscheduler-daemon.sh | 8 +-
18 files changed, 117 insertions(+), 275 deletions(-)
diff --git a/dockerfile/conf/dolphinscheduler/conf/worker_logback.xml
b/dockerfile/conf/dolphinscheduler/conf/worker_logback.xml
index bf4a651..1b09260 100644
--- a/dockerfile/conf/dolphinscheduler/conf/worker_logback.xml
+++ b/dockerfile/conf/dolphinscheduler/conf/worker_logback.xml
@@ -31,8 +31,8 @@
<filter class="ch.qos.logback.classic.filter.ThresholdFilter">
<level>INFO</level>
</filter>
- <filter
class="org.apache.dolphinscheduler.common.log.TaskLogFilter"></filter>
- <Discriminator
class="org.apache.dolphinscheduler.common.log.TaskLogDiscriminator">
+ <filter
class="org.apache.dolphinscheduler.server.log.TaskLogFilter"></filter>
+ <Discriminator
class="org.apache.dolphinscheduler.server.log.TaskLogDiscriminator">
<key>taskAppId</key>
<logBase>${log.base}</logBase>
</Discriminator>
@@ -52,7 +52,7 @@
<appender name="WORKERLOGFILE"
class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>${log.base}/dolphinscheduler-worker.log</file>
- <filter class="org.apache.dolphinscheduler.common.log.WorkerLogFilter">
+ <filter class="org.apache.dolphinscheduler.server.log.WorkerLogFilter">
<level>INFO</level>
</filter>
diff --git a/dolphinscheduler-alert/src/main/resources/logback-alert.xml
b/dolphinscheduler-alert/src/main/resources/logback-alert.xml
new file mode 100644
index 0000000..5d1c078
--- /dev/null
+++ b/dolphinscheduler-alert/src/main/resources/logback-alert.xml
@@ -0,0 +1,52 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<!-- Logback configuration. See http://logback.qos.ch/manual/index.html -->
+<configuration scan="true" scanPeriod="120 seconds"> <!--debug="true" -->
+
+ <property name="log.base" value="logs"/>
+ <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
+ <encoder>
+ <pattern>
+ [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] -
%msg%n
+ </pattern>
+ <charset>UTF-8</charset>
+ </encoder>
+ </appender>
+
+ <appender name="ALERTLOGFILE"
class="ch.qos.logback.core.rolling.RollingFileAppender">
+ <file>${log.base}/dolphinscheduler-alert.log</file>
+ <rollingPolicy
class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
+
<fileNamePattern>${log.base}/dolphinscheduler-alert.%d{yyyy-MM-dd_HH}.%i.log</fileNamePattern>
+ <maxHistory>20</maxHistory>
+ <maxFileSize>64MB</maxFileSize>
+ </rollingPolicy>
+ <encoder>
+ <pattern>
+ [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] -
%msg%n
+ </pattern>
+ <charset>UTF-8</charset>
+ </encoder>
+ </appender>
+
+ <root level="INFO">
+ <appender-ref ref="STDOUT"/>
+ <appender-ref ref="APILOGFILE"/>
+ </root>
+
+</configuration>
\ No newline at end of file
diff --git a/dockerfile/conf/dolphinscheduler/conf/worker_logback.xml
b/dolphinscheduler-api/src/main/resources/logback-api.xml
similarity index 53%
copy from dockerfile/conf/dolphinscheduler/conf/worker_logback.xml
copy to dolphinscheduler-api/src/main/resources/logback-api.xml
index bf4a651..2df90d8 100644
--- a/dockerfile/conf/dolphinscheduler/conf/worker_logback.xml
+++ b/dolphinscheduler-api/src/main/resources/logback-api.xml
@@ -1,4 +1,4 @@
-<?xml version="1.0" encoding="UTF-8" ?>
+<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
@@ -17,7 +17,8 @@
-->
<!-- Logback configuration. See http://logback.qos.ch/manual/index.html -->
-<configuration scan="true" scanPeriod="120 seconds">
+<configuration scan="true" scanPeriod="120 seconds"> <!--debug="true" -->
+
<property name="log.base" value="logs"/>
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
@@ -27,53 +28,35 @@
<charset>UTF-8</charset>
</encoder>
</appender>
- <appender name="TASKLOGFILE"
class="ch.qos.logback.classic.sift.SiftingAppender">
- <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
- <level>INFO</level>
- </filter>
- <filter
class="org.apache.dolphinscheduler.common.log.TaskLogFilter"></filter>
- <Discriminator
class="org.apache.dolphinscheduler.common.log.TaskLogDiscriminator">
- <key>taskAppId</key>
- <logBase>${log.base}</logBase>
- </Discriminator>
- <sift>
- <appender name="FILE-${taskAppId}"
class="ch.qos.logback.core.FileAppender">
- <file>${log.base}/${taskAppId}.log</file>
- <encoder>
- <pattern>
- [%level] %date{yyyy-MM-dd HH:mm:ss.SSS}
%logger{96}:[%line] - %msg%n
- </pattern>
- <charset>UTF-8</charset>
- </encoder>
- <append>true</append>
- </appender>
- </sift>
- </appender>
- <appender name="WORKERLOGFILE"
class="ch.qos.logback.core.rolling.RollingFileAppender">
- <file>${log.base}/dolphinscheduler-worker.log</file>
- <filter class="org.apache.dolphinscheduler.common.log.WorkerLogFilter">
+ <!-- api server logback config start -->
+ <appender name="APILOGFILE"
class="ch.qos.logback.core.rolling.RollingFileAppender">
+ <file>${log.base}/dolphinscheduler-api-server.log</file>
+ <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
<level>INFO</level>
</filter>
-
<rollingPolicy
class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
-
<fileNamePattern>${log.base}/dolphinscheduler-worker.%d{yyyy-MM-dd_HH}.%i.log</fileNamePattern>
+
<fileNamePattern>${log.base}/dolphinscheduler-api-server.%d{yyyy-MM-dd_HH}.%i.log</fileNamePattern>
<maxHistory>168</maxHistory>
- <maxFileSize>200MB</maxFileSize>
+ <maxFileSize>64MB</maxFileSize>
</rollingPolicy>
-
<encoder>
<pattern>
[%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] -
%msg%n
</pattern>
<charset>UTF-8</charset>
</encoder>
-
</appender>
+ <!-- api server logback config end -->
+
+ <logger name="org.apache.zookeeper" level="WARN"/>
+ <logger name="org.apache.hbase" level="WARN"/>
+ <logger name="org.apache.hadoop" level="WARN"/>
<root level="INFO">
- <appender-ref ref="TASKLOGFILE"/>
- <appender-ref ref="WORKERLOGFILE"/>
+ <appender-ref ref="STDOUT"/>
+ <appender-ref ref="APILOGFILE"/>
</root>
+
</configuration>
\ No newline at end of file
diff --git a/dolphinscheduler-common/src/main/resources/logback.xml
b/dolphinscheduler-common/src/main/resources/logback.xml
deleted file mode 100644
index 7f634da..0000000
--- a/dolphinscheduler-common/src/main/resources/logback.xml
+++ /dev/null
@@ -1,169 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one or more
- ~ contributor license agreements. See the NOTICE file distributed with
- ~ this work for additional information regarding copyright ownership.
- ~ The ASF licenses this file to You under the Apache License, Version 2.0
- ~ (the "License"); you may not use this file except in compliance with
- ~ the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing, software
- ~ distributed under the License is distributed on an "AS IS" BASIS,
- ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- ~ See the License for the specific language governing permissions and
- ~ limitations under the License.
- -->
-
-<!-- Logback configuration. See http://logback.qos.ch/manual/index.html -->
-<configuration scan="true" scanPeriod="120 seconds"> <!--debug="true" -->
-
- <property name="log.base" value="logs"/>
- <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
- <encoder>
- <pattern>
- [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] -
%msg%n
- </pattern>
- <charset>UTF-8</charset>
- </encoder>
- </appender>
-
-
- <!-- master server logback config start -->
- <appender name="MASTERLOGFILE"
class="ch.qos.logback.core.rolling.RollingFileAppender">
- <file>${log.base}/dolphinscheduler-master.log</file>
- <!--<filter
class="org.apache.dolphinscheduler.common.log.MasterLogFilter">
- <level>INFO</level>
- </filter>-->
- <rollingPolicy
class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
-
<fileNamePattern>${log.base}/dolphinscheduler-master.%d{yyyy-MM-dd_HH}.%i.log</fileNamePattern>
- <maxHistory>168</maxHistory>
- <maxFileSize>200MB</maxFileSize>
- </rollingPolicy>
- <encoder>
- <pattern>
- [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] -
%msg%n
- </pattern>
- <charset>UTF-8</charset>
- </encoder>
- </appender>
- <!-- master server logback config end -->
-
-
- <!-- worker server logback config start -->
- <conversionRule conversionWord="messsage"
-
converterClass="org.apache.dolphinscheduler.common.log.SensitiveDataConverter"/>
- <appender name="TASKLOGFILE"
class="ch.qos.logback.classic.sift.SiftingAppender">
- <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
- <level>INFO</level>
- </filter>
- <filter class="org.apache.dolphinscheduler.common.log.TaskLogFilter"/>
- <Discriminator
class="org.apache.dolphinscheduler.common.log.TaskLogDiscriminator">
- <key>taskAppId</key>
- <logBase>${log.base}</logBase>
- </Discriminator>
- <sift>
- <appender name="FILE-${taskAppId}"
class="ch.qos.logback.core.FileAppender">
- <file>${log.base}/${taskAppId}.log</file>
- <encoder>
- <pattern>
- [%level] %date{yyyy-MM-dd HH:mm:ss.SSS}
%logger{96}:[%line] - %messsage%n
- </pattern>
- <charset>UTF-8</charset>
- </encoder>
- <append>true</append>
- </appender>
- </sift>
- </appender>
- <appender name="WORKERLOGFILE"
class="ch.qos.logback.core.rolling.RollingFileAppender">
- <file>${log.base}/dolphinscheduler-worker.log</file>
- <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
- <level>INFO</level>
- </filter>
- <filter
class="org.apache.dolphinscheduler.common.log.WorkerLogFilter"/>
- <rollingPolicy
class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
-
<fileNamePattern>${log.base}/dolphinscheduler-worker.%d{yyyy-MM-dd_HH}.%i.log</fileNamePattern>
- <maxHistory>168</maxHistory>
- <maxFileSize>200MB</maxFileSize>
- </rollingPolicy>
- <encoder>
- <pattern>
- [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] -
%messsage%n
- </pattern>
- <charset>UTF-8</charset>
- </encoder>
- </appender>
- <!-- worker server logback config end -->
-
-
- <!-- alert server logback config start -->
- <appender name="ALERTLOGFILE"
class="ch.qos.logback.core.rolling.RollingFileAppender">
- <file>${log.base}/dolphinscheduler-alert.log</file>
- <rollingPolicy
class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
-
<fileNamePattern>${log.base}/dolphinscheduler-alert.%d{yyyy-MM-dd_HH}.%i.log</fileNamePattern>
- <maxHistory>20</maxHistory>
- <maxFileSize>64MB</maxFileSize>
- </rollingPolicy>
- <encoder>
- <pattern>
- [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] -
%msg%n
- </pattern>
- <charset>UTF-8</charset>
- </encoder>
- </appender>
- <!-- alert server logback config end -->
-
-
- <!-- api server logback config start -->
- <appender name="APILOGFILE"
class="ch.qos.logback.core.rolling.RollingFileAppender">
- <file>${log.base}/dolphinscheduler-api-server.log</file>
- <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
- <level>INFO</level>
- </filter>
- <rollingPolicy
class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
-
<fileNamePattern>${log.base}/dolphinscheduler-api-server.%d{yyyy-MM-dd_HH}.%i.log</fileNamePattern>
- <maxHistory>168</maxHistory>
- <maxFileSize>64MB</maxFileSize>
- </rollingPolicy>
- <encoder>
- <pattern>
- [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] -
%msg%n
- </pattern>
- <charset>UTF-8</charset>
- </encoder>
- </appender>
- <!-- api server logback config end -->
-
- <logger name="org.apache.zookeeper" level="WARN"/>
- <logger name="org.apache.hbase" level="WARN"/>
- <logger name="org.apache.hadoop" level="WARN"/>
-
-
- <root level="INFO">
- <appender-ref ref="STDOUT"/>
-
- <if condition='p("server").contains("master-server")'>
- <then>
- <appender-ref ref="MASTERLOGFILE"/>
- </then>
- </if>
- <if condition='p("server").contains("worker-server")'>
- <then>
- <appender-ref ref="TASKLOGFILE"/>
- <appender-ref ref="WORKERLOGFILE"/>
- </then>
- </if>
- <if condition='p("server").contains("alert-server")'>
- <then>
- <appender-ref ref="ALERTLOGFILE"/>
- </then>
- </if>
- <if condition='p("server").contains("api-server")'>
- <then>
- <appender-ref ref="APILOGFILE"/>
- </then>
- </if>
- </root>
-
-</configuration>
\ No newline at end of file
diff --git
a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/MasterLogFilter.java
b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/MasterLogFilter.java
similarity index 96%
rename from
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/MasterLogFilter.java
rename to
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/MasterLogFilter.java
index 7b5d53a..575571d 100644
---
a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/MasterLogFilter.java
+++
b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/MasterLogFilter.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.dolphinscheduler.common.log;
+package org.apache.dolphinscheduler.server.log;
import ch.qos.logback.classic.Level;
import ch.qos.logback.classic.spi.ILoggingEvent;
diff --git
a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/SensitiveDataConverter.java
b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/SensitiveDataConverter.java
similarity index 98%
rename from
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/SensitiveDataConverter.java
rename to
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/SensitiveDataConverter.java
index 971ce71..16101c0 100644
---
a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/SensitiveDataConverter.java
+++
b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/SensitiveDataConverter.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.dolphinscheduler.common.log;
+package org.apache.dolphinscheduler.server.log;
import ch.qos.logback.classic.pattern.MessageConverter;
diff --git
a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/TaskLogDiscriminator.java
b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/TaskLogDiscriminator.java
similarity index 97%
rename from
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/TaskLogDiscriminator.java
rename to
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/TaskLogDiscriminator.java
index fd2b076..029242f 100644
---
a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/TaskLogDiscriminator.java
+++
b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/TaskLogDiscriminator.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.dolphinscheduler.common.log;
+package org.apache.dolphinscheduler.server.log;
import ch.qos.logback.classic.spi.ILoggingEvent;
import ch.qos.logback.core.sift.AbstractDiscriminator;
diff --git
a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/TaskLogFilter.java
b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/TaskLogFilter.java
similarity index 97%
rename from
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/TaskLogFilter.java
rename to
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/TaskLogFilter.java
index ac258da..9543416 100644
---
a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/TaskLogFilter.java
+++
b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/TaskLogFilter.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.dolphinscheduler.common.log;
+package org.apache.dolphinscheduler.server.log;
import ch.qos.logback.classic.Level;
import ch.qos.logback.classic.spi.ILoggingEvent;
diff --git
a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/WorkerLogFilter.java
b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/WorkerLogFilter.java
similarity index 91%
rename from
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/WorkerLogFilter.java
rename to
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/WorkerLogFilter.java
index abcc8bc..1a75e59 100644
---
a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/log/WorkerLogFilter.java
+++
b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/WorkerLogFilter.java
@@ -14,15 +14,12 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.dolphinscheduler.common.log;
+package org.apache.dolphinscheduler.server.log;
import ch.qos.logback.classic.Level;
import ch.qos.logback.classic.spi.ILoggingEvent;
import ch.qos.logback.core.filter.Filter;
import ch.qos.logback.core.spi.FilterReply;
-import org.apache.dolphinscheduler.common.utils.LoggerUtils;
-
-import java.util.Arrays;
/**
* worker log filter
diff --git
a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessor.java
b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessor.java
index 5042c97..80ba649 100644
---
a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessor.java
+++
b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessor.java
@@ -24,10 +24,9 @@ import io.netty.channel.Channel;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.ExecutionStatus;
import org.apache.dolphinscheduler.common.enums.TaskType;
-import org.apache.dolphinscheduler.common.log.TaskLogDiscriminator;
+import org.apache.dolphinscheduler.server.log.TaskLogDiscriminator;
import org.apache.dolphinscheduler.common.thread.ThreadUtils;
import org.apache.dolphinscheduler.common.utils.FileUtils;
-import org.apache.dolphinscheduler.common.utils.OSUtils;
import org.apache.dolphinscheduler.common.utils.Preconditions;
import org.apache.dolphinscheduler.remote.command.Command;
import org.apache.dolphinscheduler.remote.command.CommandType;
diff --git a/dockerfile/conf/dolphinscheduler/conf/worker_logback.xml
b/dolphinscheduler-server/src/main/resources/logback-master.xml
similarity index 54%
copy from dockerfile/conf/dolphinscheduler/conf/worker_logback.xml
copy to dolphinscheduler-server/src/main/resources/logback-master.xml
index bf4a651..58193ca 100644
--- a/dockerfile/conf/dolphinscheduler/conf/worker_logback.xml
+++ b/dolphinscheduler-server/src/main/resources/logback-master.xml
@@ -1,4 +1,4 @@
-<?xml version="1.0" encoding="UTF-8" ?>
+<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
@@ -17,7 +17,8 @@
-->
<!-- Logback configuration. See http://logback.qos.ch/manual/index.html -->
-<configuration scan="true" scanPeriod="120 seconds">
+<configuration scan="true" scanPeriod="120 seconds"> <!--debug="true" -->
+
<property name="log.base" value="logs"/>
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
@@ -27,53 +28,31 @@
<charset>UTF-8</charset>
</encoder>
</appender>
- <appender name="TASKLOGFILE"
class="ch.qos.logback.classic.sift.SiftingAppender">
- <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
- <level>INFO</level>
- </filter>
- <filter
class="org.apache.dolphinscheduler.common.log.TaskLogFilter"></filter>
- <Discriminator
class="org.apache.dolphinscheduler.common.log.TaskLogDiscriminator">
- <key>taskAppId</key>
- <logBase>${log.base}</logBase>
- </Discriminator>
- <sift>
- <appender name="FILE-${taskAppId}"
class="ch.qos.logback.core.FileAppender">
- <file>${log.base}/${taskAppId}.log</file>
- <encoder>
- <pattern>
- [%level] %date{yyyy-MM-dd HH:mm:ss.SSS}
%logger{96}:[%line] - %msg%n
- </pattern>
- <charset>UTF-8</charset>
- </encoder>
- <append>true</append>
- </appender>
- </sift>
- </appender>
- <appender name="WORKERLOGFILE"
class="ch.qos.logback.core.rolling.RollingFileAppender">
- <file>${log.base}/dolphinscheduler-worker.log</file>
- <filter class="org.apache.dolphinscheduler.common.log.WorkerLogFilter">
- <level>INFO</level>
- </filter>
+ <!-- master server logback config start -->
+ <appender name="MASTERLOGFILE"
class="ch.qos.logback.core.rolling.RollingFileAppender">
+ <file>${log.base}/dolphinscheduler-master.log</file>
+ <!--<filter
class="org.apache.dolphinscheduler.server.log.MasterLogFilter">
+ <level>INFO</level>
+ </filter>-->
<rollingPolicy
class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
-
<fileNamePattern>${log.base}/dolphinscheduler-worker.%d{yyyy-MM-dd_HH}.%i.log</fileNamePattern>
+
<fileNamePattern>${log.base}/dolphinscheduler-master.%d{yyyy-MM-dd_HH}.%i.log</fileNamePattern>
<maxHistory>168</maxHistory>
<maxFileSize>200MB</maxFileSize>
</rollingPolicy>
-
<encoder>
<pattern>
[%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] -
%msg%n
</pattern>
<charset>UTF-8</charset>
</encoder>
-
</appender>
-
+ <!-- master server logback config end -->
<root level="INFO">
- <appender-ref ref="TASKLOGFILE"/>
- <appender-ref ref="WORKERLOGFILE"/>
+ <appender-ref ref="STDOUT"/>
+ <appender-ref ref="MASTERLOGFILE"/>
</root>
+
</configuration>
\ No newline at end of file
diff --git a/dockerfile/conf/dolphinscheduler/conf/worker_logback.xml
b/dolphinscheduler-server/src/main/resources/logback-worker.xml
similarity index 79%
copy from dockerfile/conf/dolphinscheduler/conf/worker_logback.xml
copy to dolphinscheduler-server/src/main/resources/logback-worker.xml
index bf4a651..be1d0ac 100644
--- a/dockerfile/conf/dolphinscheduler/conf/worker_logback.xml
+++ b/dolphinscheduler-server/src/main/resources/logback-worker.xml
@@ -1,4 +1,4 @@
-<?xml version="1.0" encoding="UTF-8" ?>
+<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
@@ -17,7 +17,8 @@
-->
<!-- Logback configuration. See http://logback.qos.ch/manual/index.html -->
-<configuration scan="true" scanPeriod="120 seconds">
+<configuration scan="true" scanPeriod="120 seconds"> <!--debug="true" -->
+
<property name="log.base" value="logs"/>
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
@@ -27,12 +28,16 @@
<charset>UTF-8</charset>
</encoder>
</appender>
+
+ <!-- worker server logback config start -->
+ <conversionRule conversionWord="messsage"
+
converterClass="org.apache.dolphinscheduler.server.log.SensitiveDataConverter"/>
<appender name="TASKLOGFILE"
class="ch.qos.logback.classic.sift.SiftingAppender">
<filter class="ch.qos.logback.classic.filter.ThresholdFilter">
<level>INFO</level>
</filter>
- <filter
class="org.apache.dolphinscheduler.common.log.TaskLogFilter"></filter>
- <Discriminator
class="org.apache.dolphinscheduler.common.log.TaskLogDiscriminator">
+ <filter class="org.apache.dolphinscheduler.server.log.TaskLogFilter"/>
+ <Discriminator
class="org.apache.dolphinscheduler.server.log.TaskLogDiscriminator">
<key>taskAppId</key>
<logBase>${log.base}</logBase>
</Discriminator>
@@ -41,7 +46,7 @@
<file>${log.base}/${taskAppId}.log</file>
<encoder>
<pattern>
- [%level] %date{yyyy-MM-dd HH:mm:ss.SSS}
%logger{96}:[%line] - %msg%n
+ [%level] %date{yyyy-MM-dd HH:mm:ss.SSS}
%logger{96}:[%line] - %messsage%n
</pattern>
<charset>UTF-8</charset>
</encoder>
@@ -49,31 +54,30 @@
</appender>
</sift>
</appender>
-
<appender name="WORKERLOGFILE"
class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>${log.base}/dolphinscheduler-worker.log</file>
- <filter class="org.apache.dolphinscheduler.common.log.WorkerLogFilter">
+ <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
<level>INFO</level>
</filter>
-
+ <filter
class="org.apache.dolphinscheduler.server.log.WorkerLogFilter"/>
<rollingPolicy
class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
<fileNamePattern>${log.base}/dolphinscheduler-worker.%d{yyyy-MM-dd_HH}.%i.log</fileNamePattern>
<maxHistory>168</maxHistory>
<maxFileSize>200MB</maxFileSize>
</rollingPolicy>
-
<encoder>
<pattern>
- [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] -
%msg%n
+ [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] -
%messsage%n
</pattern>
<charset>UTF-8</charset>
</encoder>
-
</appender>
-
+ <!-- worker server logback config end -->
<root level="INFO">
+ <appender-ref ref="STDOUT"/>
<appender-ref ref="TASKLOGFILE"/>
<appender-ref ref="WORKERLOGFILE"/>
</root>
+
</configuration>
\ No newline at end of file
diff --git
a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/MasterLogFilterTest.java
b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/log/MasterLogFilterTest.java
similarity index 98%
rename from
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/MasterLogFilterTest.java
rename to
dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/log/MasterLogFilterTest.java
index 8cf6cfc..1a54695 100644
---
a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/MasterLogFilterTest.java
+++
b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/log/MasterLogFilterTest.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.dolphinscheduler.common.log;
+package org.apache.dolphinscheduler.server.log;
import ch.qos.logback.classic.Level;
import ch.qos.logback.classic.spi.ILoggingEvent;
diff --git
a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/SensitiveDataConverterTest.java
b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/log/SensitiveDataConverterTest.java
similarity index 99%
rename from
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/SensitiveDataConverterTest.java
rename to
dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/log/SensitiveDataConverterTest.java
index 727ab41..6319bf1 100644
---
a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/SensitiveDataConverterTest.java
+++
b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/log/SensitiveDataConverterTest.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.dolphinscheduler.common.log;
+package org.apache.dolphinscheduler.server.log;
import ch.qos.logback.classic.Level;
diff --git
a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/TaskLogDiscriminatorTest.java
b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/log/TaskLogDiscriminatorTest.java
similarity index 97%
rename from
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/TaskLogDiscriminatorTest.java
rename to
dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/log/TaskLogDiscriminatorTest.java
index 8745a4f..1908475 100644
---
a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/TaskLogDiscriminatorTest.java
+++
b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/log/TaskLogDiscriminatorTest.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.dolphinscheduler.common.log;
+package org.apache.dolphinscheduler.server.log;
import ch.qos.logback.classic.Level;
import ch.qos.logback.classic.spi.ILoggingEvent;
@@ -27,8 +27,6 @@ import org.slf4j.Marker;
import java.util.Map;
-import static org.junit.Assert.*;
-
public class TaskLogDiscriminatorTest {
/**
diff --git
a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/TaskLogFilterTest.java
b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/log/TaskLogFilterTest.java
similarity index 96%
rename from
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/TaskLogFilterTest.java
rename to
dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/log/TaskLogFilterTest.java
index 5276707..d8abb48 100644
---
a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/TaskLogFilterTest.java
+++
b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/log/TaskLogFilterTest.java
@@ -14,14 +14,13 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.dolphinscheduler.common.log;
+package org.apache.dolphinscheduler.server.log;
import ch.qos.logback.classic.Level;
import ch.qos.logback.classic.spi.ILoggingEvent;
import ch.qos.logback.classic.spi.IThrowableProxy;
import ch.qos.logback.classic.spi.LoggerContextVO;
import ch.qos.logback.core.spi.FilterReply;
-import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.utils.LoggerUtils;
import org.junit.Assert;
import org.junit.Test;
diff --git
a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/WorkerLogFilterTest.java
b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/log/WorkerLogFilterTest.java
similarity index 98%
rename from
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/WorkerLogFilterTest.java
rename to
dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/log/WorkerLogFilterTest.java
index 90b1544..dbcd4b8 100644
---
a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/WorkerLogFilterTest.java
+++
b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/log/WorkerLogFilterTest.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.dolphinscheduler.common.log;
+package org.apache.dolphinscheduler.server.log;
import ch.qos.logback.classic.Level;
import ch.qos.logback.classic.spi.ILoggingEvent;
diff --git a/script/dolphinscheduler-daemon.sh
b/script/dolphinscheduler-daemon.sh
index abc33a4..d7ef66f 100644
--- a/script/dolphinscheduler-daemon.sh
+++ b/script/dolphinscheduler-daemon.sh
@@ -57,16 +57,16 @@ pid=$DOLPHINSCHEDULER_LOG_DIR/dolphinscheduler-$command.pid
cd $DOLPHINSCHEDULER_HOME
if [ "$command" = "api-server" ]; then
- LOG_FILE="-Dserver=api-server -Dspring.profiles.active=api"
+ LOG_FILE="-Dlogging.config=classpath:logback-api.xml
-Dspring.profiles.active=api"
CLASS=org.apache.dolphinscheduler.api.ApiApplicationServer
elif [ "$command" = "master-server" ]; then
- LOG_FILE="-Dserver=master-server -Ddruid.mysql.usePingMethod=false"
+ LOG_FILE="-Dlogging.config=classpath:logback-master.xml
-Ddruid.mysql.usePingMethod=false"
CLASS=org.apache.dolphinscheduler.server.master.MasterServer
elif [ "$command" = "worker-server" ]; then
- LOG_FILE="-Dserver=worker-server -Ddruid.mysql.usePingMethod=false"
+ LOG_FILE="-Dlogging.config=classpath:logback-worker.xml
-Ddruid.mysql.usePingMethod=false"
CLASS=org.apache.dolphinscheduler.server.worker.WorkerServer
elif [ "$command" = "alert-server" ]; then
- LOG_FILE="-Dserver=alert-server"
+ LOG_FILE="-Dlogback.configurationFile=conf/logback-alert.xml"
CLASS=org.apache.dolphinscheduler.alert.AlertServer
elif [ "$command" = "logger-server" ]; then
CLASS=org.apache.dolphinscheduler.server.log.LoggerServer
