[dolphinscheduler] branch dev updated: [Improvement][Task] Mask configYaml in task log (#14571)

[chufenggao]

This is an automated email from the ASF dual-hosted git repository.

chufenggao pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git


The following commit(s) were added to refs/heads/dev by this push:
     new 9d10de6efa [Improvement][Task] Mask configYaml in task log (#14571)
9d10de6efa is described below

commit 9d10de6efa2b0fa610ae1d53646a38ad58254a45
Author: Gallardot <gallar...@apache.org>
AuthorDate: Tue Jul 18 15:02:20 2023 +0800

    [Improvement][Task] Mask configYaml in task log (#14571)
    
    * [Improvement][Task] Mask configYaml in task log
    
    Signed-off-by: Gallardot <gallar...@apache.org>
    ---------
    
    Signed-off-by: Gallardot <gallar...@apache.org>
    Co-authored-by: Aaron Wang <wangweira...@gmail.com>
---
 .../common/constants/Constants.java                |  5 ++++
 .../common/log/SensitiveDataConverterTest.java     | 29 ++++++++++++++++++++++
 .../worker/runner/WorkerTaskExecuteRunnable.java   |  3 +++
 3 files changed, 37 insertions(+)

diff --git 
a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/Constants.java
 
b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/Constants.java
index 3bef092488..f214e7aea7 100644
--- 
a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/Constants.java
+++ 
b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/Constants.java
@@ -856,4 +856,9 @@ public final class Constants {
      */
     public static final String DATABASES_QUERY = "show databases";
     public static final String DATABASES_QUERY_PG = "SELECT datname FROM 
pg_database";
+
+    /**
+     * K8S sensitive param
+     */
+    public static final String K8S_CONFIG_REGEX = "(?<=((?i)configYaml(\" : 
\"))).*?(?=(\",\\n))";
 }
diff --git 
a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/SensitiveDataConverterTest.java
 
b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/SensitiveDataConverterTest.java
index 2268865708..17dcaf6274 100644
--- 
a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/SensitiveDataConverterTest.java
+++ 
b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/log/SensitiveDataConverterTest.java
@@ -17,6 +17,8 @@
 
 package org.apache.dolphinscheduler.common.log;
 
+import static 
org.apache.dolphinscheduler.common.constants.Constants.K8S_CONFIG_REGEX;
+
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.Test;
 import org.slf4j.Logger;
@@ -62,4 +64,31 @@ public class SensitiveDataConverterTest {
         Assertions.assertEquals(expectedMsg, maskedLog);
     }
 
+    @Test
+    public void testK8SLogMsgConverter() {
+        String msg = "End initialize task {\n" +
+                "  \"taskName\" : \"echo\",\n" +
+                "  \"k8sTaskExecutionContext\" : {\n" +
+                "    \"configYaml\" : \"apiVersion: v1 xxx client-key-data: 
==\",\n" +
+                "    \"namespace\" : \"abc\"\n" +
+                "  },\n" +
+                "  \"logBufferEnable\" : false\n" +
+                "}";
+        String maskMsg = "End initialize task {\n" +
+                "  \"taskName\" : \"echo\",\n" +
+                "  \"k8sTaskExecutionContext\" : {\n" +
+                "    \"configYaml\" : 
\"**************************************\",\n" +
+                "    \"namespace\" : \"abc\"\n" +
+                "  },\n" +
+                "  \"logBufferEnable\" : false\n" +
+                "}";
+        SensitiveDataConverter.addMaskPattern(K8S_CONFIG_REGEX);
+        final String maskedLog = SensitiveDataConverter.maskSensitiveData(msg);
+
+        logger.info("original parameter : {}", msg);
+        logger.info("masked parameter : {}", maskedLog);
+
+        Assertions.assertEquals(maskMsg, maskedLog);
+
+    }
 }
diff --git 
a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerTaskExecuteRunnable.java
 
b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerTaskExecuteRunnable.java
index 46629f3634..5e985c628c 100644
--- 
a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerTaskExecuteRunnable.java
+++ 
b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerTaskExecuteRunnable.java
@@ -19,9 +19,11 @@ package org.apache.dolphinscheduler.server.worker.runner;
 
 import static ch.qos.logback.classic.ClassicConstants.FINALIZE_SESSION_MARKER;
 import static 
org.apache.dolphinscheduler.common.constants.Constants.DRY_RUN_FLAG_YES;
+import static 
org.apache.dolphinscheduler.common.constants.Constants.K8S_CONFIG_REGEX;
 import static 
org.apache.dolphinscheduler.common.constants.Constants.SINGLE_SLASH;
 
 import org.apache.dolphinscheduler.common.enums.WarningType;
+import org.apache.dolphinscheduler.common.log.SensitiveDataConverter;
 import org.apache.dolphinscheduler.common.log.remote.RemoteLogUtils;
 import org.apache.dolphinscheduler.common.utils.JSONUtils;
 import org.apache.dolphinscheduler.plugin.datasource.api.utils.CommonUtils;
@@ -93,6 +95,7 @@ public abstract class WorkerTaskExecuteRunnable implements 
Runnable {
         this.taskPluginManager = taskPluginManager;
         this.storageOperate = storageOperate;
         this.workerRegistryClient = workerRegistryClient;
+        SensitiveDataConverter.addMaskPattern(K8S_CONFIG_REGEX);
     }
 
     protected abstract void executeTask(TaskCallBack taskCallBack);