This is an automated email from the ASF dual-hosted git repository.
dockerzhang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
The following commit(s) were added to refs/heads/master by this push:
new d674bfe284 [INLONG-9330][Manager] Add encoding check to the StarRocks
JDBC URL (#9331)
d674bfe284 is described below
commit d674bfe28416aff728eabafc1f6b8bb9ba5a5b8e
Author: Hao <[email protected]>
AuthorDate: Tue Nov 28 12:52:45 2023 +0800
[INLONG-9330][Manager] Add encoding check to the StarRocks JDBC URL (#9331)
Co-authored-by: healchow <[email protected]>
---
.../pojo/node/starrocks/StarRocksDataNodeDTO.java | 8 ++
.../manager/pojo/sink/mysql/MySQLSinkDTO.java | 73 +-------------
.../manager/pojo/util/MySQLSensitiveUrlUtils.java | 105 +++++++++++++++++++++
.../node/starrocks/StarRocksDataNodeOperator.java | 2 +-
4 files changed, 116 insertions(+), 72 deletions(-)
diff --git
a/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/node/starrocks/StarRocksDataNodeDTO.java
b/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/node/starrocks/StarRocksDataNodeDTO.java
index 29823947b3..5d5238b42f 100644
---
a/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/node/starrocks/StarRocksDataNodeDTO.java
+++
b/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/node/starrocks/StarRocksDataNodeDTO.java
@@ -21,6 +21,7 @@ import org.apache.inlong.manager.common.enums.ErrorCodeEnum;
import org.apache.inlong.manager.common.exceptions.BusinessException;
import org.apache.inlong.manager.common.util.CommonBeanUtils;
import org.apache.inlong.manager.common.util.JsonUtils;
+import org.apache.inlong.manager.pojo.util.MySQLSensitiveUrlUtils;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
@@ -67,4 +68,11 @@ public class StarRocksDataNodeDTO {
}
}
+ /**
+ * Convert ip:post to jdbcurl.
+ */
+ public static String convertToJdbcUrl(String url) {
+ return MySQLSensitiveUrlUtils.filterSensitive(url);
+ }
+
}
diff --git
a/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/sink/mysql/MySQLSinkDTO.java
b/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/sink/mysql/MySQLSinkDTO.java
index 983d5da3ef..5b5750a803 100644
---
a/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/sink/mysql/MySQLSinkDTO.java
+++
b/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/sink/mysql/MySQLSinkDTO.java
@@ -22,6 +22,7 @@ import org.apache.inlong.manager.common.enums.ErrorCodeEnum;
import org.apache.inlong.manager.common.exceptions.BusinessException;
import org.apache.inlong.manager.common.util.CommonBeanUtils;
import org.apache.inlong.manager.common.util.JsonUtils;
+import org.apache.inlong.manager.pojo.util.MySQLSensitiveUrlUtils;
import com.google.common.base.Strings;
import io.swagger.annotations.ApiModelProperty;
@@ -35,13 +36,8 @@ import org.slf4j.LoggerFactory;
import javax.validation.constraints.NotNull;
-import java.net.URLDecoder;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
import java.util.List;
import java.util.Map;
-import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -54,25 +50,6 @@ import java.util.regex.Pattern;
@AllArgsConstructor
public class MySQLSinkDTO {
- /**
- * The sensitive param may lead the attack.
- */
- private static final Map<String, String> SENSITIVE_REPLACE_PARAM_MAP = new
HashMap<String, String>() {
-
- {
- put("autoDeserialize", "false");
- put("allowLoadLocalInfile", "false");
- put("allowUrlInLocalInfile", "false");
- }
- };
-
- private static final Set<String> SENSITIVE_REMOVE_PARAM_MAP = new
HashSet<String>() {
-
- {
- add("allowLoadLocalInfileInPath");
- }
- };
-
private static final Logger LOGGER =
LoggerFactory.getLogger(MySQLSinkDTO.class);
private static final String MYSQL_JDBC_PREFIX = "jdbc:mysql://";
@@ -216,54 +193,8 @@ public class MySQLSinkDTO {
return resultUrl.toString();
}
- /**
- * Filter the sensitive params for the given URL.
- *
- * @param url str may have some sensitive params
- * @return str without sensitive param
- */
public static String filterSensitive(String url) {
- if (StringUtils.isBlank(url)) {
- return url;
- }
-
- try {
- String resultUrl = url;
- while (resultUrl.contains(InlongConstants.PERCENT)) {
- resultUrl = URLDecoder.decode(resultUrl, "UTF-8");
- }
- resultUrl = resultUrl.replaceAll(InlongConstants.REGEX_WHITESPACE,
InlongConstants.EMPTY);
-
- if (resultUrl.contains(InlongConstants.QUESTION_MARK)) {
- StringBuilder builder = new StringBuilder();
- builder.append(StringUtils.substringBefore(resultUrl,
InlongConstants.QUESTION_MARK));
- builder.append(InlongConstants.QUESTION_MARK);
-
- List<String> paramList = new ArrayList<>();
- String queryString = StringUtils.substringAfter(resultUrl,
InlongConstants.QUESTION_MARK);
- for (String param :
queryString.split(InlongConstants.AMPERSAND)) {
- String key = StringUtils.substringBefore(param,
InlongConstants.EQUAL);
- String value = StringUtils.substringAfter(param,
InlongConstants.EQUAL);
-
- if (SENSITIVE_REMOVE_PARAM_MAP.contains(key) ||
SENSITIVE_REPLACE_PARAM_MAP.containsKey(key)) {
- continue;
- }
-
- paramList.add(key + InlongConstants.EQUAL + value);
- }
- SENSITIVE_REPLACE_PARAM_MAP.forEach((key, value) ->
paramList.add(key + InlongConstants.EQUAL + value));
-
- String params = StringUtils.join(paramList,
InlongConstants.AMPERSAND);
- builder.append(params);
- resultUrl = builder.toString();
- }
-
- LOGGER.info("the origin url [{}] was replaced to: [{}]", url,
resultUrl);
- return resultUrl;
- } catch (Exception e) {
- throw new BusinessException(ErrorCodeEnum.SINK_INFO_INCORRECT,
- ErrorCodeEnum.SINK_INFO_INCORRECT.getMessage() + ": " +
e.getMessage());
- }
+ return MySQLSensitiveUrlUtils.filterSensitive(url);
}
}
diff --git
a/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/util/MySQLSensitiveUrlUtils.java
b/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/util/MySQLSensitiveUrlUtils.java
new file mode 100644
index 0000000000..ea9361ae42
--- /dev/null
+++
b/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/util/MySQLSensitiveUrlUtils.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.manager.pojo.util;
+
+import org.apache.inlong.manager.common.consts.InlongConstants;
+import org.apache.inlong.manager.common.exceptions.BaseException;
+
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
+
+import java.net.URLDecoder;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+@Slf4j
+public class MySQLSensitiveUrlUtils {
+
+ /**
+ * The sensitive param may lead the attack.
+ */
+ private static final Map<String, String> SENSITIVE_REPLACE_PARAM_MAP = new
HashMap<String, String>() {
+
+ {
+ put("autoDeserialize", "false");
+ put("allowLoadLocalInfile", "false");
+ put("allowUrlInLocalInfile", "false");
+ }
+ };
+
+ private static final Set<String> SENSITIVE_REMOVE_PARAM_MAP = new
HashSet<String>() {
+
+ {
+ add("allowLoadLocalInfileInPath");
+ }
+ };
+
+ /**
+ * Filter the sensitive params for the given URL.
+ *
+ * @param url str may have some sensitive params
+ * @return str without sensitive param
+ */
+ public static String filterSensitive(String url) {
+ if (StringUtils.isBlank(url)) {
+ return url;
+ }
+
+ try {
+ String resultUrl = url;
+ while (resultUrl.contains(InlongConstants.PERCENT)) {
+ resultUrl = URLDecoder.decode(resultUrl, "UTF-8");
+ }
+ resultUrl = resultUrl.replaceAll(InlongConstants.REGEX_WHITESPACE,
InlongConstants.EMPTY);
+
+ if (resultUrl.contains(InlongConstants.QUESTION_MARK)) {
+ StringBuilder builder = new StringBuilder();
+ builder.append(StringUtils.substringBefore(resultUrl,
InlongConstants.QUESTION_MARK));
+ builder.append(InlongConstants.QUESTION_MARK);
+
+ List<String> paramList = new ArrayList<>();
+ String queryString = StringUtils.substringAfter(resultUrl,
InlongConstants.QUESTION_MARK);
+ for (String param :
queryString.split(InlongConstants.AMPERSAND)) {
+ String key = StringUtils.substringBefore(param,
InlongConstants.EQUAL);
+ String value = StringUtils.substringAfter(param,
InlongConstants.EQUAL);
+
+ if (SENSITIVE_REMOVE_PARAM_MAP.contains(key) ||
SENSITIVE_REPLACE_PARAM_MAP.containsKey(key)) {
+ continue;
+ }
+
+ paramList.add(key + InlongConstants.EQUAL + value);
+ }
+ SENSITIVE_REPLACE_PARAM_MAP.forEach((key, value) ->
paramList.add(key + InlongConstants.EQUAL + value));
+
+ String params = StringUtils.join(paramList,
InlongConstants.AMPERSAND);
+ builder.append(params);
+ resultUrl = builder.toString();
+ }
+
+ log.info("MySQL original URL {} was replaced to {}", url,
resultUrl);
+ return resultUrl;
+ } catch (Exception e) {
+ throw new BaseException(String.format("Failed to filter MySQL
sensitive URL: %s, error: %s",
+ url, e.getMessage()));
+ }
+ }
+}
diff --git
a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/node/starrocks/StarRocksDataNodeOperator.java
b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/node/starrocks/StarRocksDataNodeOperator.java
index 24cf88ee61..974e2b32a1 100644
---
a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/node/starrocks/StarRocksDataNodeOperator.java
+++
b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/node/starrocks/StarRocksDataNodeOperator.java
@@ -88,7 +88,7 @@ public class StarRocksDataNodeOperator extends
AbstractDataNodeOperator {
@Override
public Boolean testConnection(DataNodeRequest request) {
- String jdbcUrl = request.getUrl();
+ String jdbcUrl =
StarRocksDataNodeDTO.convertToJdbcUrl(request.getUrl());
String username = request.getUsername();
String password = request.getToken();
Preconditions.expectNotBlank(jdbcUrl, ErrorCodeEnum.INVALID_PARAMETER,
"connection jdbcUrl cannot be empty");
