github-advanced-security[bot] commented on code in PR #4419: URL: https://github.com/apache/streampark/pull/4419#discussion_r3537393307
########## streampark-flink/streampark-flink-connector/streampark-flink-connector-clickhouse/src/main/java/org/apache/streampark/flink/connector/clickhouse/internal/ClickHouseSinkFunction.java: ########## @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.flink.connector.clickhouse.internal; + +import org.apache.streampark.common.util.JdbcUtils; +import org.apache.streampark.flink.connector.clickhouse.conf.ClickHouseJdbcConfig; +import org.apache.streampark.flink.connector.clickhouse.util.ClickhouseConvertUtils; +import org.apache.streampark.flink.connector.function.TransformFunction; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.streaming.api.functions.sink.RichSinkFunction; +import org.apache.flink.streaming.api.functions.sink.SinkFunction; +import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import ru.yandex.clickhouse.ClickHouseDataSource; +import ru.yandex.clickhouse.settings.ClickHouseProperties; +import java.lang.reflect.Field; +import java.sql.Connection; import java.sql.Statement; +import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Properties; import java.util.concurrent.atomic.AtomicLong; + +public class ClickHouseSinkFunction<T> extends RichSinkFunction<T> { + private static final Logger LOG = LoggerFactory.getLogger(ClickHouseSinkFunction.class); + private Connection connection; private Statement statement; + private final ClickHouseJdbcConfig clickHouseConf; + private final int batchSize; private final AtomicLong offset = new AtomicLong(0L); + private long timestamp = 0L; private final long flushInterval; + private final List<String> sqlValues = new ArrayList<>(); + private String insertSqlPrefixes; + private final TransformFunction<T, String> sqlFunc; + + public ClickHouseSinkFunction(Properties properties, TransformFunction<T, String> sqlFunc) { + this.clickHouseConf = new ClickHouseJdbcConfig(properties); + this.batchSize = clickHouseConf.batchSize; + this.flushInterval = clickHouseConf.flushInterval; + this.sqlFunc = sqlFunc; + } + + @Override public void open(Configuration parameters) throws Exception { + String user = clickHouseConf.user; String driver = clickHouseConf.driverClassName; + ClickHouseProperties chProps = new ClickHouseProperties(); + if (user != null && driver != null) { Class.forName(driver); chProps.setUser(user); } + else if (driver != null) Class.forName(driver); + else if (user != null) chProps.setUser(user); + for (Map.Entry<Object,Object> x : clickHouseConf.sinkOption.getInternalConfig().entrySet()) { + try { + Field field = chProps.getClass().getDeclaredField(x.getKey().toString()); + field.setAccessible(true); + String tn = field.getType().getSimpleName(); + if ("String".equals(tn)) field.set(chProps, x.getValue()); + else if ("int".equals(tn) || "Integer".equals(tn)) field.set(chProps, Integer.parseInt(x.getValue().toString())); + else if ("long".equals(tn) || "Long".equals(tn)) field.set(chProps, Long.parseLong(x.getValue().toString())); + else if ("boolean".equals(tn) || "Boolean".equals(tn)) field.set(chProps, Boolean.parseBoolean(x.getValue().toString())); + } catch (NoSuchFieldException e) { + LOG.warn("ClickHouseProperties config error, property:{} invalid", x.getKey()); + } + } + connection = new ClickHouseDataSource(clickHouseConf.jdbcUrl, chProps).getConnection(); + } + + @Override public void invoke(T value, SinkFunction.Context context) throws Exception { + String sql = sqlFunc != null ? sqlFunc.transform(value) : ClickhouseConvertUtils.convert(value); + if (batchSize == 1) { + connection.prepareStatement(sql).executeUpdate(); + } else { + sqlValues.add(sql); + long cnt = offset.incrementAndGet(); + long now = System.currentTimeMillis(); + if (cnt % batchSize == 0 || now - timestamp > flushInterval) execBatch(); + } + } + + @Override public void close() throws Exception { execBatch(); JdbcUtils.close(statement, connection); } + + private void execBatch() throws Exception { + if (offset.get() > 0) { + try { + LOG.info("ClickHouseSink batch {} insert begin..", offset.get()); + offset.set(0); + String valuesStr = String.join(",", sqlValues); + connection.prepareStatement(insertSqlPrefixes + " " + valuesStr).executeUpdate(); Review Comment: ## SonarCloud / SQL queries should not be dynamically formatted <!--SONAR_ISSUE_KEY:AZ8yvm3flWsfFY_QVhgz-->Make sure using a dynamically formatted SQL query is safe here. <p>See more on <a href="https://sonarcloud.io/project/issues?id=apache_incubator-streampark&issues=AZ8yvm3flWsfFY_QVhgz&open=AZ8yvm3flWsfFY_QVhgz&pullRequest=4419";>SonarQube Cloud</a></p> [Show more details](https://github.com/apache/streampark/security/code-scanning/122) ########## streampark-common/src/main/java/org/apache/streampark/common/util/ZooKeeperUtils.java: ########## @@ -0,0 +1,161 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.common.util; + +import org.apache.curator.RetryPolicy; +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.CuratorFrameworkFactory; +import org.apache.curator.retry.RetryNTimes; +import org.apache.zookeeper.CreateMode; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +public final class ZooKeeperUtils { + + private static final String CONNECT = "localhost:2181"; + private static final Map<String, CuratorFramework> CLIENT_MAP = new ConcurrentHashMap<>(); + + private ZooKeeperUtils() {} + + public static CuratorFramework getClient() { + return getClient(CONNECT); + } + + public static CuratorFramework getClient(String url) { + CuratorFramework existing = CLIENT_MAP.get(url); + if (existing != null) { + return existing; + } + try { + RetryPolicy retryPolicy = new RetryNTimes(5, 2000); + CuratorFramework client = + CuratorFrameworkFactory.builder() + .connectString(url) + .retryPolicy(retryPolicy) + .connectionTimeoutMs(2000) + .build(); + client.start(); + CLIENT_MAP.put(url, client); + return client; + } catch (Exception e) { + throw new IllegalStateException(e.getMessage(), e); + } + } + + public static void close(String url) { + CuratorFramework client = getClient(url); + if (client != null) { + client.close(); + CLIENT_MAP.remove(url); + } + } + + public static List<String> listChildren(String path) throws Exception { + return listChildren(path, CONNECT); + } + + public static List<String> listChildren(String path, String url) throws Exception { + CuratorFramework client = getClient(url); + if (client.checkExists().forPath(path) == null) { + return new ArrayList<>(); + } + return client.getChildren().forPath(path); + } + + public static boolean create(String path) throws Exception { + return create(path, null, CONNECT, false); + } + + public static boolean create(String path, String value, String url, boolean persistent) + throws Exception { + try { + CuratorFramework client = getClient(url); + if (client.checkExists().forPath(path) == null) { + byte[] data = + value == null || value.isEmpty() + ? new byte[0] + : value.getBytes(StandardCharsets.UTF_8); + CreateMode mode = persistent ? CreateMode.PERSISTENT : CreateMode.EPHEMERAL; + String opResult = + client.create().creatingParentsIfNeeded().withMode(mode).forPath(path, data); + return path.equals(opResult); + } + return false; + } catch (Exception e) { + e.printStackTrace(); + return false; + } + } + + public static boolean update(String path, String value, String url, boolean persistent) + throws Exception { + try { + CuratorFramework client = getClient(url); + if (client.checkExists().forPath(path) == null) { + CreateMode mode = persistent ? CreateMode.PERSISTENT : CreateMode.EPHEMERAL; + String opResult = + client.create() + .creatingParentsIfNeeded() + .withMode(mode) + .forPath(path, value.getBytes(StandardCharsets.UTF_8)); + return path.equals(opResult); + } + client.setData().forPath(path, value.getBytes(StandardCharsets.UTF_8)); + return true; + } catch (Exception e) { + e.printStackTrace(); + return false; + } + } + + public static void delete(String path) throws Exception { + delete(path, CONNECT); + } + + public static void delete(String path, String url) throws Exception { + try { + CuratorFramework client = getClient(url); + if (client.checkExists().forPath(path) != null) { + client.delete().deletingChildrenIfNeeded().forPath(path); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + + public static String get(String path) throws Exception { + return get(path, CONNECT); + } + + public static String get(String path, String url) throws Exception { + try { + CuratorFramework client = getClient(url); + if (client.checkExists().forPath(path) == null) { + return null; + } + return new String(client.getData().forPath(path)); + } catch (Exception e) { + e.printStackTrace(); Review Comment: ## SonarCloud / Debugging features should not be enabled in production <!--SONAR_ISSUE_KEY:AZ8yvnLwlWsfFY_QVhpw-->Make sure this debug feature is deactivated before delivering the code in production. <p>See more on <a href="https://sonarcloud.io/project/issues?id=apache_incubator-streampark&issues=AZ8yvnLwlWsfFY_QVhpw&open=AZ8yvnLwlWsfFY_QVhpw&pullRequest=4419";>SonarQube Cloud</a></p> [Show more details](https://github.com/apache/streampark/security/code-scanning/134) ########## streampark-flink/streampark-flink-shims/streampark-flink-shims-base-v2/src/main/java/org/apache/streampark/flink/core/conf/ParameterCli.java: ########## @@ -0,0 +1,206 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.flink.core.conf; + +import org.apache.streampark.common.conf.ConfigKeys; +import org.apache.streampark.common.util.PropertiesUtils; + +import org.apache.commons.cli.DefaultParser; +import org.apache.commons.cli.Options; + +import java.net.URLClassLoader; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** Parses Flink application CLI parameters from configuration files. */ +public final class ParameterCli { + + private static final String PROPERTY_PREFIX = ConfigKeys.KEY_FLINK_PROPERTY_PREFIX(); + private static final String OPTION_PREFIX = ConfigKeys.KEY_FLINK_OPTION_PREFIX(); + private static final String OPTION_MAIN = PROPERTY_PREFIX + "$internal.application.main"; + + private static final Options FLINK_OPTIONS = FlinkRunOption.allOptions(); + private static final DefaultParser PARSER = new DefaultParser(); + + private ParameterCli() { + } + + public static void main(String[] args) { + System.out.print(read(args)); + } + + public static String read(String[] args) { + switch (args[0]) { + case "--vmopt": + ClassLoader classLoader = ClassLoader.getSystemClassLoader(); + if (classLoader instanceof URLClassLoader) { + return ""; + } + return "--add-opens java.base/jdk.internal.loader=ALL-UNNAMED " + + "--add-opens jdk.zipfs/jdk.nio.zipfs=ALL-UNNAMED"; + default: + String action = args[0]; + String conf = args[1]; + Map<String, String> map; + try { + String extension = conf.substring(conf.lastIndexOf('.') + 1).toLowerCase(); + switch (extension) { + case "yml": + case "yaml": + map = PropertiesUtils.fromYamlFile(conf); + break; + case "conf": + map = PropertiesUtils.fromHoconFile(conf); + break; + case "properties": + map = PropertiesUtils.fromPropertiesFile(conf); + break; + default: + throw new IllegalArgumentException( + "[StreamPark] Usage:flink.conf file error,must be (yml|conf|properties)"); + } + } catch (Exception e) { + map = Collections.emptyMap(); + } + String[] programArgs = new String[args.length - 2]; + System.arraycopy(args, 2, programArgs, 0, programArgs.length); + switch (action) { + case "--option": + String[] option = getOption(map, programArgs); + StringBuilder buffer = new StringBuilder(); + try { + org.apache.commons.cli.CommandLine line = + PARSER.parse(FLINK_OPTIONS, option, false); + for (org.apache.commons.cli.Option x : line.getOptions()) { + buffer.append(" -").append(x.getOpt()); + if (x.hasArg()) { + buffer.append(" ").append(x.getValue()); + } + } + } catch (Exception exception) { + exception.printStackTrace(); + } + String mainClass = map.get(OPTION_MAIN); + if (mainClass != null) { + buffer.append(" -c ").append(mainClass); + } + return buffer.toString().trim(); + case "--property": + StringBuilder propertyBuffer = new StringBuilder(); + for (Map.Entry<String, String> entry : map.entrySet()) { + String key = entry.getKey(); + String value = entry.getValue(); + if (!OPTION_MAIN.equals(key) + && key.startsWith(PROPERTY_PREFIX) + && value != null + && !value.isEmpty()) { + String propertyKey = key.substring(PROPERTY_PREFIX.length()).trim(); + String propertyValue = value.trim(); + if (ConfigKeys.KEY_FLINK_APP_NAME().equals(propertyKey)) { + propertyBuffer + .append(" -D") + .append(propertyKey) + .append("=") + .append(propertyValue.replace(" ", "_")); + } else { + propertyBuffer + .append(" -D") + .append(propertyKey) + .append("=") + .append(propertyValue); + } + } + } + return propertyBuffer.toString().trim(); + case "--name": + String appName = + map.getOrDefault( + PROPERTY_PREFIX.concat(ConfigKeys.KEY_FLINK_APP_NAME()), ""); + appName = appName.trim(); + return appName.isEmpty() ? "" : appName; + case "--detached": + String[] detachedOption = getOption(map, programArgs); + try { + org.apache.commons.cli.CommandLine line = + PARSER.parse(FlinkRunOption.allOptions(), detachedOption, false); + boolean detached = + line.hasOption(FlinkRunOption.DETACHED_OPTION.getOpt()) + || line.hasOption( + FlinkRunOption.DETACHED_OPTION.getLongOpt()); + return detached ? "Detached" : "Attach"; + } catch (Exception e) { + e.printStackTrace(); Review Comment: ## SonarCloud / Debugging features should not be enabled in production <!--SONAR_ISSUE_KEY:AZ8yvm9olWsfFY_QVhia-->Make sure this debug feature is deactivated before delivering the code in production. <p>See more on <a href="https://sonarcloud.io/project/issues?id=apache_incubator-streampark&issues=AZ8yvm9olWsfFY_QVhia&open=AZ8yvm9olWsfFY_QVhia&pullRequest=4419";>SonarQube Cloud</a></p> [Show more details](https://github.com/apache/streampark/security/code-scanning/123) ########## streampark-flink/streampark-flink-shims/streampark-flink-shims-base-v2/src/main/java/org/apache/streampark/flink/core/conf/ParameterCli.java: ########## @@ -0,0 +1,206 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.flink.core.conf; + +import org.apache.streampark.common.conf.ConfigKeys; +import org.apache.streampark.common.util.PropertiesUtils; + +import org.apache.commons.cli.DefaultParser; +import org.apache.commons.cli.Options; + +import java.net.URLClassLoader; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** Parses Flink application CLI parameters from configuration files. */ +public final class ParameterCli { + + private static final String PROPERTY_PREFIX = ConfigKeys.KEY_FLINK_PROPERTY_PREFIX(); + private static final String OPTION_PREFIX = ConfigKeys.KEY_FLINK_OPTION_PREFIX(); + private static final String OPTION_MAIN = PROPERTY_PREFIX + "$internal.application.main"; + + private static final Options FLINK_OPTIONS = FlinkRunOption.allOptions(); + private static final DefaultParser PARSER = new DefaultParser(); + + private ParameterCli() { + } + + public static void main(String[] args) { + System.out.print(read(args)); + } + + public static String read(String[] args) { + switch (args[0]) { + case "--vmopt": + ClassLoader classLoader = ClassLoader.getSystemClassLoader(); + if (classLoader instanceof URLClassLoader) { + return ""; + } + return "--add-opens java.base/jdk.internal.loader=ALL-UNNAMED " + + "--add-opens jdk.zipfs/jdk.nio.zipfs=ALL-UNNAMED"; + default: + String action = args[0]; + String conf = args[1]; + Map<String, String> map; + try { + String extension = conf.substring(conf.lastIndexOf('.') + 1).toLowerCase(); + switch (extension) { + case "yml": + case "yaml": + map = PropertiesUtils.fromYamlFile(conf); + break; + case "conf": + map = PropertiesUtils.fromHoconFile(conf); + break; + case "properties": + map = PropertiesUtils.fromPropertiesFile(conf); + break; + default: + throw new IllegalArgumentException( + "[StreamPark] Usage:flink.conf file error,must be (yml|conf|properties)"); + } + } catch (Exception e) { + map = Collections.emptyMap(); + } + String[] programArgs = new String[args.length - 2]; + System.arraycopy(args, 2, programArgs, 0, programArgs.length); + switch (action) { + case "--option": + String[] option = getOption(map, programArgs); + StringBuilder buffer = new StringBuilder(); + try { + org.apache.commons.cli.CommandLine line = + PARSER.parse(FLINK_OPTIONS, option, false); + for (org.apache.commons.cli.Option x : line.getOptions()) { + buffer.append(" -").append(x.getOpt()); + if (x.hasArg()) { + buffer.append(" ").append(x.getValue()); + } + } + } catch (Exception exception) { + exception.printStackTrace(); Review Comment: ## SonarCloud / Debugging features should not be enabled in production <!--SONAR_ISSUE_KEY:AZ8yvm9olWsfFY_QVhiZ-->Make sure this debug feature is deactivated before delivering the code in production. <p>See more on <a href="https://sonarcloud.io/project/issues?id=apache_incubator-streampark&issues=AZ8yvm9olWsfFY_QVhiZ&open=AZ8yvm9olWsfFY_QVhiZ&pullRequest=4419";>SonarQube Cloud</a></p> [Show more details](https://github.com/apache/streampark/security/code-scanning/125) ########## streampark-common/src/main/java/org/apache/streampark/common/util/YarnUtils.java: ########## @@ -0,0 +1,299 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.common.util; + +import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeys; +import org.apache.hadoop.net.NetUtils; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ApplicationReport; +import org.apache.hadoop.yarn.api.records.YarnApplicationState; +import org.apache.hadoop.yarn.conf.HAUtil; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.util.RMHAUtils; +import org.apache.hc.client5.http.config.RequestConfig; +import org.apache.hc.core5.util.Timeout; +import org.apache.streampark.common.conf.CommonConfig; +import org.apache.streampark.common.conf.InternalConfigHolder; +import org.apache.streampark.common.constants.Constants; +import org.apache.streampark.shaded.org.slf4j.Logger; + +import java.io.IOException; +import java.net.InetAddress; +import java.security.PrivilegedExceptionAction; +import java.time.Duration; +import java.util.ArrayList; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.TimeUnit; + +/** YARN cluster utilities. */ +public final class YarnUtils { + + private static final Logger LOG = + StreamParkLoggerFactory.loggerFactory().getLogger(YarnUtils.class.getName()); + + private static String rmHttpURL; + + public static final String PROXY_YARN_URL = + InternalConfigHolder.get(CommonConfig.STREAMPARK_PROXY_YARN_URL()); + + public static final boolean HAS_YARN_HTTP_KERBEROS_AUTH = + "kerberos" + .equalsIgnoreCase(InternalConfigHolder.get(CommonConfig.STREAMPARK_YARN_AUTH())); + + public static final boolean HAS_YARN_HTTP_SIMPLE_AUTH = + "simple".equalsIgnoreCase(InternalConfigHolder.get(CommonConfig.STREAMPARK_YARN_AUTH())); + + private YarnUtils() {} + + public static boolean hasYarnHttpKerberosAuth() { + return HAS_YARN_HTTP_KERBEROS_AUTH; + } + + public static boolean hasYarnHttpSimpleAuth() { + return HAS_YARN_HTTP_SIMPLE_AUTH; + } + + public static List<ApplicationId> getAppId(String appName) { + EnumSet<YarnApplicationState> appStates = + EnumSet.of( + YarnApplicationState.RUNNING, + YarnApplicationState.ACCEPTED, + YarnApplicationState.SUBMITTED); + try { + List<ApplicationId> appIds = new ArrayList<>(); + for (ApplicationReport report : HadoopUtils.yarnClient().getApplications(appStates)) { + if (appName.equals(report.getName())) { + appIds.add(report.getApplicationId()); + } + } + return appIds; + } catch (Exception e) { + e.printStackTrace(); + return new ArrayList<>(); + } + } + + public static YarnApplicationState getState(String appId) { + ApplicationId applicationId = ApplicationId.fromString(appId); + try { + ApplicationReport applicationReport = + HadoopUtils.yarnClient().getApplicationReport(applicationId); + return applicationReport.getYarnApplicationState(); + } catch (Exception e) { + e.printStackTrace(); + return null; + } + } + + public static boolean isContains(String appName) { + try { + List<ApplicationReport> runningApps = + HadoopUtils.yarnClient().getApplications(EnumSet.of(YarnApplicationState.RUNNING)); + if (runningApps != null) { + for (ApplicationReport app : runningApps) { + if (appName.equals(app.getName())) { + return true; + } + } + } + } catch (Exception e) { + e.printStackTrace(); Review Comment: ## SonarCloud / Debugging features should not be enabled in production <!--SONAR_ISSUE_KEY:AZ8yvnK0lWsfFY_QVhpX-->Make sure this debug feature is deactivated before delivering the code in production. <p>See more on <a href="https://sonarcloud.io/project/issues?id=apache_incubator-streampark&issues=AZ8yvnK0lWsfFY_QVhpX&open=AZ8yvnK0lWsfFY_QVhpX&pullRequest=4419";>SonarQube Cloud</a></p> [Show more details](https://github.com/apache/streampark/security/code-scanning/129) ########## streampark-common/src/main/java/org/apache/streampark/common/util/YarnUtils.java: ########## @@ -0,0 +1,299 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.common.util; + +import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeys; +import org.apache.hadoop.net.NetUtils; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ApplicationReport; +import org.apache.hadoop.yarn.api.records.YarnApplicationState; +import org.apache.hadoop.yarn.conf.HAUtil; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.util.RMHAUtils; +import org.apache.hc.client5.http.config.RequestConfig; +import org.apache.hc.core5.util.Timeout; +import org.apache.streampark.common.conf.CommonConfig; +import org.apache.streampark.common.conf.InternalConfigHolder; +import org.apache.streampark.common.constants.Constants; +import org.apache.streampark.shaded.org.slf4j.Logger; + +import java.io.IOException; +import java.net.InetAddress; +import java.security.PrivilegedExceptionAction; +import java.time.Duration; +import java.util.ArrayList; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.TimeUnit; + +/** YARN cluster utilities. */ +public final class YarnUtils { + + private static final Logger LOG = + StreamParkLoggerFactory.loggerFactory().getLogger(YarnUtils.class.getName()); + + private static String rmHttpURL; + + public static final String PROXY_YARN_URL = + InternalConfigHolder.get(CommonConfig.STREAMPARK_PROXY_YARN_URL()); + + public static final boolean HAS_YARN_HTTP_KERBEROS_AUTH = + "kerberos" + .equalsIgnoreCase(InternalConfigHolder.get(CommonConfig.STREAMPARK_YARN_AUTH())); + + public static final boolean HAS_YARN_HTTP_SIMPLE_AUTH = + "simple".equalsIgnoreCase(InternalConfigHolder.get(CommonConfig.STREAMPARK_YARN_AUTH())); + + private YarnUtils() {} + + public static boolean hasYarnHttpKerberosAuth() { + return HAS_YARN_HTTP_KERBEROS_AUTH; + } + + public static boolean hasYarnHttpSimpleAuth() { + return HAS_YARN_HTTP_SIMPLE_AUTH; + } + + public static List<ApplicationId> getAppId(String appName) { + EnumSet<YarnApplicationState> appStates = + EnumSet.of( + YarnApplicationState.RUNNING, + YarnApplicationState.ACCEPTED, + YarnApplicationState.SUBMITTED); + try { + List<ApplicationId> appIds = new ArrayList<>(); + for (ApplicationReport report : HadoopUtils.yarnClient().getApplications(appStates)) { + if (appName.equals(report.getName())) { + appIds.add(report.getApplicationId()); + } + } + return appIds; + } catch (Exception e) { + e.printStackTrace(); Review Comment: ## SonarCloud / Debugging features should not be enabled in production <!--SONAR_ISSUE_KEY:AZ8yvnK0lWsfFY_QVhpV-->Make sure this debug feature is deactivated before delivering the code in production. <p>See more on <a href="https://sonarcloud.io/project/issues?id=apache_incubator-streampark&issues=AZ8yvnK0lWsfFY_QVhpV&open=AZ8yvnK0lWsfFY_QVhpV&pullRequest=4419";>SonarQube Cloud</a></p> [Show more details](https://github.com/apache/streampark/security/code-scanning/127) ########## streampark-common/src/main/java/org/apache/streampark/common/util/JdbcUtils.java: ########## @@ -0,0 +1,329 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.common.util; + +import org.apache.streampark.common.conf.ConfigKeys; + +import com.zaxxer.hikari.HikariConfig; +import com.zaxxer.hikari.HikariDataSource; + +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.locks.ReentrantLock; +import java.util.function.Consumer; + +/** + * Based on the hikari connection pool implementation. Support multiple data sources, note that all + * modifications and additions are automatically committed transactions. + */ +public final class JdbcUtils { + + private static final ConcurrentHashMap<String, ReentrantLock> LOCK_MAP = new ConcurrentHashMap<>(); + + private static final ConcurrentHashMap<String, HikariDataSource> DATA_SOURCE_HOLDER = + new ConcurrentHashMap<>(); + + private JdbcUtils() {} + + public static List<Map<String, Object>> select(String sql, Properties jdbcConfig) { + return select(sql, null, jdbcConfig); + } + + public static List<Map<String, Object>> select( + String sql, Consumer<ResultSet> func, Properties jdbcConfig) { + if (sql == null || sql.isEmpty()) { + return Collections.emptyList(); + } + Connection conn = getConnection(jdbcConfig); + Statement stmt = null; + ResultSet result = null; + try { + stmt = createStatement(conn); + result = stmt.executeQuery(sql); + if (func != null) { + func.accept(result); + } + int count = result.getMetaData().getColumnCount(); + List<Map<String, Object>> array = new ArrayList<>(); + while (result.next()) { + Map<String, Object> map = new HashMap<>(); + for (int x = 1; x <= count; x++) { + String key = result.getMetaData().getColumnLabel(x); + Object value = result.getObject(x); + map.put(key, value); + } + array.add(map); + } + return array.isEmpty() ? Collections.emptyList() : array; + } catch (Exception ex) { + ex.printStackTrace(); + return Collections.emptyList(); + } finally { + close(result, stmt, conn); + } + } + + public static long count(String sql, Properties jdbcConfig) { + Map<String, Object> row = unique(sql, jdbcConfig); + if (row.isEmpty()) { + return 0L; + } + return Long.parseLong(row.values().iterator().next().toString()); + } + + public static long count(Connection conn, String sql) { + Map<String, Object> row = unique(conn, sql); + if (row.isEmpty()) { + return 0L; + } + return Long.parseLong(row.values().iterator().next().toString()); + } + + public static int batch(Iterable<String> sql, Properties jdbcConfig) { + int size = 0; + for (String ignored : sql) { + size++; + } + if (size == 0) { + return 0; + } + if (size == 1) { + for (String s : sql) { + return update(s, jdbcConfig); + } + } + Connection conn = getConnection(jdbcConfig); + try { + Statement prepStat = conn.createStatement(); + try { + int index = 0; + int batchSize = 1000; + int total = 0; + for (String x : sql) { + prepStat.addBatch(x); + index++; + if (index > 0 && index % batchSize == 0) { + int count = 0; + for (int c : prepStat.executeBatch()) { + count += c; + } + conn.commit(); + prepStat.clearBatch(); + total += count; + } + } + for (int c : prepStat.executeBatch()) { + total += c; + } + return total; + } catch (Exception ex) { + ex.printStackTrace(); + return 0; + } finally { + conn.commit(); + close(conn); + } + } catch (Exception ex) { + ex.printStackTrace(); + return 0; + } + } + + public static int update(String sql, Properties jdbcConfig) { + return update(getConnection(jdbcConfig), sql); + } + + public static int update(Connection conn, String sql) { + Statement statement = null; + try { + statement = conn.createStatement(); + return statement.executeUpdate(sql); + } catch (Exception ex) { + ex.printStackTrace(); + return -1; + } finally { + close(statement, conn); + } + } + + public static Map<String, Object> unique(String sql, Properties jdbcConfig) { + return unique(getConnection(jdbcConfig), sql); + } + + public static Map<String, Object> unique(Connection conn, String sql) { + Statement stmt = null; + ResultSet result = null; + try { + stmt = createStatement(conn); + result = stmt.executeQuery(sql); + int count = result.getMetaData().getColumnCount(); + if (!result.next()) { + return Collections.emptyMap(); + } + Map<String, Object> map = new HashMap<>(); + for (int x = 1; x <= count; x++) { + String key = result.getMetaData().getColumnLabel(x); + Object value = result.getObject(x); + map.put(key, value); + } + return map; + } catch (Exception ex) { + ex.printStackTrace(); + return Collections.emptyMap(); + } finally { + close(result, stmt, conn); + } + } + + public static boolean execute(String sql, Properties jdbcConfig) { + return execute(getConnection(jdbcConfig), sql); + } + + public static boolean execute(Connection conn, String sql) { + Statement stmt = null; + try { + stmt = conn.createStatement(); + return stmt.execute(sql); + } catch (Exception ex) { + ex.printStackTrace(); Review Comment: ## SonarCloud / Debugging features should not be enabled in production <!--SONAR_ISSUE_KEY:AZ8yvnMilWsfFY_QVhqB-->Make sure this debug feature is deactivated before delivering the code in production. <p>See more on <a href="https://sonarcloud.io/project/issues?id=apache_incubator-streampark&issues=AZ8yvnMilWsfFY_QVhqB&open=AZ8yvnMilWsfFY_QVhqB&pullRequest=4419";>SonarQube Cloud</a></p> [Show more details](https://github.com/apache/streampark/security/code-scanning/140) ########## streampark-common/src/main/java/org/apache/streampark/common/util/StreamParkLoggerFactory.java: ########## @@ -0,0 +1,124 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.common.util; + +import org.apache.streampark.shaded.ch.qos.logback.classic.LoggerContext; +import org.apache.streampark.shaded.ch.qos.logback.classic.joran.JoranConfigurator; +import org.apache.streampark.shaded.ch.qos.logback.classic.util.ContextInitializer; +import org.apache.streampark.shaded.ch.qos.logback.classic.util.ContextSelectorStaticBinder; +import org.apache.streampark.shaded.ch.qos.logback.core.CoreConstants; +import org.apache.streampark.shaded.ch.qos.logback.core.LogbackException; +import org.apache.streampark.shaded.ch.qos.logback.core.status.StatusUtil; +import org.apache.streampark.shaded.ch.qos.logback.core.util.StatusPrinter; +import org.apache.streampark.shaded.org.slf4j.ILoggerFactory; +import org.apache.streampark.shaded.org.slf4j.spi.LoggerFactoryBinder; + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.net.URL; +import java.nio.charset.StandardCharsets; + +/** Shaded logback SLF4J logger factory binder. */ +public final class StreamParkLoggerFactory implements LoggerFactoryBinder { + + private static final StreamParkLoggerFactory INSTANCE = new StreamParkLoggerFactory(); + + private static final ContextSelectorStaticBinder CONTEXT_SELECTOR_BINDER; + + static { + LoggerContext defaultLoggerContext = new LoggerContext(); + try { + new ShadedContextInitializer(defaultLoggerContext).autoConfig(); + } catch (Exception e) { + System.err.println("Failed to auto configure default logger context"); + System.err.println("Reported exception:"); + e.printStackTrace(); Review Comment: ## SonarCloud / Debugging features should not be enabled in production <!--SONAR_ISSUE_KEY:AZ8yvnJLlWsfFY_QVho7-->Make sure this debug feature is deactivated before delivering the code in production. <p>See more on <a href="https://sonarcloud.io/project/issues?id=apache_incubator-streampark&issues=AZ8yvnJLlWsfFY_QVho7&open=AZ8yvnJLlWsfFY_QVho7&pullRequest=4419";>SonarQube Cloud</a></p> [Show more details](https://github.com/apache/streampark/security/code-scanning/126) ########## streampark-common/src/main/java/org/apache/streampark/common/util/JdbcUtils.java: ########## @@ -0,0 +1,329 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.common.util; + +import org.apache.streampark.common.conf.ConfigKeys; + +import com.zaxxer.hikari.HikariConfig; +import com.zaxxer.hikari.HikariDataSource; + +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.locks.ReentrantLock; +import java.util.function.Consumer; + +/** + * Based on the hikari connection pool implementation. Support multiple data sources, note that all + * modifications and additions are automatically committed transactions. + */ +public final class JdbcUtils { + + private static final ConcurrentHashMap<String, ReentrantLock> LOCK_MAP = new ConcurrentHashMap<>(); + + private static final ConcurrentHashMap<String, HikariDataSource> DATA_SOURCE_HOLDER = + new ConcurrentHashMap<>(); + + private JdbcUtils() {} + + public static List<Map<String, Object>> select(String sql, Properties jdbcConfig) { + return select(sql, null, jdbcConfig); + } + + public static List<Map<String, Object>> select( + String sql, Consumer<ResultSet> func, Properties jdbcConfig) { + if (sql == null || sql.isEmpty()) { + return Collections.emptyList(); + } + Connection conn = getConnection(jdbcConfig); + Statement stmt = null; + ResultSet result = null; + try { + stmt = createStatement(conn); + result = stmt.executeQuery(sql); + if (func != null) { + func.accept(result); + } + int count = result.getMetaData().getColumnCount(); + List<Map<String, Object>> array = new ArrayList<>(); + while (result.next()) { + Map<String, Object> map = new HashMap<>(); + for (int x = 1; x <= count; x++) { + String key = result.getMetaData().getColumnLabel(x); + Object value = result.getObject(x); + map.put(key, value); + } + array.add(map); + } + return array.isEmpty() ? Collections.emptyList() : array; + } catch (Exception ex) { + ex.printStackTrace(); + return Collections.emptyList(); + } finally { + close(result, stmt, conn); + } + } + + public static long count(String sql, Properties jdbcConfig) { + Map<String, Object> row = unique(sql, jdbcConfig); + if (row.isEmpty()) { + return 0L; + } + return Long.parseLong(row.values().iterator().next().toString()); + } + + public static long count(Connection conn, String sql) { + Map<String, Object> row = unique(conn, sql); + if (row.isEmpty()) { + return 0L; + } + return Long.parseLong(row.values().iterator().next().toString()); + } + + public static int batch(Iterable<String> sql, Properties jdbcConfig) { + int size = 0; + for (String ignored : sql) { + size++; + } + if (size == 0) { + return 0; + } + if (size == 1) { + for (String s : sql) { + return update(s, jdbcConfig); + } + } + Connection conn = getConnection(jdbcConfig); + try { + Statement prepStat = conn.createStatement(); + try { + int index = 0; + int batchSize = 1000; + int total = 0; + for (String x : sql) { + prepStat.addBatch(x); + index++; + if (index > 0 && index % batchSize == 0) { + int count = 0; + for (int c : prepStat.executeBatch()) { + count += c; + } + conn.commit(); + prepStat.clearBatch(); + total += count; + } + } + for (int c : prepStat.executeBatch()) { + total += c; + } + return total; + } catch (Exception ex) { + ex.printStackTrace(); + return 0; + } finally { + conn.commit(); + close(conn); + } + } catch (Exception ex) { + ex.printStackTrace(); Review Comment: ## SonarCloud / Debugging features should not be enabled in production <!--SONAR_ISSUE_KEY:AZ8yvnMilWsfFY_QVhp--->Make sure this debug feature is deactivated before delivering the code in production. <p>See more on <a href="https://sonarcloud.io/project/issues?id=apache_incubator-streampark&issues=AZ8yvnMilWsfFY_QVhp-&open=AZ8yvnMilWsfFY_QVhp-&pullRequest=4419";>SonarQube Cloud</a></p> [Show more details](https://github.com/apache/streampark/security/code-scanning/136) ########## streampark-common/src/main/java/org/apache/streampark/common/util/JdbcUtils.java: ########## @@ -0,0 +1,329 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.common.util; + +import org.apache.streampark.common.conf.ConfigKeys; + +import com.zaxxer.hikari.HikariConfig; +import com.zaxxer.hikari.HikariDataSource; + +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.locks.ReentrantLock; +import java.util.function.Consumer; + +/** + * Based on the hikari connection pool implementation. Support multiple data sources, note that all + * modifications and additions are automatically committed transactions. + */ +public final class JdbcUtils { + + private static final ConcurrentHashMap<String, ReentrantLock> LOCK_MAP = new ConcurrentHashMap<>(); + + private static final ConcurrentHashMap<String, HikariDataSource> DATA_SOURCE_HOLDER = + new ConcurrentHashMap<>(); + + private JdbcUtils() {} + + public static List<Map<String, Object>> select(String sql, Properties jdbcConfig) { + return select(sql, null, jdbcConfig); + } + + public static List<Map<String, Object>> select( + String sql, Consumer<ResultSet> func, Properties jdbcConfig) { + if (sql == null || sql.isEmpty()) { + return Collections.emptyList(); + } + Connection conn = getConnection(jdbcConfig); + Statement stmt = null; + ResultSet result = null; + try { + stmt = createStatement(conn); + result = stmt.executeQuery(sql); + if (func != null) { + func.accept(result); + } + int count = result.getMetaData().getColumnCount(); + List<Map<String, Object>> array = new ArrayList<>(); + while (result.next()) { + Map<String, Object> map = new HashMap<>(); + for (int x = 1; x <= count; x++) { + String key = result.getMetaData().getColumnLabel(x); + Object value = result.getObject(x); + map.put(key, value); + } + array.add(map); + } + return array.isEmpty() ? Collections.emptyList() : array; + } catch (Exception ex) { + ex.printStackTrace(); + return Collections.emptyList(); + } finally { + close(result, stmt, conn); + } + } + + public static long count(String sql, Properties jdbcConfig) { + Map<String, Object> row = unique(sql, jdbcConfig); + if (row.isEmpty()) { + return 0L; + } + return Long.parseLong(row.values().iterator().next().toString()); + } + + public static long count(Connection conn, String sql) { + Map<String, Object> row = unique(conn, sql); + if (row.isEmpty()) { + return 0L; + } + return Long.parseLong(row.values().iterator().next().toString()); + } + + public static int batch(Iterable<String> sql, Properties jdbcConfig) { + int size = 0; + for (String ignored : sql) { + size++; + } + if (size == 0) { + return 0; + } + if (size == 1) { + for (String s : sql) { + return update(s, jdbcConfig); + } + } + Connection conn = getConnection(jdbcConfig); + try { + Statement prepStat = conn.createStatement(); + try { + int index = 0; + int batchSize = 1000; + int total = 0; + for (String x : sql) { + prepStat.addBatch(x); + index++; + if (index > 0 && index % batchSize == 0) { + int count = 0; + for (int c : prepStat.executeBatch()) { + count += c; + } + conn.commit(); + prepStat.clearBatch(); + total += count; + } + } + for (int c : prepStat.executeBatch()) { + total += c; + } + return total; + } catch (Exception ex) { + ex.printStackTrace(); Review Comment: ## SonarCloud / Debugging features should not be enabled in production <!--SONAR_ISSUE_KEY:AZ8yvnMilWsfFY_QVhp9-->Make sure this debug feature is deactivated before delivering the code in production. <p>See more on <a href="https://sonarcloud.io/project/issues?id=apache_incubator-streampark&issues=AZ8yvnMilWsfFY_QVhp9&open=AZ8yvnMilWsfFY_QVhp9&pullRequest=4419";>SonarQube Cloud</a></p> [Show more details](https://github.com/apache/streampark/security/code-scanning/138) ########## streampark-common/src/main/java/org/apache/streampark/common/util/JdbcUtils.java: ########## @@ -0,0 +1,329 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.common.util; + +import org.apache.streampark.common.conf.ConfigKeys; + +import com.zaxxer.hikari.HikariConfig; +import com.zaxxer.hikari.HikariDataSource; + +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.locks.ReentrantLock; +import java.util.function.Consumer; + +/** + * Based on the hikari connection pool implementation. Support multiple data sources, note that all + * modifications and additions are automatically committed transactions. + */ +public final class JdbcUtils { + + private static final ConcurrentHashMap<String, ReentrantLock> LOCK_MAP = new ConcurrentHashMap<>(); + + private static final ConcurrentHashMap<String, HikariDataSource> DATA_SOURCE_HOLDER = + new ConcurrentHashMap<>(); + + private JdbcUtils() {} + + public static List<Map<String, Object>> select(String sql, Properties jdbcConfig) { + return select(sql, null, jdbcConfig); + } + + public static List<Map<String, Object>> select( + String sql, Consumer<ResultSet> func, Properties jdbcConfig) { + if (sql == null || sql.isEmpty()) { + return Collections.emptyList(); + } + Connection conn = getConnection(jdbcConfig); + Statement stmt = null; + ResultSet result = null; + try { + stmt = createStatement(conn); + result = stmt.executeQuery(sql); + if (func != null) { + func.accept(result); + } + int count = result.getMetaData().getColumnCount(); + List<Map<String, Object>> array = new ArrayList<>(); + while (result.next()) { + Map<String, Object> map = new HashMap<>(); + for (int x = 1; x <= count; x++) { + String key = result.getMetaData().getColumnLabel(x); + Object value = result.getObject(x); + map.put(key, value); + } + array.add(map); + } + return array.isEmpty() ? Collections.emptyList() : array; + } catch (Exception ex) { + ex.printStackTrace(); Review Comment: ## SonarCloud / Debugging features should not be enabled in production <!--SONAR_ISSUE_KEY:AZ8yvnMilWsfFY_QVhp7-->Make sure this debug feature is deactivated before delivering the code in production. <p>See more on <a href="https://sonarcloud.io/project/issues?id=apache_incubator-streampark&issues=AZ8yvnMilWsfFY_QVhp7&open=AZ8yvnMilWsfFY_QVhp7&pullRequest=4419";>SonarQube Cloud</a></p> [Show more details](https://github.com/apache/streampark/security/code-scanning/137) ########## streampark-flink/streampark-flink-shims/streampark-flink-shims-base-v2/src/main/java/org/apache/streampark/flink/core/conf/ParameterCli.java: ########## @@ -0,0 +1,206 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.flink.core.conf; + +import org.apache.streampark.common.conf.ConfigKeys; +import org.apache.streampark.common.util.PropertiesUtils; + +import org.apache.commons.cli.DefaultParser; +import org.apache.commons.cli.Options; + +import java.net.URLClassLoader; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** Parses Flink application CLI parameters from configuration files. */ +public final class ParameterCli { + + private static final String PROPERTY_PREFIX = ConfigKeys.KEY_FLINK_PROPERTY_PREFIX(); + private static final String OPTION_PREFIX = ConfigKeys.KEY_FLINK_OPTION_PREFIX(); + private static final String OPTION_MAIN = PROPERTY_PREFIX + "$internal.application.main"; + + private static final Options FLINK_OPTIONS = FlinkRunOption.allOptions(); + private static final DefaultParser PARSER = new DefaultParser(); + + private ParameterCli() { + } + + public static void main(String[] args) { + System.out.print(read(args)); + } + + public static String read(String[] args) { + switch (args[0]) { + case "--vmopt": + ClassLoader classLoader = ClassLoader.getSystemClassLoader(); + if (classLoader instanceof URLClassLoader) { + return ""; + } + return "--add-opens java.base/jdk.internal.loader=ALL-UNNAMED " + + "--add-opens jdk.zipfs/jdk.nio.zipfs=ALL-UNNAMED"; + default: + String action = args[0]; + String conf = args[1]; + Map<String, String> map; + try { + String extension = conf.substring(conf.lastIndexOf('.') + 1).toLowerCase(); + switch (extension) { + case "yml": + case "yaml": + map = PropertiesUtils.fromYamlFile(conf); + break; + case "conf": + map = PropertiesUtils.fromHoconFile(conf); + break; + case "properties": + map = PropertiesUtils.fromPropertiesFile(conf); + break; + default: + throw new IllegalArgumentException( + "[StreamPark] Usage:flink.conf file error,must be (yml|conf|properties)"); + } + } catch (Exception e) { + map = Collections.emptyMap(); + } + String[] programArgs = new String[args.length - 2]; + System.arraycopy(args, 2, programArgs, 0, programArgs.length); + switch (action) { + case "--option": + String[] option = getOption(map, programArgs); + StringBuilder buffer = new StringBuilder(); + try { + org.apache.commons.cli.CommandLine line = + PARSER.parse(FLINK_OPTIONS, option, false); + for (org.apache.commons.cli.Option x : line.getOptions()) { + buffer.append(" -").append(x.getOpt()); + if (x.hasArg()) { + buffer.append(" ").append(x.getValue()); + } + } + } catch (Exception exception) { + exception.printStackTrace(); + } + String mainClass = map.get(OPTION_MAIN); + if (mainClass != null) { + buffer.append(" -c ").append(mainClass); + } + return buffer.toString().trim(); + case "--property": + StringBuilder propertyBuffer = new StringBuilder(); + for (Map.Entry<String, String> entry : map.entrySet()) { + String key = entry.getKey(); + String value = entry.getValue(); + if (!OPTION_MAIN.equals(key) + && key.startsWith(PROPERTY_PREFIX) + && value != null + && !value.isEmpty()) { + String propertyKey = key.substring(PROPERTY_PREFIX.length()).trim(); + String propertyValue = value.trim(); + if (ConfigKeys.KEY_FLINK_APP_NAME().equals(propertyKey)) { + propertyBuffer + .append(" -D") + .append(propertyKey) + .append("=") + .append(propertyValue.replace(" ", "_")); + } else { + propertyBuffer + .append(" -D") + .append(propertyKey) + .append("=") + .append(propertyValue); + } + } + } + return propertyBuffer.toString().trim(); + case "--name": + String appName = + map.getOrDefault( + PROPERTY_PREFIX.concat(ConfigKeys.KEY_FLINK_APP_NAME()), ""); + appName = appName.trim(); + return appName.isEmpty() ? "" : appName; + case "--detached": + String[] detachedOption = getOption(map, programArgs); + try { + org.apache.commons.cli.CommandLine line = + PARSER.parse(FlinkRunOption.allOptions(), detachedOption, false); + boolean detached = + line.hasOption(FlinkRunOption.DETACHED_OPTION.getOpt()) + || line.hasOption( + FlinkRunOption.DETACHED_OPTION.getLongOpt()); + return detached ? "Detached" : "Attach"; + } catch (Exception e) { + e.printStackTrace(); + return "Attach"; + } + default: + return null; + } + } + } + + public static String[] getOption(Map<String, String> map, String[] args) { + Map<String, Object> optionMap = new HashMap<>(); + for (Map.Entry<String, String> entry : map.entrySet()) { + String key = entry.getKey(); + String value = entry.getValue(); + if (key.startsWith(OPTION_PREFIX) && value != null && !value.isEmpty()) { + String optionKey = key.substring(OPTION_PREFIX.length()); + if (FLINK_OPTIONS.hasOption(optionKey)) { + Object parsedValue; + if ("true".equalsIgnoreCase(value) || "false".equalsIgnoreCase(value)) { + parsedValue = Boolean.parseBoolean(value); + } else { + parsedValue = value; + } + if (parsedValue instanceof Boolean) { + if ((Boolean) parsedValue) { + optionMap.put("-" + optionKey.trim(), true); + } + } else { + optionMap.put("-" + optionKey.trim(), parsedValue); + } + } + } + } + if (args.length > 0) { + try { + org.apache.commons.cli.CommandLine line = PARSER.parse(FLINK_OPTIONS, args, false); + for (org.apache.commons.cli.Option x : line.getOptions()) { + if (x.hasArg()) { + optionMap.put("-" + x.getLongOpt().trim(), x.getValue()); + } else { + optionMap.put("-" + x.getLongOpt().trim(), true); + } + } + } catch (Exception e) { + e.printStackTrace(); Review Comment: ## SonarCloud / Debugging features should not be enabled in production <!--SONAR_ISSUE_KEY:AZ8yvm9olWsfFY_QVhic-->Make sure this debug feature is deactivated before delivering the code in production. <p>See more on <a href="https://sonarcloud.io/project/issues?id=apache_incubator-streampark&issues=AZ8yvm9olWsfFY_QVhic&open=AZ8yvm9olWsfFY_QVhic&pullRequest=4419";>SonarQube Cloud</a></p> [Show more details](https://github.com/apache/streampark/security/code-scanning/124) ########## streampark-common/src/main/java/org/apache/streampark/common/util/ZooKeeperUtils.java: ########## @@ -0,0 +1,161 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.common.util; + +import org.apache.curator.RetryPolicy; +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.CuratorFrameworkFactory; +import org.apache.curator.retry.RetryNTimes; +import org.apache.zookeeper.CreateMode; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +public final class ZooKeeperUtils { + + private static final String CONNECT = "localhost:2181"; + private static final Map<String, CuratorFramework> CLIENT_MAP = new ConcurrentHashMap<>(); + + private ZooKeeperUtils() {} + + public static CuratorFramework getClient() { + return getClient(CONNECT); + } + + public static CuratorFramework getClient(String url) { + CuratorFramework existing = CLIENT_MAP.get(url); + if (existing != null) { + return existing; + } + try { + RetryPolicy retryPolicy = new RetryNTimes(5, 2000); + CuratorFramework client = + CuratorFrameworkFactory.builder() + .connectString(url) + .retryPolicy(retryPolicy) + .connectionTimeoutMs(2000) + .build(); + client.start(); + CLIENT_MAP.put(url, client); + return client; + } catch (Exception e) { + throw new IllegalStateException(e.getMessage(), e); + } + } + + public static void close(String url) { + CuratorFramework client = getClient(url); + if (client != null) { + client.close(); + CLIENT_MAP.remove(url); + } + } + + public static List<String> listChildren(String path) throws Exception { + return listChildren(path, CONNECT); + } + + public static List<String> listChildren(String path, String url) throws Exception { + CuratorFramework client = getClient(url); + if (client.checkExists().forPath(path) == null) { + return new ArrayList<>(); + } + return client.getChildren().forPath(path); + } + + public static boolean create(String path) throws Exception { + return create(path, null, CONNECT, false); + } + + public static boolean create(String path, String value, String url, boolean persistent) + throws Exception { + try { + CuratorFramework client = getClient(url); + if (client.checkExists().forPath(path) == null) { + byte[] data = + value == null || value.isEmpty() + ? new byte[0] + : value.getBytes(StandardCharsets.UTF_8); + CreateMode mode = persistent ? CreateMode.PERSISTENT : CreateMode.EPHEMERAL; + String opResult = + client.create().creatingParentsIfNeeded().withMode(mode).forPath(path, data); + return path.equals(opResult); + } + return false; + } catch (Exception e) { + e.printStackTrace(); Review Comment: ## SonarCloud / Debugging features should not be enabled in production <!--SONAR_ISSUE_KEY:AZ8yvnLwlWsfFY_QVhpt-->Make sure this debug feature is deactivated before delivering the code in production. <p>See more on <a href="https://sonarcloud.io/project/issues?id=apache_incubator-streampark&issues=AZ8yvnLwlWsfFY_QVhpt&open=AZ8yvnLwlWsfFY_QVhpt&pullRequest=4419";>SonarQube Cloud</a></p> [Show more details](https://github.com/apache/streampark/security/code-scanning/131) ########## streampark-common/src/main/java/org/apache/streampark/common/util/ZooKeeperUtils.java: ########## @@ -0,0 +1,161 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.common.util; + +import org.apache.curator.RetryPolicy; +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.CuratorFrameworkFactory; +import org.apache.curator.retry.RetryNTimes; +import org.apache.zookeeper.CreateMode; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +public final class ZooKeeperUtils { + + private static final String CONNECT = "localhost:2181"; + private static final Map<String, CuratorFramework> CLIENT_MAP = new ConcurrentHashMap<>(); + + private ZooKeeperUtils() {} + + public static CuratorFramework getClient() { + return getClient(CONNECT); + } + + public static CuratorFramework getClient(String url) { + CuratorFramework existing = CLIENT_MAP.get(url); + if (existing != null) { + return existing; + } + try { + RetryPolicy retryPolicy = new RetryNTimes(5, 2000); + CuratorFramework client = + CuratorFrameworkFactory.builder() + .connectString(url) + .retryPolicy(retryPolicy) + .connectionTimeoutMs(2000) + .build(); + client.start(); + CLIENT_MAP.put(url, client); + return client; + } catch (Exception e) { + throw new IllegalStateException(e.getMessage(), e); + } + } + + public static void close(String url) { + CuratorFramework client = getClient(url); + if (client != null) { + client.close(); + CLIENT_MAP.remove(url); + } + } + + public static List<String> listChildren(String path) throws Exception { + return listChildren(path, CONNECT); + } + + public static List<String> listChildren(String path, String url) throws Exception { + CuratorFramework client = getClient(url); + if (client.checkExists().forPath(path) == null) { + return new ArrayList<>(); + } + return client.getChildren().forPath(path); + } + + public static boolean create(String path) throws Exception { + return create(path, null, CONNECT, false); + } + + public static boolean create(String path, String value, String url, boolean persistent) + throws Exception { + try { + CuratorFramework client = getClient(url); + if (client.checkExists().forPath(path) == null) { + byte[] data = + value == null || value.isEmpty() + ? new byte[0] + : value.getBytes(StandardCharsets.UTF_8); + CreateMode mode = persistent ? CreateMode.PERSISTENT : CreateMode.EPHEMERAL; + String opResult = + client.create().creatingParentsIfNeeded().withMode(mode).forPath(path, data); + return path.equals(opResult); + } + return false; + } catch (Exception e) { + e.printStackTrace(); + return false; + } + } + + public static boolean update(String path, String value, String url, boolean persistent) + throws Exception { + try { + CuratorFramework client = getClient(url); + if (client.checkExists().forPath(path) == null) { + CreateMode mode = persistent ? CreateMode.PERSISTENT : CreateMode.EPHEMERAL; + String opResult = + client.create() + .creatingParentsIfNeeded() + .withMode(mode) + .forPath(path, value.getBytes(StandardCharsets.UTF_8)); + return path.equals(opResult); + } + client.setData().forPath(path, value.getBytes(StandardCharsets.UTF_8)); + return true; + } catch (Exception e) { + e.printStackTrace(); Review Comment: ## SonarCloud / Debugging features should not be enabled in production <!--SONAR_ISSUE_KEY:AZ8yvnLwlWsfFY_QVhpu-->Make sure this debug feature is deactivated before delivering the code in production. <p>See more on <a href="https://sonarcloud.io/project/issues?id=apache_incubator-streampark&issues=AZ8yvnLwlWsfFY_QVhpu&open=AZ8yvnLwlWsfFY_QVhpu&pullRequest=4419";>SonarQube Cloud</a></p> [Show more details](https://github.com/apache/streampark/security/code-scanning/132) ########## streampark-common/src/main/java/org/apache/streampark/common/util/YarnUtils.java: ########## @@ -0,0 +1,299 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.common.util; + +import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeys; +import org.apache.hadoop.net.NetUtils; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ApplicationReport; +import org.apache.hadoop.yarn.api.records.YarnApplicationState; +import org.apache.hadoop.yarn.conf.HAUtil; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.util.RMHAUtils; +import org.apache.hc.client5.http.config.RequestConfig; +import org.apache.hc.core5.util.Timeout; +import org.apache.streampark.common.conf.CommonConfig; +import org.apache.streampark.common.conf.InternalConfigHolder; +import org.apache.streampark.common.constants.Constants; +import org.apache.streampark.shaded.org.slf4j.Logger; + +import java.io.IOException; +import java.net.InetAddress; +import java.security.PrivilegedExceptionAction; +import java.time.Duration; +import java.util.ArrayList; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.TimeUnit; + +/** YARN cluster utilities. */ +public final class YarnUtils { + + private static final Logger LOG = + StreamParkLoggerFactory.loggerFactory().getLogger(YarnUtils.class.getName()); + + private static String rmHttpURL; + + public static final String PROXY_YARN_URL = + InternalConfigHolder.get(CommonConfig.STREAMPARK_PROXY_YARN_URL()); + + public static final boolean HAS_YARN_HTTP_KERBEROS_AUTH = + "kerberos" + .equalsIgnoreCase(InternalConfigHolder.get(CommonConfig.STREAMPARK_YARN_AUTH())); + + public static final boolean HAS_YARN_HTTP_SIMPLE_AUTH = + "simple".equalsIgnoreCase(InternalConfigHolder.get(CommonConfig.STREAMPARK_YARN_AUTH())); + + private YarnUtils() {} + + public static boolean hasYarnHttpKerberosAuth() { + return HAS_YARN_HTTP_KERBEROS_AUTH; + } + + public static boolean hasYarnHttpSimpleAuth() { + return HAS_YARN_HTTP_SIMPLE_AUTH; + } + + public static List<ApplicationId> getAppId(String appName) { + EnumSet<YarnApplicationState> appStates = + EnumSet.of( + YarnApplicationState.RUNNING, + YarnApplicationState.ACCEPTED, + YarnApplicationState.SUBMITTED); + try { + List<ApplicationId> appIds = new ArrayList<>(); + for (ApplicationReport report : HadoopUtils.yarnClient().getApplications(appStates)) { + if (appName.equals(report.getName())) { + appIds.add(report.getApplicationId()); + } + } + return appIds; + } catch (Exception e) { + e.printStackTrace(); + return new ArrayList<>(); + } + } + + public static YarnApplicationState getState(String appId) { + ApplicationId applicationId = ApplicationId.fromString(appId); + try { + ApplicationReport applicationReport = + HadoopUtils.yarnClient().getApplicationReport(applicationId); + return applicationReport.getYarnApplicationState(); + } catch (Exception e) { + e.printStackTrace(); Review Comment: ## SonarCloud / Debugging features should not be enabled in production <!--SONAR_ISSUE_KEY:AZ8yvnK0lWsfFY_QVhpW-->Make sure this debug feature is deactivated before delivering the code in production. <p>See more on <a href="https://sonarcloud.io/project/issues?id=apache_incubator-streampark&issues=AZ8yvnK0lWsfFY_QVhpW&open=AZ8yvnK0lWsfFY_QVhpW&pullRequest=4419";>SonarQube Cloud</a></p> [Show more details](https://github.com/apache/streampark/security/code-scanning/128) ########## streampark-common/src/main/java/org/apache/streampark/common/util/ZooKeeperUtils.java: ########## @@ -0,0 +1,161 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.common.util; + +import org.apache.curator.RetryPolicy; +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.CuratorFrameworkFactory; +import org.apache.curator.retry.RetryNTimes; +import org.apache.zookeeper.CreateMode; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +public final class ZooKeeperUtils { + + private static final String CONNECT = "localhost:2181"; + private static final Map<String, CuratorFramework> CLIENT_MAP = new ConcurrentHashMap<>(); + + private ZooKeeperUtils() {} + + public static CuratorFramework getClient() { + return getClient(CONNECT); + } + + public static CuratorFramework getClient(String url) { + CuratorFramework existing = CLIENT_MAP.get(url); + if (existing != null) { + return existing; + } + try { + RetryPolicy retryPolicy = new RetryNTimes(5, 2000); + CuratorFramework client = + CuratorFrameworkFactory.builder() + .connectString(url) + .retryPolicy(retryPolicy) + .connectionTimeoutMs(2000) + .build(); + client.start(); + CLIENT_MAP.put(url, client); + return client; + } catch (Exception e) { + throw new IllegalStateException(e.getMessage(), e); + } + } + + public static void close(String url) { + CuratorFramework client = getClient(url); + if (client != null) { + client.close(); + CLIENT_MAP.remove(url); + } + } + + public static List<String> listChildren(String path) throws Exception { + return listChildren(path, CONNECT); + } + + public static List<String> listChildren(String path, String url) throws Exception { + CuratorFramework client = getClient(url); + if (client.checkExists().forPath(path) == null) { + return new ArrayList<>(); + } + return client.getChildren().forPath(path); + } + + public static boolean create(String path) throws Exception { + return create(path, null, CONNECT, false); + } + + public static boolean create(String path, String value, String url, boolean persistent) + throws Exception { + try { + CuratorFramework client = getClient(url); + if (client.checkExists().forPath(path) == null) { + byte[] data = + value == null || value.isEmpty() + ? new byte[0] + : value.getBytes(StandardCharsets.UTF_8); + CreateMode mode = persistent ? CreateMode.PERSISTENT : CreateMode.EPHEMERAL; + String opResult = + client.create().creatingParentsIfNeeded().withMode(mode).forPath(path, data); + return path.equals(opResult); + } + return false; + } catch (Exception e) { + e.printStackTrace(); + return false; + } + } + + public static boolean update(String path, String value, String url, boolean persistent) + throws Exception { + try { + CuratorFramework client = getClient(url); + if (client.checkExists().forPath(path) == null) { + CreateMode mode = persistent ? CreateMode.PERSISTENT : CreateMode.EPHEMERAL; + String opResult = + client.create() + .creatingParentsIfNeeded() + .withMode(mode) + .forPath(path, value.getBytes(StandardCharsets.UTF_8)); + return path.equals(opResult); + } + client.setData().forPath(path, value.getBytes(StandardCharsets.UTF_8)); + return true; + } catch (Exception e) { + e.printStackTrace(); + return false; + } + } + + public static void delete(String path) throws Exception { + delete(path, CONNECT); + } + + public static void delete(String path, String url) throws Exception { + try { + CuratorFramework client = getClient(url); + if (client.checkExists().forPath(path) != null) { + client.delete().deletingChildrenIfNeeded().forPath(path); + } + } catch (Exception e) { + e.printStackTrace(); Review Comment: ## SonarCloud / Debugging features should not be enabled in production <!--SONAR_ISSUE_KEY:AZ8yvnLwlWsfFY_QVhpv-->Make sure this debug feature is deactivated before delivering the code in production. <p>See more on <a href="https://sonarcloud.io/project/issues?id=apache_incubator-streampark&issues=AZ8yvnLwlWsfFY_QVhpv&open=AZ8yvnLwlWsfFY_QVhpv&pullRequest=4419";>SonarQube Cloud</a></p> [Show more details](https://github.com/apache/streampark/security/code-scanning/133) ########## streampark-common/src/main/java/org/apache/streampark/common/util/JdbcUtils.java: ########## @@ -0,0 +1,329 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.common.util; + +import org.apache.streampark.common.conf.ConfigKeys; + +import com.zaxxer.hikari.HikariConfig; +import com.zaxxer.hikari.HikariDataSource; + +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.locks.ReentrantLock; +import java.util.function.Consumer; + +/** + * Based on the hikari connection pool implementation. Support multiple data sources, note that all + * modifications and additions are automatically committed transactions. + */ +public final class JdbcUtils { + + private static final ConcurrentHashMap<String, ReentrantLock> LOCK_MAP = new ConcurrentHashMap<>(); + + private static final ConcurrentHashMap<String, HikariDataSource> DATA_SOURCE_HOLDER = + new ConcurrentHashMap<>(); + + private JdbcUtils() {} + + public static List<Map<String, Object>> select(String sql, Properties jdbcConfig) { + return select(sql, null, jdbcConfig); + } + + public static List<Map<String, Object>> select( + String sql, Consumer<ResultSet> func, Properties jdbcConfig) { + if (sql == null || sql.isEmpty()) { + return Collections.emptyList(); + } + Connection conn = getConnection(jdbcConfig); + Statement stmt = null; + ResultSet result = null; + try { + stmt = createStatement(conn); + result = stmt.executeQuery(sql); + if (func != null) { + func.accept(result); + } + int count = result.getMetaData().getColumnCount(); + List<Map<String, Object>> array = new ArrayList<>(); + while (result.next()) { + Map<String, Object> map = new HashMap<>(); + for (int x = 1; x <= count; x++) { + String key = result.getMetaData().getColumnLabel(x); + Object value = result.getObject(x); + map.put(key, value); + } + array.add(map); + } + return array.isEmpty() ? Collections.emptyList() : array; + } catch (Exception ex) { + ex.printStackTrace(); + return Collections.emptyList(); + } finally { + close(result, stmt, conn); + } + } + + public static long count(String sql, Properties jdbcConfig) { + Map<String, Object> row = unique(sql, jdbcConfig); + if (row.isEmpty()) { + return 0L; + } + return Long.parseLong(row.values().iterator().next().toString()); + } + + public static long count(Connection conn, String sql) { + Map<String, Object> row = unique(conn, sql); + if (row.isEmpty()) { + return 0L; + } + return Long.parseLong(row.values().iterator().next().toString()); + } + + public static int batch(Iterable<String> sql, Properties jdbcConfig) { + int size = 0; + for (String ignored : sql) { + size++; + } + if (size == 0) { + return 0; + } + if (size == 1) { + for (String s : sql) { + return update(s, jdbcConfig); + } + } + Connection conn = getConnection(jdbcConfig); + try { + Statement prepStat = conn.createStatement(); + try { + int index = 0; + int batchSize = 1000; + int total = 0; + for (String x : sql) { + prepStat.addBatch(x); + index++; + if (index > 0 && index % batchSize == 0) { + int count = 0; + for (int c : prepStat.executeBatch()) { + count += c; + } + conn.commit(); + prepStat.clearBatch(); + total += count; + } + } + for (int c : prepStat.executeBatch()) { + total += c; + } + return total; + } catch (Exception ex) { + ex.printStackTrace(); + return 0; + } finally { + conn.commit(); + close(conn); + } + } catch (Exception ex) { + ex.printStackTrace(); + return 0; + } + } + + public static int update(String sql, Properties jdbcConfig) { + return update(getConnection(jdbcConfig), sql); + } + + public static int update(Connection conn, String sql) { + Statement statement = null; + try { + statement = conn.createStatement(); + return statement.executeUpdate(sql); + } catch (Exception ex) { + ex.printStackTrace(); + return -1; + } finally { + close(statement, conn); + } + } + + public static Map<String, Object> unique(String sql, Properties jdbcConfig) { + return unique(getConnection(jdbcConfig), sql); + } + + public static Map<String, Object> unique(Connection conn, String sql) { + Statement stmt = null; + ResultSet result = null; + try { + stmt = createStatement(conn); + result = stmt.executeQuery(sql); + int count = result.getMetaData().getColumnCount(); + if (!result.next()) { + return Collections.emptyMap(); + } + Map<String, Object> map = new HashMap<>(); + for (int x = 1; x <= count; x++) { + String key = result.getMetaData().getColumnLabel(x); + Object value = result.getObject(x); + map.put(key, value); + } + return map; + } catch (Exception ex) { + ex.printStackTrace(); Review Comment: ## SonarCloud / Debugging features should not be enabled in production <!--SONAR_ISSUE_KEY:AZ8yvnMilWsfFY_QVhqA-->Make sure this debug feature is deactivated before delivering the code in production. <p>See more on <a href="https://sonarcloud.io/project/issues?id=apache_incubator-streampark&issues=AZ8yvnMilWsfFY_QVhqA&open=AZ8yvnMilWsfFY_QVhqA&pullRequest=4419";>SonarQube Cloud</a></p> [Show more details](https://github.com/apache/streampark/security/code-scanning/139) ########## streampark-common/src/main/java/org/apache/streampark/common/util/JdbcUtils.java: ########## @@ -0,0 +1,329 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.common.util; + +import org.apache.streampark.common.conf.ConfigKeys; + +import com.zaxxer.hikari.HikariConfig; +import com.zaxxer.hikari.HikariDataSource; + +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.locks.ReentrantLock; +import java.util.function.Consumer; + +/** + * Based on the hikari connection pool implementation. Support multiple data sources, note that all + * modifications and additions are automatically committed transactions. + */ +public final class JdbcUtils { + + private static final ConcurrentHashMap<String, ReentrantLock> LOCK_MAP = new ConcurrentHashMap<>(); + + private static final ConcurrentHashMap<String, HikariDataSource> DATA_SOURCE_HOLDER = + new ConcurrentHashMap<>(); + + private JdbcUtils() {} + + public static List<Map<String, Object>> select(String sql, Properties jdbcConfig) { + return select(sql, null, jdbcConfig); + } + + public static List<Map<String, Object>> select( + String sql, Consumer<ResultSet> func, Properties jdbcConfig) { + if (sql == null || sql.isEmpty()) { + return Collections.emptyList(); + } + Connection conn = getConnection(jdbcConfig); + Statement stmt = null; + ResultSet result = null; + try { + stmt = createStatement(conn); + result = stmt.executeQuery(sql); + if (func != null) { + func.accept(result); + } + int count = result.getMetaData().getColumnCount(); + List<Map<String, Object>> array = new ArrayList<>(); + while (result.next()) { + Map<String, Object> map = new HashMap<>(); + for (int x = 1; x <= count; x++) { + String key = result.getMetaData().getColumnLabel(x); + Object value = result.getObject(x); + map.put(key, value); + } + array.add(map); + } + return array.isEmpty() ? Collections.emptyList() : array; + } catch (Exception ex) { + ex.printStackTrace(); + return Collections.emptyList(); + } finally { + close(result, stmt, conn); + } + } + + public static long count(String sql, Properties jdbcConfig) { + Map<String, Object> row = unique(sql, jdbcConfig); + if (row.isEmpty()) { + return 0L; + } + return Long.parseLong(row.values().iterator().next().toString()); + } + + public static long count(Connection conn, String sql) { + Map<String, Object> row = unique(conn, sql); + if (row.isEmpty()) { + return 0L; + } + return Long.parseLong(row.values().iterator().next().toString()); + } + + public static int batch(Iterable<String> sql, Properties jdbcConfig) { + int size = 0; + for (String ignored : sql) { + size++; + } + if (size == 0) { + return 0; + } + if (size == 1) { + for (String s : sql) { + return update(s, jdbcConfig); + } + } + Connection conn = getConnection(jdbcConfig); + try { + Statement prepStat = conn.createStatement(); + try { + int index = 0; + int batchSize = 1000; + int total = 0; + for (String x : sql) { + prepStat.addBatch(x); + index++; + if (index > 0 && index % batchSize == 0) { + int count = 0; + for (int c : prepStat.executeBatch()) { + count += c; + } + conn.commit(); + prepStat.clearBatch(); + total += count; + } + } + for (int c : prepStat.executeBatch()) { + total += c; + } + return total; + } catch (Exception ex) { + ex.printStackTrace(); + return 0; + } finally { + conn.commit(); + close(conn); + } + } catch (Exception ex) { + ex.printStackTrace(); + return 0; + } + } + + public static int update(String sql, Properties jdbcConfig) { + return update(getConnection(jdbcConfig), sql); + } + + public static int update(Connection conn, String sql) { + Statement statement = null; + try { + statement = conn.createStatement(); + return statement.executeUpdate(sql); + } catch (Exception ex) { + ex.printStackTrace(); Review Comment: ## SonarCloud / Debugging features should not be enabled in production <!--SONAR_ISSUE_KEY:AZ8yvnMilWsfFY_QVhp_-->Make sure this debug feature is deactivated before delivering the code in production. <p>See more on <a href="https://sonarcloud.io/project/issues?id=apache_incubator-streampark&issues=AZ8yvnMilWsfFY_QVhp_&open=AZ8yvnMilWsfFY_QVhp_&pullRequest=4419";>SonarQube Cloud</a></p> [Show more details](https://github.com/apache/streampark/security/code-scanning/135) ########## streampark-common/src/main/java/org/apache/streampark/common/util/DeflaterUtils.java: ########## @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.streampark.common.util; + +import org.apache.commons.lang3.StringUtils; + +import java.io.ByteArrayOutputStream; +import java.util.Base64; +import java.util.zip.DataFormatException; +import java.util.zip.Deflater; +import java.util.zip.Inflater; + +public final class DeflaterUtils { + + private DeflaterUtils() {} + + public static String zipString(String text) { + if (StringUtils.isBlank(text)) { + return ""; + } + Deflater deflater = new Deflater(Deflater.BEST_COMPRESSION); + deflater.setInput(text.getBytes()); + deflater.finish(); + byte[] bytes = new byte[256]; + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(256); + while (!deflater.finished()) { + int length = deflater.deflate(bytes); + outputStream.write(bytes, 0, length); + } + deflater.end(); + return Base64.getEncoder().encodeToString(outputStream.toByteArray()); + } + + public static String unzipString(String zipString) { + byte[] decode = Base64.getDecoder().decode(zipString); + Inflater inflater = new Inflater(); + inflater.setInput(decode); + byte[] bytes = new byte[256]; + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(256); + try { + while (!inflater.finished()) { + int length = inflater.inflate(bytes); + outputStream.write(bytes, 0, length); + } + } catch (DataFormatException e) { + e.printStackTrace(); Review Comment: ## SonarCloud / Debugging features should not be enabled in production <!--SONAR_ISSUE_KEY:AZ8yvnKplWsfFY_QVhpU-->Make sure this debug feature is deactivated before delivering the code in production. <p>See more on <a href="https://sonarcloud.io/project/issues?id=apache_incubator-streampark&issues=AZ8yvnKplWsfFY_QVhpU&open=AZ8yvnKplWsfFY_QVhpU&pullRequest=4419";>SonarQube Cloud</a></p> [Show more details](https://github.com/apache/streampark/security/code-scanning/130) -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
