This is an automated email from the ASF dual-hosted git repository.
benjobs pushed a commit to branch conf
in repository https://gitbox.apache.org/repos/asf/incubator-streampark.git
The following commit(s) were added to refs/heads/conf by this push:
new 4fe797819 [Improve] config improvements
4fe797819 is described below
commit 4fe7978197a292aee8246eedeacc1937687d3037
Author: benjobs <[email protected]>
AuthorDate: Sat Mar 30 17:23:26 2024 +0800
[Improve] config improvements
---
.../src/main/assembly/bin/streampark.sh | 101 ++++++++-------------
.../console/StreamParkConsoleBootstrap.java | 10 +-
.../console/base/config/SpringProperties.java | 46 ++++++----
.../console/system/authentication/JWTUtil.java | 14 +--
.../src/main/resources/config.yaml | 13 ++-
5 files changed, 83 insertions(+), 101 deletions(-)
diff --git
a/streampark-console/streampark-console-service/src/main/assembly/bin/streampark.sh
b/streampark-console/streampark-console-service/src/main/assembly/bin/streampark.sh
index 90ac0471e..72e4bd32e 100755
---
a/streampark-console/streampark-console-service/src/main/assembly/bin/streampark.sh
+++
b/streampark-console/streampark-console-service/src/main/assembly/bin/streampark.sh
@@ -137,6 +137,12 @@ APP_OUT="$APP_LOG"/streampark.out
# shellcheck disable=SC2034
APP_TMPDIR="$APP_BASE"/temp
+CONFIG="${APP_CONF}/config.yaml"
+if [[ ! -f "$CONFIG" ]] ; then
+ echo_r "ERROR: $CONFIG invalid or not found! please check.";
+ exit 1;
+fi
+
# Ensure that any user defined CLASSPATH variables are not used on startup,
# but allow them to be specified in setenv.sh, in rare case when it is needed.
CLASSPATH=
@@ -278,21 +284,18 @@ print_logo() {
printf ' %s ──────── Apache StreamPark, Make stream processing easier
ô~ô!%s\n\n' $PRIMARY $RESET
}
-parse_yaml() {
- local prefix=$2
- local s='[[:space:]]*' w='[a-zA-Z0-9_]*' fs=$(echo @|tr @ '\034')
- sed -ne "s|^\($s\):|\1|" \
- -e "s|^\($s\)\($w\)$s:$s[\"']\(.*\)[\"']$s\$|\1$fs\2$fs\3|p" \
- -e "s|^\($s\)\($w\)$s:$s\(.*\)$s\$|\1$fs\2$fs\3|p" $1 |
- awk -F$fs '{
- indent = length($1)/2;
- vname[indent] = $2;
- for (i in vname) {if (i > indent) {delete vname[i]}}
- if (length($3) > 0) {
- vn=""; for (i=0; i<indent; i++) {vn=(vn)(vname[i])("_")}
- printf("%s%s%s=\"%s\"\n", "'$prefix'",vn, $2, $3);
- }
- }'
+read_config() {
+ local prop_key=$1
+ local value
+ while IFS=':' read -r k v; do
+ k="${k/[[:space:]]/}"
+ v="${v/[[:space:]]/}"
+ if [[ ! $k = \#* ]] && [[ $k = $prop_key ]]; then
+ value=$v
+ break
+ fi
+ done < "$CONFIG"
+ echo "$value"
}
# shellcheck disable=SC2120
@@ -313,32 +316,25 @@ get_pid() {
fi
fi
- # shellcheck disable=SC2006
- local PROPER="${APP_CONF}/application.yml"
- if [[ ! -f "$PROPER" ]] ; then
- echo_r "ERROR: config file application.yml invalid or not found! ";
+ local serverPort=$(read_config "server.port")
+ if [ x"${serverPort}" == x"" ]; then
+ echo_r "server.port is required, please check $CONFIG"
exit 1;
- fi
-
- # shellcheck disable=SC2046
- eval $(parse_yaml "${PROPER}" "conf_")
- # shellcheck disable=SC2154
- # shellcheck disable=SC2155
- # shellcheck disable=SC2116
- local serverPort=$(echo "$conf_server_port")
- # shellcheck disable=SC2006
- # shellcheck disable=SC2155
- local used=`lsof -i:"$serverPort" | wc -l`
- if [ "$used" -gt 0 ]; then
- # shellcheck disable=SC2006
- local PID=`jps -l | grep "$APP_MAIN" | awk '{print $1}'`
- if [ ! -z $PID ]; then
- echo $PID
- else
- echo 0
- fi
else
- echo 0
+ # shellcheck disable=SC2006
+ # shellcheck disable=SC2155
+ local used=`lsof -i:"$serverPort" | wc -l`
+ if [ "$used" -gt 0 ]; then
+ # shellcheck disable=SC2006
+ local PID=`jps -l | grep "$APP_MAIN" | awk '{print $1}'`
+ if [ ! -z $PID ]; then
+ echo $PID
+ else
+ echo 0
+ fi
+ else
+ echo 0
+ fi
fi
}
@@ -365,22 +361,9 @@ start() {
echo_w "Using APP_PID: $APP_PID"
fi
- local PROPER="${APP_CONF}/application.yml"
- if [[ ! -f "$PROPER" ]] ; then
- echo_r "ERROR: config file application.yml invalid or not found! ";
- exit 1;
- else
- echo_g "Usage: config file: $PROPER ";
- fi
-
- # shellcheck disable=SC2046
- eval $(parse_yaml "${PROPER}" "conf_")
- # shellcheck disable=SC2001
- # shellcheck disable=SC2154
- # shellcheck disable=SC2155
- local workspace=$(echo "$conf_streampark_workspace_local" | sed 's/#.*$//g')
+ local workspace=$(read_config "streampark.workspace.local")
if [[ ! -d $workspace ]]; then
- echo_r "ERROR: streampark.workspace.local: \"$workspace\" is invalid path,
Please reconfigure in application.yml"
+ echo_r "ERROR: streampark.workspace.local: \"$workspace\" is invalid path,
Please check $CONFIG"
echo_r "NOTE: \"streampark.workspace.local\" Do not set under
APP_HOME($APP_HOME). Set it to a secure directory outside of APP_HOME. "
exit 1;
fi
@@ -429,8 +412,6 @@ start() {
eval $NOHUP $_RUNJAVA $JAVA_OPTS \
-classpath "$APP_CLASSPATH" \
-Dapp.home="${APP_HOME}" \
- -Dlogging.config="${APP_CONF}/logback-spring.xml" \
- -Dspring.config.location="${PROPER}" \
-Djava.io.tmpdir="$APP_TMPDIR" \
$APP_MAIN >> "$APP_OUT" 2>&1 "&"
@@ -462,14 +443,6 @@ start_docker() {
echo_w "Using APP_PID: $APP_PID"
fi
- local PROPER="${APP_CONF}/application.yml"
- if [[ ! -f "$PROPER" ]] ; then
- echo_r "ERROR: config file application.yml invalid or not found! ";
- exit 1;
- else
- echo_g "Usage: config file: $PROPER ";
- fi
-
if [ "${HADOOP_HOME}"x == ""x ]; then
echo_y "WARN: HADOOP_HOME is undefined on your system env,please check it."
else
diff --git
a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/StreamParkConsoleBootstrap.java
b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/StreamParkConsoleBootstrap.java
index 21c17e6d0..b54c99013 100644
---
a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/StreamParkConsoleBootstrap.java
+++
b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/StreamParkConsoleBootstrap.java
@@ -25,8 +25,6 @@ import
org.springframework.boot.builder.SpringApplicationBuilder;
import
org.springframework.boot.web.servlet.support.SpringBootServletInitializer;
import org.springframework.scheduling.annotation.EnableScheduling;
-import java.util.Map;
-
/**
*
*
@@ -52,9 +50,9 @@ import java.util.Map;
public class StreamParkConsoleBootstrap extends SpringBootServletInitializer {
public static void main(String[] args) throws Exception {
- Map<String, Object> properties = SpringProperties.getProperties();
- properties.forEach((k, v) -> System.setProperty(k, v.toString()));
- SpringApplicationBuilder builder = new
SpringApplicationBuilder().properties(properties);
- builder.sources(StreamParkConsoleBootstrap.class).run(args);
+ new SpringApplicationBuilder()
+ .properties(SpringProperties.get())
+ .sources(StreamParkConsoleBootstrap.class)
+ .run(args);
}
}
diff --git
a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/base/config/SpringProperties.java
b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/base/config/SpringProperties.java
index eff97497c..48c1fb0b7 100644
---
a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/base/config/SpringProperties.java
+++
b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/base/config/SpringProperties.java
@@ -14,11 +14,11 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.streampark.console.base.config;
-import org.apache.streampark.common.conf.ConfigConst;
import org.apache.streampark.common.util.PropertiesUtils;
-import org.apache.streampark.common.util.SystemPropertyUtils;
+import org.apache.streampark.console.base.util.WebUtils;
import org.apache.commons.lang3.StringUtils;
@@ -27,15 +27,16 @@ import com.google.common.collect.Maps;
import java.io.File;
import java.io.InputStream;
import java.util.Map;
+import java.util.Properties;
public class SpringProperties {
- public static Map<String, Object> getProperties() {
+ public static Properties get() {
// 1) get spring config
- Map<String, Object> springConfig = getSpringConfig();
+ Properties springConfig = getSpringConfig();
// 2) get user config
- Map<String, String> userConfig = getUserConfig();
+ Properties userConfig = getUserConfig();
// 3) merge config
mergeConfig(userConfig, springConfig);
@@ -43,13 +44,14 @@ public class SpringProperties {
// 4) datasource
dataSourceConfig(userConfig, springConfig);
+ // 5) system.setProperties
+ springConfig.forEach((k, v) -> System.setProperty(k.toString(),
v.toString()));
+
return springConfig;
}
- private static void dataSourceConfig(
- Map<String, String> userConfig, Map<String, Object> springConfig) {
-
- String dialect = userConfig.remove("datasource.dialect");
+ private static void dataSourceConfig(Properties userConfig, Properties
springConfig) {
+ String dialect = userConfig.getProperty("datasource.dialect",
"").toString();
if (StringUtils.isBlank(dialect)) {
throw new ExceptionInInitializerError(
"datasource.dialect is required, please check config.yaml");
@@ -79,8 +81,7 @@ public class SpringProperties {
}
}
- private static void mergeConfig(
- Map<String, String> userConfig, Map<String, Object> springConfig) {
+ private static void mergeConfig(Properties userConfig, Properties
springConfig) {
Map<String, String> configMapping = Maps.newHashMap();
configMapping.put("datasource.username", "spring.datasource.username");
@@ -89,7 +90,7 @@ public class SpringProperties {
userConfig.forEach(
(k, v) -> {
- if (StringUtils.isNotBlank(v)) {
+ if (StringUtils.isNoneBlank(k.toString(), v.toString())) {
String key = configMapping.get(k);
if (key != null) {
springConfig.put(key, v);
@@ -100,23 +101,30 @@ public class SpringProperties {
});
}
- private static Map<String, String> getUserConfig() {
- String appHome = SystemPropertyUtils.get(ConfigConst.KEY_APP_HOME(), null);
+ private static Properties getUserConfig() {
+ String appHome = WebUtils.getAppHome();
if (appHome != null) {
File file = new File(appHome + "/conf/config.yaml");
if (file.exists() && file.isFile()) {
- return PropertiesUtils.fromYamlFileAsJava(file.getAbsolutePath());
+ Properties properties = new Properties();
+ Map<String, String> configMapping =
+ PropertiesUtils.fromYamlFileAsJava(file.getAbsolutePath());
+ properties.putAll(configMapping);
+ return properties;
}
+ throw new ExceptionInInitializerError(file.getAbsolutePath() + " not
found, please check.");
} else {
InputStream inputStream =
SpringProperties.class.getClassLoader().getResourceAsStream("config.yaml");
- return PropertiesUtils.fromYamlFileAsJava(inputStream);
+ Properties properties = new Properties();
+ Map<String, String> configMapping =
PropertiesUtils.fromYamlFileAsJava(inputStream);
+ properties.putAll(configMapping);
+ return properties;
}
- throw new ExceptionInInitializerError("config.yaml not found");
}
- private static Map<String, Object> getSpringConfig() {
- Map<String, Object> config = Maps.newHashMap();
+ private static Properties getSpringConfig() {
+ Properties config = new Properties();
// env
config.put("spring.devtools.restart.enabled", "false");
config.put("spring.aop.proxy-target-class", "true");
diff --git
a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/system/authentication/JWTUtil.java
b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/system/authentication/JWTUtil.java
index 8975a69a5..8b033d512 100644
---
a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/system/authentication/JWTUtil.java
+++
b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/system/authentication/JWTUtil.java
@@ -31,7 +31,7 @@ import java.util.regex.Pattern;
@Slf4j
public class JWTUtil {
- private static Long TTL_SECOND;
+ private static Long ttlOfSecond;
/**
* verify token
@@ -114,7 +114,7 @@ public class JWTUtil {
}
public static Long getTTLOfSecond() {
- if (TTL_SECOND == null) {
+ if (ttlOfSecond == null) {
String ttl = System.getProperty("server.session.ttl", "24h").trim();
String regexp = "^\\d+(s|m|h|d)$";
Pattern pattern = Pattern.compile(regexp);
@@ -127,15 +127,15 @@ public class JWTUtil {
Long second = Long.parseLong(time);
switch (unit) {
case "m":
- return TTL_SECOND = second * 60;
+ return ttlOfSecond = second * 60;
case "h":
- return TTL_SECOND = second * 60 * 60;
+ return ttlOfSecond = second * 60 * 60;
case "d":
- return TTL_SECOND = second * 24 * 60 * 60;
+ return ttlOfSecond = second * 24 * 60 * 60;
default:
- return TTL_SECOND = second;
+ return ttlOfSecond = second;
}
}
- return TTL_SECOND;
+ return ttlOfSecond;
}
}
diff --git
a/streampark-console/streampark-console-service/src/main/resources/config.yaml
b/streampark-console/streampark-console-service/src/main/resources/config.yaml
index 36e64fdab..a9df3d8f7 100644
---
a/streampark-console/streampark-console-service/src/main/resources/config.yaml
+++
b/streampark-console/streampark-console-service/src/main/resources/config.yaml
@@ -24,18 +24,21 @@ server.port: 10000
server.session.ttl: 2h # unit[s|m|h|d], e.g: 24h, 2d....
# see:
https://github.com/undertow-io/undertow/blob/master/core/src/main/java/io/undertow/Undertow.java
+server.undertow.direct-buffers: true
server.undertow.buffer-size: 1024
-server.undertow.direct-buffers: 1024
server.undertow.threads.io: 16
server.undertow.threads.worker: 256
# system database, default h2, mysql|pgsql|h2
-datasource.dialect: mysql # h2, pgsql
+datasource.dialect: h2 # h2, pgsql
#-------if datasource.dialect is mysql or pgsql, it is necessary to set-------
-datasource.username: root
-datasource.password: streampark
+datasource.username:
+datasource.password:
+# mysql jdbc url example:
+# datasource.url:
jdbc:mysql://localhost:3306/streampark?useUnicode=true&characterEncoding=UTF-8&useJDBCCompliantTimezoneShift=true&useLegacyDatetimeCode=false&serverTimezone=GMT%2B8
+# postgresql jdbc url example:
# datasource.url:
jdbc:postgresql://localhost:5432/streampark?stringtype=unspecified
-datasource.url:
jdbc:mysql://localhost:3306/streampark?useUnicode=true&characterEncoding=UTF-8&useJDBCCompliantTimezoneShift=true&useLegacyDatetimeCode=false&serverTimezone=GMT%2B8
+datasource.url:
#---------------------------------------------------------------------------------
# Directory for storing locally built project
