This is an automated email from the ASF dual-hosted git repository.
benjobs pushed a commit to branch dev-2.1.4
in repository https://gitbox.apache.org/repos/asf/incubator-streampark.git
The following commit(s) were added to refs/heads/dev-2.1.4 by this push:
new c3c468c91 [Improve] Limit the number of running build projects #3687
(#3695)
c3c468c91 is described below
commit c3c468c9192dd87b4ae430a41735bde7a391dfba
Author: benjobs <[email protected]>
AuthorDate: Wed May 1 01:15:28 2024 +0800
[Improve] Limit the number of running build projects #3687 (#3695)
* [Improve] Limit the number of running build projects #3687
---------
Co-authored-by: benjobs <[email protected]>
---
.../src/main/assembly/assembly.xml | 6 +++-
.../main/{resources => assembly/conf}/config.yaml | 3 ++
.../console/base/config/SpringProperties.java | 42 +++++++++-------------
.../console/core/mapper/ProjectMapper.java | 2 ++
.../core/service/impl/ProjectServiceImpl.java | 12 +++++++
.../main/resources/mapper/core/ProjectMapper.xml | 4 +++
6 files changed, 43 insertions(+), 26 deletions(-)
diff --git
a/streampark-console/streampark-console-service/src/main/assembly/assembly.xml
b/streampark-console/streampark-console-service/src/main/assembly/assembly.xml
index dc877c5ac..e9e555601 100644
---
a/streampark-console/streampark-console-service/src/main/assembly/assembly.xml
+++
b/streampark-console/streampark-console-service/src/main/assembly/assembly.xml
@@ -63,6 +63,11 @@
<outputDirectory>lib</outputDirectory>
<fileMode>0755</fileMode>
</fileSet>
+ <fileSet>
+
<directory>${project.build.directory}/../src/main/assembly/conf</directory>
+ <outputDirectory>conf</outputDirectory>
+ <fileMode>0755</fileMode>
+ </fileSet>
<fileSet>
<directory>${project.build.directory}/../src/main/assembly/logs</directory>
<outputDirectory>logs</outputDirectory>
@@ -95,7 +100,6 @@
<lineEnding>unix</lineEnding>
<fileMode>0755</fileMode>
<includes>
- <include>config.yaml</include>
<include>logback-spring.xml</include>
</includes>
</fileSet>
diff --git
a/streampark-console/streampark-console-service/src/main/resources/config.yaml
b/streampark-console/streampark-console-service/src/main/assembly/conf/config.yaml
similarity index 96%
rename from
streampark-console/streampark-console-service/src/main/resources/config.yaml
rename to
streampark-console/streampark-console-service/src/main/assembly/conf/config.yaml
index 5edf8cdfe..c6845eaad 100644
---
a/streampark-console/streampark-console-service/src/main/resources/config.yaml
+++
b/streampark-console/streampark-console-service/src/main/assembly/conf/config.yaml
@@ -61,6 +61,9 @@ streampark:
http-auth: 'simple' # default simple, or kerberos
# flink on yarn or spark on yarn, HADOOP_USER_NAME
hadoop-user-name: hdfs
+ project:
+ # Number of projects allowed to be running at the same time , If there is
no limit, -1 can be configured
+ max-build: 16
# flink on yarn or spark on yarn, when the hadoop cluster enable kerberos
authentication, it is necessary to set Kerberos authentication parameters.
security:
diff --git
a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/base/config/SpringProperties.java
b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/base/config/SpringProperties.java
index 1a53cbf12..1d755f668 100644
---
a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/base/config/SpringProperties.java
+++
b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/base/config/SpringProperties.java
@@ -17,6 +17,7 @@
package org.apache.streampark.console.base.config;
+import org.apache.streampark.common.conf.ConfigConst;
import org.apache.streampark.common.util.PropertiesUtils;
import org.apache.streampark.common.util.SystemPropertyUtils;
import org.apache.streampark.console.base.util.WebUtils;
@@ -28,7 +29,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
-import java.io.InputStream;
import java.util.Map;
import java.util.Properties;
@@ -46,10 +46,10 @@ public class SpringProperties {
SystemPropertyUtils.set("spring.config.location",
oldConfig.getAbsolutePath());
return new Properties();
} else {
- // 1) get spring config
- Properties springConfig = getSpringConfig();
- // 2) get user config
+ // 1) get user config
Properties userConfig = getUserConfig();
+ // 2) get spring config
+ Properties springConfig = getSpringConfig();
// 3) merge config
mergeConfig(userConfig, springConfig);
// 4) datasource
@@ -122,32 +122,24 @@ public class SpringProperties {
});
}
- private static boolean useOldConfig() {
- String appHome = WebUtils.getAppHome();
- if (appHome == null) {
- return false;
- }
- File file = new File(appHome + "/conf/application.yml");
- return file.exists();
- }
-
private static Properties getUserConfig() {
String appHome = WebUtils.getAppHome();
+ if (StringUtils.isBlank(appHome)) {
+ throw new ExceptionInInitializerError(
+ String.format(
+ "[StreamPark] The system initialization check failed. If started
local for development and debugging,"
+ + " please ensure the -D%s parameter is clearly specified,"
+ + " more detail:
https://streampark.apache.org/docs/user-guide/deployment";,
+ ConfigConst.KEY_APP_HOME()));
+ }
Properties properties = new Properties();
- if (appHome != null) {
- File file = new File(appHome + "/conf/config.yaml");
- if (file.exists() && file.isFile()) {
- Map<String, String> config =
PropertiesUtils.fromYamlFileAsJava(file.getAbsolutePath());
- properties.putAll(config);
- return properties;
- }
- throw new ExceptionInInitializerError(file.getAbsolutePath() + " not
found, please check.");
- } else {
- InputStream inputStream =
-
SpringProperties.class.getClassLoader().getResourceAsStream("config.yaml");
- Map<String, String> config =
PropertiesUtils.fromYamlFileAsJava(inputStream);
+ File file = new File(appHome, "conf/config.yaml");
+ if (file.exists() && file.isFile()) {
+ Map<String, String> config =
PropertiesUtils.fromYamlFileAsJava(file.getAbsolutePath());
properties.putAll(config);
return properties;
+ } else {
+ throw new ExceptionInInitializerError(file.getAbsolutePath() + " not
found, please check.");
}
}
diff --git
a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/mapper/ProjectMapper.java
b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/mapper/ProjectMapper.java
index 91752d5df..d5f62ce0d 100644
---
a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/mapper/ProjectMapper.java
+++
b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/mapper/ProjectMapper.java
@@ -38,4 +38,6 @@ public interface ProjectMapper extends BaseMapper<Project> {
Boolean existsByTeamId(@Param("teamId") Long teamId);
List<Project> selectByTeamId(@Param("teamId") Long teamId);
+
+ Long getBuildingCount();
}
diff --git
a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/ProjectServiceImpl.java
b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/ProjectServiceImpl.java
index d749e062b..a2d38da26 100644
---
a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/ProjectServiceImpl.java
+++
b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/ProjectServiceImpl.java
@@ -50,6 +50,7 @@ import
com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Propagation;
import org.springframework.transaction.annotation.Transactional;
@@ -83,6 +84,9 @@ public class ProjectServiceImpl extends
ServiceImpl<ProjectMapper, Project>
@Autowired private FlinkAppHttpWatcher flinkAppHttpWatcher;
+ @Value("${streampark.project.max-build:6}")
+ public Long maxProjectBuildNum;
+
private static final int CPU_NUM = Math.max(4,
Runtime.getRuntime().availableProcessors() * 2);
private final ExecutorService projectBuildExecutor =
@@ -193,6 +197,14 @@ public class ProjectServiceImpl extends
ServiceImpl<ProjectMapper, Project>
@Override
public void build(Long id) throws Exception {
+ Long currentBuildCount = this.baseMapper.getBuildingCount();
+
+ ApiAlertException.throwIfTrue(
+ maxProjectBuildNum > -1 && currentBuildCount > maxProjectBuildNum,
+ String.format(
+ "The number of running Build projects exceeds the maximum number:
%d of max-build-num",
+ maxProjectBuildNum));
+
Project project = getById(id);
this.baseMapper.updateBuildState(project.getId(),
BuildState.BUILDING.get());
String logPath = getBuildLogPath(id);
diff --git
a/streampark-console/streampark-console-service/src/main/resources/mapper/core/ProjectMapper.xml
b/streampark-console/streampark-console-service/src/main/resources/mapper/core/ProjectMapper.xml
index eb242ba38..a221b3bd8 100644
---
a/streampark-console/streampark-console-service/src/main/resources/mapper/core/ProjectMapper.xml
+++
b/streampark-console/streampark-console-service/src/main/resources/mapper/core/ProjectMapper.xml
@@ -85,4 +85,8 @@
</where>
</select>
+ <select id="getBuildingCount" resultType="java.lang.Long">
+ select count(1) from t_flink_project where build_state = 0
+ </select>
+
</mapper>
