This is an automated email from the ASF dual-hosted git repository.
benjobs pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/incubator-streampark.git
The following commit(s) were added to refs/heads/dev by this push:
new b836c2041 [ISSUES-3623] Project management encrypts user warehouse
passwords (#3630)
b836c2041 is described below
commit b836c20419ec5a9dcd3cb3d7e440e53ac2887e30
Author: jianjun.xu <[email protected]>
AuthorDate: Wed Mar 27 18:47:08 2024 +0800
[ISSUES-3623] Project management encrypts user warehouse passwords (#3630)
* [ISSUES-3623] Project management encrypts user warehouse passwords
* [ISSUES-3623] Project management encrypts user warehouse passwords
* [ISSUES-3623] checkstyle & spotless:apply
---------
Co-authored-by: jianjun.xu <[email protected]>
---
.../src/main/assembly/script/data/mysql-data.sql | 6 +-
.../src/main/assembly/script/data/pgsql-data.sql | 6 +-
.../main/assembly/script/schema/mysql-schema.sql | 3 +-
.../main/assembly/script/schema/pgsql-schema.sql | 3 +-
.../main/assembly/script/upgrade/mysql/2.2.2.sql | 4 ++
.../main/assembly/script/upgrade/pgsql/2.2.2.sql | 8 +++
.../streampark/console/base/util/GitUtils.java | 15 ++++-
.../console/core/controller/ProjectController.java | 4 +-
.../streampark/console/core/entity/Project.java | 3 +
.../console/core/service/ProjectService.java | 17 ++++++
.../core/service/impl/ProjectServiceImpl.java | 70 +++++++++++++++++++++-
.../src/views/resource/project/useProject.tsx | 4 +-
12 files changed, 128 insertions(+), 15 deletions(-)
diff --git
a/streampark-console/streampark-console-service/src/main/assembly/script/data/mysql-data.sql
b/streampark-console/streampark-console-service/src/main/assembly/script/data/mysql-data.sql
index b47810339..8b232c9ed 100644
---
a/streampark-console/streampark-console-service/src/main/assembly/script/data/mysql-data.sql
+++
b/streampark-console/streampark-console-service/src/main/assembly/script/data/mysql-data.sql
@@ -40,7 +40,7 @@ insert into `t_flink_effective` values (100000, 100000, 2,
100000, now());
-- ----------------------------
-- Records of t_flink_project
-- ----------------------------
-insert into `t_flink_project` values (100000, 100000, 'streampark-quickstart',
'https://github.com/apache/incubator-streampark-quickstart', 'release-2.0.0',
null, null, null, null, null, 1, 1, null, 'streampark-quickstart', -1, now(),
now());
+insert into `t_flink_project` values (100000, 100000, 'streampark-quickstart',
'https://github.com/apache/incubator-streampark-quickstart', 'release-2.0.0',
null, null, null, null, null, null, 1, 1, null, 'streampark-quickstart', -1,
now(), now());
-- ----------------------------
-- Records of t_flink_sql
@@ -87,10 +87,10 @@ insert into `t_menu` values (130200, 130000,
'resource.variable', '/resource/var
insert into `t_menu` values (130300, 130000, 'resource.upload',
'/resource/upload', 'resource/upload/View', null, null, '0', 1, 3, now(),
now());
insert into `t_menu` values (130101, 130100, 'project view', null, null,
'project:view', null, '1', 1, null, now(), now());
-insert into `t_menu` values (130102, 130100, 'project add', '/project/add',
'project/Add', 'project:create', '', '0', 0, null, now(), now());
+insert into `t_menu` values (130102, 130100, 'project add', '/project/add',
'resource/project/Add', 'project:create', '', '0', 0, null, now(), now());
insert into `t_menu` values (130103, 130100, 'project build', null, null,
'project:build', null, '1', 1, null, now(), now());
insert into `t_menu` values (130104, 130100, 'project delete', null, null,
'project:delete', null, '1', 1, null, now(), now());
-insert into `t_menu` values (130105, 130100, 'project edit', '/project/edit',
'project/Edit', 'project:update', null, '0', 0, null, now(), now());
+insert into `t_menu` values (130105, 130100, 'project edit', '/project/edit',
'resource/project/Edit', 'project:update', null, '0', 0, null, now(), now());
insert into `t_menu` values (130201, 130200, 'variable view', NULL, NULL,
'variable:view', NULL, '1', 1, null, now(), now());
insert into `t_menu` values (130202, 130200, 'variable depend view', null,
null, 'variable:depend_apps', null, '1', 1, NULL, now(), now());
diff --git
a/streampark-console/streampark-console-service/src/main/assembly/script/data/pgsql-data.sql
b/streampark-console/streampark-console-service/src/main/assembly/script/data/pgsql-data.sql
index d9105e592..ee914d3af 100644
---
a/streampark-console/streampark-console-service/src/main/assembly/script/data/pgsql-data.sql
+++
b/streampark-console/streampark-console-service/src/main/assembly/script/data/pgsql-data.sql
@@ -35,7 +35,7 @@ insert into "public"."t_flink_effective" values (100000,
100000, 2, 100000, now(
-- ----------------------------
-- Records of t_flink_project
-- ----------------------------
-insert into "public"."t_flink_project" values (100000, 100000,
'streampark-quickstart',
'https://github.com/apache/incubator-streampark-quickstart', 'release-2.0.0',
null, null, null, null, null, 1, 1, null, 'streampark-quickstart', -1, now(),
now());
+insert into "public"."t_flink_project" values (100000, 100000,
'streampark-quickstart',
'https://github.com/apache/incubator-streampark-quickstart', 'release-2.0.0',
null, null, null, null, null, null, 1, 1, null, 'streampark-quickstart', -1,
now(), now());
-- ----------------------------
@@ -87,10 +87,10 @@ insert into "public"."t_menu" values (110602, 110600,
'update', null, null, 'mem
insert into "public"."t_menu" values (110603, 110600, 'delete', null, null,
'member:delete', null, '1', '1', null, now(), now());
insert into "public"."t_menu" values (110604, 110600, 'role view', null, null,
'role:view', null, '1', '1', null, now(), now());
insert into "public"."t_menu" values (110605, 110600, 'view', null, null,
'member:view', null, '1', '1', null, now(), now());
-insert into "public"."t_menu" values (120101, 120100, 'add',
'/flink/project/add', 'flink/project/Add', 'project:create', '', '0', '0',
null, now(), now());
+insert into "public"."t_menu" values (120101, 120100, 'add',
'/flink/project/add', 'resource/project/Add', 'project:create', '', '0', '0',
null, now(), now());
insert into "public"."t_menu" values (120102, 120100, 'build', null, null,
'project:build', null, '1', '1', null, now(), now());
insert into "public"."t_menu" values (120103, 120100, 'delete', null, null,
'project:delete', null, '1', '1', null, now(), now());
-insert into "public"."t_menu" values (120104, 120100, 'edit',
'/flink/project/edit', 'flink/project/Edit', 'project:update', null, '0', '0',
null, now(), now());
+insert into "public"."t_menu" values (120104, 120100, 'edit',
'/flink/project/edit', 'resource/project/Edit', 'project:update', null, '0',
'0', null, now(), now());
insert into "public"."t_menu" values (120105, 120100, 'view', null, null,
'project:view', null, '1', '1', null, now(), now());
insert into "public"."t_menu" values (120201, 120200, 'add', '/flink/app/add',
'flink/app/Add', 'app:create', '', '0', '0', null, now(), now());
insert into "public"."t_menu" values (120202, 120200, 'detail app',
'/flink/app/detail', 'flink/app/Detail', 'app:detail', '', '0', '0', null,
now(), now());
diff --git
a/streampark-console/streampark-console-service/src/main/assembly/script/schema/mysql-schema.sql
b/streampark-console/streampark-console-service/src/main/assembly/script/schema/mysql-schema.sql
index ac4e5f2dc..08fce997a 100644
---
a/streampark-console/streampark-console-service/src/main/assembly/script/schema/mysql-schema.sql
+++
b/streampark-console/streampark-console-service/src/main/assembly/script/schema/mysql-schema.sql
@@ -189,8 +189,9 @@ create table `t_flink_project` (
`url` varchar(255) collate utf8mb4_general_ci default null,
`branches` varchar(64) collate utf8mb4_general_ci default null,
`user_name` varchar(64) collate utf8mb4_general_ci default null,
- `password` varchar(64) collate utf8mb4_general_ci default null,
+ `password` varchar(512) collate utf8mb4_general_ci default null,
`prvkey_path` varchar(128) collate utf8mb4_general_ci default null,
+ `salt` varchar(26) collate utf8mb4_general_ci default null,
`pom` varchar(255) collate utf8mb4_general_ci default null,
`build_args` varchar(255) default null,
`type` tinyint default null,
diff --git
a/streampark-console/streampark-console-service/src/main/assembly/script/schema/pgsql-schema.sql
b/streampark-console/streampark-console-service/src/main/assembly/script/schema/pgsql-schema.sql
index 3b4608b1b..c3eb9dca3 100644
---
a/streampark-console/streampark-console-service/src/main/assembly/script/schema/pgsql-schema.sql
+++
b/streampark-console/streampark-console-service/src/main/assembly/script/schema/pgsql-schema.sql
@@ -434,8 +434,9 @@ create table "public"."t_flink_project" (
"url" varchar(255) collate "pg_catalog"."default",
"branches" varchar(64) collate "pg_catalog"."default",
"user_name" varchar(64) collate "pg_catalog"."default",
- "password" varchar(64) collate "pg_catalog"."default",
+ "password" varchar(512) collate "pg_catalog"."default",
"prvkey_path" varchar(128) collate "pg_catalog"."default",
+ "salt" varchar(26) collate "pg_catalog"."default",
"pom" varchar(255) collate "pg_catalog"."default",
"build_args" varchar(255) collate "pg_catalog"."default",
"type" int2,
diff --git
a/streampark-console/streampark-console-service/src/main/assembly/script/upgrade/mysql/2.2.2.sql
b/streampark-console/streampark-console-service/src/main/assembly/script/upgrade/mysql/2.2.2.sql
index d4707a8a3..5c6652fe9 100644
---
a/streampark-console/streampark-console-service/src/main/assembly/script/upgrade/mysql/2.2.2.sql
+++
b/streampark-console/streampark-console-service/src/main/assembly/script/upgrade/mysql/2.2.2.sql
@@ -244,3 +244,7 @@ alter table `t_flink_app`
alter table `t_flink_log`
add column `user_id` bigint default null comment 'operator user id';
+
+alter table `t_flink_project`
+ add column `salt` varchar(26) collate utf8mb4_general_ci default null
comment 'password salt',
+ modify column `password` varchar(512) collate utf8mb4_general_ci default
null comment 'password';
diff --git
a/streampark-console/streampark-console-service/src/main/assembly/script/upgrade/pgsql/2.2.2.sql
b/streampark-console/streampark-console-service/src/main/assembly/script/upgrade/pgsql/2.2.2.sql
index c8d023ba0..bc69fb9b6 100644
---
a/streampark-console/streampark-console-service/src/main/assembly/script/upgrade/pgsql/2.2.2.sql
+++
b/streampark-console/streampark-console-service/src/main/assembly/script/upgrade/pgsql/2.2.2.sql
@@ -22,3 +22,11 @@ alter table "public"."t_flink_log"
add column "user_id" int8 collate "pg_catalog"."default";
comment on column "public"."t_flink_log"."user_id" is 'operator user id';
+
+alter table "public"."t_flink_project"
+ add column `salt` varchar(26) collate "pg_catalog"."default";
+
+comment on column "public"."t_flink_project"."salt" is 'password salt';
+
+alter table "public"."t_flink_project"
+ alter column `password` type varchar(512) collate "pg_catalog"."default";
diff --git
a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/base/util/GitUtils.java
b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/base/util/GitUtils.java
index 4fa554840..72dc1f1b9 100644
---
a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/base/util/GitUtils.java
+++
b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/base/util/GitUtils.java
@@ -84,9 +84,18 @@ public class GitUtils {
private static void setCredentials(TransportCommand<?, ?> transportCommand,
Project project) {
if (project.isHttpRepositoryUrl()) {
if (!StringUtils.isAllEmpty(project.getUserName(),
project.getPassword())) {
- UsernamePasswordCredentialsProvider credentialsProvider =
- new UsernamePasswordCredentialsProvider(project.getUserName(),
project.getPassword());
- transportCommand.setCredentialsProvider(credentialsProvider);
+ try {
+ String decrypt =
+ StringUtils.isNotBlank(project.getSalt())
+ ? EncryptUtils.decrypt(project.getPassword(),
project.getSalt())
+ : project.getPassword();
+ UsernamePasswordCredentialsProvider credentialsProvider =
+ new UsernamePasswordCredentialsProvider(project.getUserName(),
decrypt);
+ transportCommand.setCredentialsProvider(credentialsProvider);
+ } catch (Exception e) {
+ throw new IllegalStateException(
+ "[StreamPark] git setCredentials: project password decrypt
failed", e);
+ }
}
} else if (project.isSshRepositoryUrl()) {
transportCommand.setTransportConfigCallback(
diff --git
a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/controller/ProjectController.java
b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/controller/ProjectController.java
index bbbb2f9d6..191e8ae83 100644
---
a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/controller/ProjectController.java
+++
b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/controller/ProjectController.java
@@ -105,7 +105,7 @@ public class ProjectController {
@Operation(summary = "List git project branches")
@PostMapping("branches")
public RestResponse branches(Project project) {
- List<String> branches = project.getAllBranches();
+ List<String> branches = projectService.getAllBranches(project);
return RestResponse.success().data(branches);
}
@@ -120,7 +120,7 @@ public class ProjectController {
@Operation(summary = "Authenticate git project")
@PostMapping("gitcheck")
public RestResponse gitCheck(Project project) {
- GitAuthorizedErrorEnum error = project.gitCheck();
+ GitAuthorizedErrorEnum error = projectService.gitCheck(project);
return RestResponse.success().data(error.getType());
}
diff --git
a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/entity/Project.java
b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/entity/Project.java
index 9f5bd6c80..8efaa0eee 100644
---
a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/entity/Project.java
+++
b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/entity/Project.java
@@ -77,6 +77,9 @@ public class Project implements Serializable {
@TableField(updateStrategy = FieldStrategy.IGNORED)
private String prvkeyPath;
+ /** No salt value is returned */
+ @JsonIgnore private String salt;
+
/** 1:git 2:svn */
private Integer repository;
diff --git
a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/ProjectService.java
b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/ProjectService.java
index ec9d09620..6767b24ba 100644
---
a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/ProjectService.java
+++
b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/ProjectService.java
@@ -21,6 +21,7 @@ import org.apache.streampark.console.base.domain.RestRequest;
import org.apache.streampark.console.base.domain.RestResponse;
import org.apache.streampark.console.core.entity.Application;
import org.apache.streampark.console.core.entity.Project;
+import org.apache.streampark.console.core.enums.GitAuthorizedErrorEnum;
import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.extension.service.IService;
@@ -145,4 +146,20 @@ public interface ProjectService extends IService<Project> {
* @return whether the corresponding project exists
*/
boolean exists(Project project);
+
+ /**
+ * Gets branch information under the project
+ *
+ * @param project Project
+ * @return branch information under the project
+ */
+ List<String> getAllBranches(Project project);
+
+ /**
+ * Check git
+ *
+ * @param project Project
+ * @return Check git
+ */
+ GitAuthorizedErrorEnum gitCheck(Project project);
}
diff --git
a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/ProjectServiceImpl.java
b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/ProjectServiceImpl.java
index 778b1a03b..1bc047830 100644
---
a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/ProjectServiceImpl.java
+++
b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/ProjectServiceImpl.java
@@ -28,11 +28,16 @@ import
org.apache.streampark.console.base.domain.ResponseCode;
import org.apache.streampark.console.base.domain.RestRequest;
import org.apache.streampark.console.base.domain.RestResponse;
import org.apache.streampark.console.base.exception.ApiAlertException;
+import org.apache.streampark.console.base.exception.ApiDetailException;
import org.apache.streampark.console.base.mybatis.pager.MybatisPager;
+import org.apache.streampark.console.base.util.EncryptUtils;
import org.apache.streampark.console.base.util.GZipUtils;
+import org.apache.streampark.console.base.util.GitUtils;
+import org.apache.streampark.console.base.util.ShaHashUtils;
import org.apache.streampark.console.core.entity.Application;
import org.apache.streampark.console.core.entity.Project;
import org.apache.streampark.console.core.enums.BuildStateEnum;
+import org.apache.streampark.console.core.enums.GitAuthorizedErrorEnum;
import org.apache.streampark.console.core.enums.ReleaseStateEnum;
import org.apache.streampark.console.core.mapper.ProjectMapper;
import org.apache.streampark.console.core.service.ProjectService;
@@ -40,6 +45,7 @@ import
org.apache.streampark.console.core.service.application.ApplicationManageS
import org.apache.streampark.console.core.task.ProjectBuildTask;
import org.apache.streampark.console.core.watcher.FlinkAppHttpWatcher;
+import org.apache.commons.lang3.StringUtils;
import org.apache.flink.configuration.MemorySize;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
@@ -94,7 +100,17 @@ public class ProjectServiceImpl extends
ServiceImpl<ProjectMapper, Project>
RestResponse response = RestResponse.success();
ApiAlertException.throwIfTrue(count > 0, "project name already exists, add
project failed");
-
+ if (StringUtils.isNotBlank(project.getPassword())) {
+ String salt = ShaHashUtils.getRandomSalt();
+ try {
+ String encrypt = EncryptUtils.encrypt(project.getPassword(), salt);
+ project.setSalt(salt);
+ project.setPassword(encrypt);
+ } catch (Exception e) {
+ log.error("Project password decrypt failed", e);
+ throw new ApiAlertException("Project github/gitlab password decrypt
failed");
+ }
+ }
Date date = new Date();
project.setCreateTime(date);
project.setModifyTime(date);
@@ -117,6 +133,26 @@ public class ProjectServiceImpl extends
ServiceImpl<ProjectMapper, Project>
!project.getBuildState().equals(BuildStateEnum.BUILDING.get()),
"The project is being built, update project failed.");
updateInternal(projectParam, project);
+ if (project.isHttpRepositoryUrl()) {
+ if (StringUtils.isBlank(projectParam.getUserName())) {
+ project.setUserName(null);
+ project.setPassword(null);
+ project.setSalt(null);
+ } else {
+ project.setUserName(projectParam.getUserName());
+ if (!Objects.equals(projectParam.getPassword(),
project.getPassword())) {
+ try {
+ String salt = ShaHashUtils.getRandomSalt();
+ String encrypt = EncryptUtils.encrypt(projectParam.getPassword(),
salt);
+ project.setPassword(encrypt);
+ project.setSalt(salt);
+ } catch (Exception e) {
+ log.error("The project github/gitlab password encrypt failed");
+ throw new ApiAlertException(e);
+ }
+ }
+ }
+ }
if (project.isSshRepositoryUrl()) {
project.setUserName(null);
} else {
@@ -378,4 +414,36 @@ public class ProjectServiceImpl extends
ServiceImpl<ProjectMapper, Project>
private String getBuildLogPath(Long projectId) {
return String.format("%s/%s/build.log",
Workspace.PROJECT_BUILD_LOG_PATH(), projectId);
}
+
+ @Override
+ public List<String> getAllBranches(Project project) {
+ try {
+ return GitUtils.getBranchList(remakeProject(project));
+ } catch (Exception e) {
+ throw new ApiDetailException(e);
+ }
+ }
+
+ @Override
+ public GitAuthorizedErrorEnum gitCheck(Project project) {
+ try {
+ GitUtils.getBranchList(remakeProject(project));
+ return GitAuthorizedErrorEnum.SUCCESS;
+ } catch (Exception e) {
+ String err = e.getMessage();
+ if (err.contains("not authorized")) {
+ return GitAuthorizedErrorEnum.ERROR;
+ } else if (err.contains("Authentication is required")) {
+ return GitAuthorizedErrorEnum.REQUIRED;
+ }
+ return GitAuthorizedErrorEnum.UNKNOW;
+ }
+ }
+
+ private Project remakeProject(Project project) {
+ if (Objects.nonNull(project.getId())) {
+ return this.baseMapper.selectById(project.getId());
+ }
+ return project;
+ }
}
diff --git
a/streampark-console/streampark-console-webapp/src/views/resource/project/useProject.tsx
b/streampark-console/streampark-console-webapp/src/views/resource/project/useProject.tsx
index 5c04c7126..67c3a15a4 100644
---
a/streampark-console/streampark-console-webapp/src/views/resource/project/useProject.tsx
+++
b/streampark-console/streampark-console-webapp/src/views/resource/project/useProject.tsx
@@ -244,6 +244,7 @@ export const useProject = () => {
userName: values.userName || null,
password: values.password || null,
prvkeyPath: values.prvkeyPath || null,
+ id: route?.query?.id || null,
});
if (res === 0) {
if (branchList.value.length === 0) {
@@ -282,8 +283,9 @@ export const useProject = () => {
const prvkeyPath = values.prvkeyPath || null;
const userNull = userName === null || userName === undefined ||
userName === '';
const passNull = password === null || password === undefined ||
password === '';
+ const id = route?.query?.id || null;
if ((userNull && passNull) || (!userNull && !passNull)) {
- const res = await fetchBranches({ url, userName, password,
prvkeyPath });
+ const res = await fetchBranches({ url, userName, password,
prvkeyPath, id });
if (res) branchList.value = res.map((i) => ({ label: i, value: i }));
}
}
