This is an automated email from the ASF dual-hosted git repository.
benjobs pushed a commit to branch dev-2.1.5
in repository https://gitbox.apache.org/repos/asf/incubator-streampark.git
The following commit(s) were added to refs/heads/dev-2.1.5 by this push:
new 4af0ea793 [Improve] git clone project improvement (#4003)
4af0ea793 is described below
commit 4af0ea793375e788d24bf39927605692506f4a59
Author: benjobs <[email protected]>
AuthorDate: Thu Aug 29 11:56:22 2024 +0800
[Improve] git clone project improvement (#4003)
* [Improve] project git clone improvement
* [Improve] apache license header minor improvement
---
.../streampark/console/base/util/GitUtils.java | 206 ++++++++++++++-------
.../console/core/controller/ProjectController.java | 4 +-
.../streampark/console/core/entity/Project.java | 36 ----
.../console/core/service/ProjectService.java | 5 +
.../core/service/impl/ProjectServiceImpl.java | 40 +++-
.../console/core/task/ProjectBuildTask.java | 25 +--
.../ProjectServiceTest.java} | 19 +-
7 files changed, 207 insertions(+), 128 deletions(-)
diff --git
a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/base/util/GitUtils.java
b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/base/util/GitUtils.java
index 02a17de35..f13c158a2 100644
---
a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/base/util/GitUtils.java
+++
b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/base/util/GitUtils.java
@@ -17,18 +17,19 @@
package org.apache.streampark.console.base.util;
-import org.apache.streampark.console.core.entity.Project;
-
import org.apache.commons.lang3.StringUtils;
import com.jcraft.jsch.JSch;
import com.jcraft.jsch.JSchException;
import com.jcraft.jsch.Session;
+import lombok.Getter;
+import lombok.Setter;
import org.eclipse.jgit.api.CloneCommand;
import org.eclipse.jgit.api.Git;
import org.eclipse.jgit.api.LsRemoteCommand;
import org.eclipse.jgit.api.TransportCommand;
import org.eclipse.jgit.api.errors.GitAPIException;
+import org.eclipse.jgit.api.errors.InvalidRemoteException;
import org.eclipse.jgit.lib.Constants;
import org.eclipse.jgit.lib.Ref;
import org.eclipse.jgit.transport.JschConfigSessionFactory;
@@ -48,81 +49,154 @@ public class GitUtils {
private GitUtils() {}
- public static Git clone(Project project) throws GitAPIException {
- CloneCommand cloneCommand =
-
Git.cloneRepository().setURI(project.getUrl()).setDirectory(project.getAppSource());
-
- if (project.getBranches() != null) {
- cloneCommand.setBranch(Constants.R_HEADS + project.getBranches());
- cloneCommand.setBranchesToClone(
- Collections.singletonList(Constants.R_HEADS +
project.getBranches()));
+ public static Git clone(GitCloneRequest request) throws GitAPIException {
+ try {
+ CloneCommand cloneCommand =
+
Git.cloneRepository().setURI(request.getUrl()).setDirectory(request.getStoreDir());
+ if (request.getBranches() != null) {
+ cloneCommand.setBranch(Constants.R_HEADS + request.getBranches());
+ cloneCommand.setBranchesToClone(
+ Collections.singletonList(Constants.R_HEADS +
request.getBranches()));
+ }
+ setCredentials(cloneCommand, request);
+ return cloneCommand.call();
+ } catch (Exception e) {
+ if (e instanceof InvalidRemoteException && request.getAuthType() ==
GitAuthType.HTTP) {
+ String url = httpUrlToSSH(request.getUrl());
+ request.setUrl(url);
+ return clone(request);
+ }
+ throw e;
}
- setCredentials(cloneCommand, project);
- return cloneCommand.call();
}
- public static List<String> getBranchList(Project project) throws
GitAPIException {
- LsRemoteCommand command =
Git.lsRemoteRepository().setRemote(project.getUrl()).setHeads(true);
- setCredentials(command, project);
- Collection<Ref> refList = command.call();
- List<String> branchList = new ArrayList<>(4);
- for (Ref ref : refList) {
- String refName = ref.getName();
- if (refName.startsWith(Constants.R_HEADS)) {
- String branchName = refName.replace(Constants.R_HEADS, "");
- branchList.add(branchName);
+ public static List<String> getBranchList(GitGetRequest request) throws
GitAPIException {
+ try {
+ LsRemoteCommand command =
Git.lsRemoteRepository().setRemote(request.getUrl()).setHeads(true);
+ setCredentials(command, request);
+ Collection<Ref> refList = command.call();
+ List<String> branchList = new ArrayList<>(4);
+ for (Ref ref : refList) {
+ String refName = ref.getName();
+ if (refName.startsWith(Constants.R_HEADS)) {
+ String branchName = refName.replace(Constants.R_HEADS, "");
+ branchList.add(branchName);
+ }
}
+ return branchList;
+ } catch (Exception e) {
+ if (e instanceof InvalidRemoteException && request.getAuthType() ==
GitAuthType.HTTP) {
+ String url = httpUrlToSSH(request.getUrl());
+ request.setUrl(url);
+ return getBranchList(request);
+ }
+ throw e;
}
- return branchList;
}
- private static void setCredentials(TransportCommand<?, ?> transportCommand,
Project project) {
- if (project.isHttpRepositoryUrl()) {
- if (!StringUtils.isAllEmpty(project.getUserName(),
project.getPassword())) {
- UsernamePasswordCredentialsProvider credentialsProvider =
- new UsernamePasswordCredentialsProvider(project.getUserName(),
project.getPassword());
- transportCommand.setCredentialsProvider(credentialsProvider);
- }
- } else if (project.isSshRepositoryUrl()) {
- transportCommand.setTransportConfigCallback(
- transport -> {
- SshTransport sshTransport = (SshTransport) transport;
- sshTransport.setSshSessionFactory(
- new JschConfigSessionFactory() {
- @Override
- protected void configure(OpenSshConfig.Host hc, Session
session) {
- session.setConfig("StrictHostKeyChecking", "no");
- }
-
- @Override
- protected JSch createDefaultJSch(FS fs) throws JSchException
{
- JSch jSch = super.createDefaultJSch(fs);
- String prvkeyPath = project.getPrvkeyPath();
- if (StringUtils.isBlank(prvkeyPath)) {
- String userHome = System.getProperty("user.home");
- if (userHome != null) {
- String rsaPath = userHome.concat("/.ssh/id_rsa");
- File resFile = new File(rsaPath);
- if (resFile.exists()) {
- prvkeyPath = rsaPath;
+ public static String httpUrlToSSH(String url) {
+ return url.replaceAll("(https://|http://)(.*?)/(.*?)/(.*?)(\\.git|)\\s*$",
"git@$2:$3/$4.git");
+ }
+
+ public static boolean isSshRepositoryUrl(String url) {
+ return url.trim().startsWith("git@");
+ }
+
+ public static boolean isHttpRepositoryUrl(String url) {
+ return !isSshRepositoryUrl(url);
+ }
+
+ private static void setCredentials(
+ TransportCommand<?, ?> transportCommand, GitAuthRequest request) {
+ switch (request.authType) {
+ case HTTP:
+ if (!StringUtils.isAllEmpty(request.getUsername(),
request.getPassword())) {
+ UsernamePasswordCredentialsProvider credentialsProvider =
+ new UsernamePasswordCredentialsProvider(request.getUsername(),
request.getPassword());
+ transportCommand.setCredentialsProvider(credentialsProvider);
+ }
+ break;
+ case SSH:
+ transportCommand.setTransportConfigCallback(
+ transport -> {
+ SshTransport sshTransport = (SshTransport) transport;
+ sshTransport.setSshSessionFactory(
+ new JschConfigSessionFactory() {
+ @Override
+ protected void configure(OpenSshConfig.Host hc, Session
session) {
+ session.setConfig("StrictHostKeyChecking", "no");
+ }
+
+ @Override
+ protected JSch createDefaultJSch(FS fs) throws
JSchException {
+ JSch jSch = super.createDefaultJSch(fs);
+ String prvkeyPath = request.getPrivateKey();
+ if (StringUtils.isBlank(prvkeyPath)) {
+ String userHome = System.getProperty("user.home");
+ if (userHome != null) {
+ String rsaPath = userHome.concat("/.ssh/id_rsa");
+ File resFile = new File(rsaPath);
+ if (resFile.exists()) {
+ prvkeyPath = rsaPath;
+ }
}
}
- }
- if (prvkeyPath == null) {
+ if (prvkeyPath == null) {
+ return jSch;
+ }
+ if (StringUtils.isEmpty(request.getPassword())) {
+ jSch.addIdentity(prvkeyPath);
+ } else {
+ jSch.addIdentity(prvkeyPath, request.getPassword());
+ }
return jSch;
}
- if (StringUtils.isEmpty(project.getPassword())) {
- jSch.addIdentity(prvkeyPath);
- } else {
- jSch.addIdentity(prvkeyPath, project.getPassword());
- }
- return jSch;
- }
- });
- });
- } else {
- throw new IllegalStateException(
- "[StreamPark] repository URL is invalid, must be ssh or http(s)");
+ });
+ });
+ break;
+ default:
+ throw new IllegalStateException(
+ "[StreamPark] repository URL is invalid, must be ssh or http(s)");
}
}
+
+ @Getter
+ public enum GitAuthType {
+ HTTP,
+ SSH
+ }
+
+ @Getter
+ @Setter
+ public static class GitAuthRequest {
+ private GitAuthType authType;
+ private String username;
+ private String password;
+ private String privateKey;
+ }
+
+ @Getter
+ @Setter
+ public static class GitGetRequest extends GitAuthRequest {
+ private String url;
+
+ public void setUrl(String url) {
+ if (StringUtils.isBlank(url)) {
+ throw new IllegalArgumentException("git url cannot be empty");
+ }
+ this.url = url;
+ if (GitUtils.isSshRepositoryUrl(url)) {
+ setAuthType(GitAuthType.SSH);
+ } else {
+ setAuthType(GitAuthType.HTTP);
+ }
+ }
+ }
+
+ @Getter
+ @Setter
+ public static class GitCloneRequest extends GitGetRequest {
+ private File storeDir;
+ private String branches;
+ }
}
diff --git
a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/controller/ProjectController.java
b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/controller/ProjectController.java
index 2fbbfa10a..accfef5a4 100644
---
a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/controller/ProjectController.java
+++
b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/controller/ProjectController.java
@@ -105,7 +105,7 @@ public class ProjectController {
@PostMapping("branches")
@PermissionScope(team = "#project.teamId")
public RestResponse branches(Project project) {
- List<String> branches = project.getAllBranches();
+ List<String> branches = projectService.getAllBranches(project);
return RestResponse.success().data(branches);
}
@@ -120,7 +120,7 @@ public class ProjectController {
@PostMapping("gitcheck")
@PermissionScope(team = "#project.teamId")
public RestResponse gitCheck(Project project) {
- GitAuthorizedError error = project.gitCheck();
+ GitAuthorizedError error = projectService.gitCheck(project);
return RestResponse.success().data(error.getType());
}
diff --git
a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/entity/Project.java
b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/entity/Project.java
index 4b2df4800..51c06f0dd 100644
---
a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/entity/Project.java
+++
b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/entity/Project.java
@@ -21,11 +21,8 @@ import org.apache.streampark.common.conf.CommonConfig;
import org.apache.streampark.common.conf.InternalConfigHolder;
import org.apache.streampark.common.conf.Workspace;
import org.apache.streampark.common.util.Utils;
-import org.apache.streampark.console.base.exception.ApiDetailException;
import org.apache.streampark.console.base.util.CommonUtils;
-import org.apache.streampark.console.base.util.GitUtils;
import org.apache.streampark.console.base.util.WebUtils;
-import org.apache.streampark.console.core.enums.GitAuthorizedError;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
@@ -47,7 +44,6 @@ import java.io.Serializable;
import java.util.Arrays;
import java.util.Date;
import java.util.Iterator;
-import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -156,30 +152,6 @@ public class Project implements Serializable {
FileUtils.deleteDirectory(getDistHome());
}
- @JsonIgnore
- public List<String> getAllBranches() {
- try {
- return GitUtils.getBranchList(this);
- } catch (Exception e) {
- throw new ApiDetailException(e);
- }
- }
-
- public GitAuthorizedError gitCheck() {
- try {
- GitUtils.getBranchList(this);
- return GitAuthorizedError.SUCCESS;
- } catch (Exception e) {
- String err = e.getMessage();
- if (err.contains("not authorized")) {
- return GitAuthorizedError.ERROR;
- } else if (err.contains("Authentication is required")) {
- return GitAuthorizedError.REQUIRED;
- }
- return GitAuthorizedError.UNKNOW;
- }
- }
-
@JsonIgnore
public boolean isCloned() {
File repository = getGitRepository();
@@ -312,12 +284,4 @@ public class Project implements Serializable {
private String getLogHeader(String header) {
return "---------------------------------[ " + header + "
]---------------------------------\n";
}
-
- public boolean isHttpRepositoryUrl() {
- return url != null && (url.trim().startsWith("https://";) ||
url.trim().startsWith("http://";));
- }
-
- public boolean isSshRepositoryUrl() {
- return url != null && url.trim().startsWith("git@");
- }
}
diff --git
a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/ProjectService.java
b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/ProjectService.java
index 647723a5d..ae4eb87d4 100644
---
a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/ProjectService.java
+++
b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/ProjectService.java
@@ -21,6 +21,7 @@ import org.apache.streampark.console.base.domain.RestRequest;
import org.apache.streampark.console.base.domain.RestResponse;
import org.apache.streampark.console.core.entity.Application;
import org.apache.streampark.console.core.entity.Project;
+import org.apache.streampark.console.core.enums.GitAuthorizedError;
import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.extension.service.IService;
@@ -57,4 +58,8 @@ public interface ProjectService extends IService<Project> {
List<Application> getApplications(Project project);
boolean checkExists(Project project);
+
+ List<String> getAllBranches(Project project);
+
+ GitAuthorizedError gitCheck(Project project);
}
diff --git
a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/ProjectServiceImpl.java
b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/ProjectServiceImpl.java
index 01312dfa9..8b9c167b6 100644
---
a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/ProjectServiceImpl.java
+++
b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/ProjectServiceImpl.java
@@ -27,14 +27,17 @@ import
org.apache.streampark.console.base.domain.ResponseCode;
import org.apache.streampark.console.base.domain.RestRequest;
import org.apache.streampark.console.base.domain.RestResponse;
import org.apache.streampark.console.base.exception.ApiAlertException;
+import org.apache.streampark.console.base.exception.ApiDetailException;
import org.apache.streampark.console.base.mybatis.pager.MybatisPager;
import org.apache.streampark.console.base.util.CommonUtils;
import org.apache.streampark.console.base.util.FileUtils;
import org.apache.streampark.console.base.util.GZipUtils;
+import org.apache.streampark.console.base.util.GitUtils;
import org.apache.streampark.console.base.util.ObjectUtils;
import org.apache.streampark.console.core.entity.Application;
import org.apache.streampark.console.core.entity.Project;
import org.apache.streampark.console.core.enums.BuildState;
+import org.apache.streampark.console.core.enums.GitAuthorizedError;
import org.apache.streampark.console.core.enums.ReleaseState;
import org.apache.streampark.console.core.mapper.ProjectMapper;
import org.apache.streampark.console.core.service.ApplicationService;
@@ -136,7 +139,7 @@ public class ProjectServiceImpl extends
ServiceImpl<ProjectMapper, Project>
project.setDescription(projectParam.getDescription());
project.setBuildArgs(projectParam.getBuildArgs());
project.setModifyTime(new Date());
- if (project.isSshRepositoryUrl()) {
+ if (GitUtils.isSshRepositoryUrl(project.getUrl())) {
project.setUserName(null);
} else {
project.setPrvkeyPath(null);
@@ -311,6 +314,41 @@ public class ProjectServiceImpl extends
ServiceImpl<ProjectMapper, Project>
return this.baseMapper.selectCount(queryWrapper) > 0;
}
+ @Override
+ public List<String> getAllBranches(Project project) {
+ try {
+ GitUtils.GitGetRequest request = new GitUtils.GitGetRequest();
+ request.setUrl(project.getUrl());
+ request.setUsername(project.getUserName());
+ request.setPassword(project.getPassword());
+ request.setPrivateKey(project.getPrvkeyPath());
+ return GitUtils.getBranchList(request);
+ } catch (Exception e) {
+ throw new ApiDetailException(e);
+ }
+ }
+
+ @Override
+ public GitAuthorizedError gitCheck(Project project) {
+ try {
+ GitUtils.GitGetRequest request = new GitUtils.GitGetRequest();
+ request.setUrl(project.getUrl());
+ request.setUsername(project.getUserName());
+ request.setPassword(project.getPassword());
+ request.setPrivateKey(project.getPrvkeyPath());
+ GitUtils.getBranchList(request);
+ return GitAuthorizedError.SUCCESS;
+ } catch (Exception e) {
+ String err = e.getMessage();
+ if (err.contains("not authorized")) {
+ return GitAuthorizedError.ERROR;
+ } else if (err.contains("Authentication is required")) {
+ return GitAuthorizedError.REQUIRED;
+ }
+ return GitAuthorizedError.UNKNOW;
+ }
+ }
+
@Override
public List<Map<String, Object>> listConf(Project project) {
try {
diff --git
a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/task/ProjectBuildTask.java
b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/task/ProjectBuildTask.java
index bfe06c444..5fe20cd13 100644
---
a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/task/ProjectBuildTask.java
+++
b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/task/ProjectBuildTask.java
@@ -26,7 +26,6 @@ import org.apache.streampark.console.core.enums.BuildState;
import ch.qos.logback.classic.Logger;
import lombok.extern.slf4j.Slf4j;
import org.eclipse.jgit.api.Git;
-import org.eclipse.jgit.api.errors.InvalidRemoteException;
import org.eclipse.jgit.lib.StoredConfig;
import java.io.File;
@@ -91,7 +90,16 @@ public class ProjectBuildTask extends AbstractLogFileTask {
project.cleanCloned();
fileLogger.info("clone {}, {} starting...", project.getName(),
project.getUrl());
fileLogger.info(project.getLog4CloneStart());
- Git git = GitUtils.clone(project);
+
+ GitUtils.GitCloneRequest request = new GitUtils.GitCloneRequest();
+ request.setUrl(project.getUrl());
+ request.setBranches(project.getBranches());
+ request.setStoreDir(project.getAppSource());
+ request.setUsername(project.getUserName());
+ request.setPassword(project.getPassword());
+ request.setPrivateKey(project.getPrvkeyPath());
+
+ Git git = GitUtils.clone(request);
StoredConfig config = git.getRepository().getConfig();
config.setBoolean("http", project.getUrl(), "sslVerify", false);
config.setBoolean("https", project.getUrl(), "sslVerify", false);
@@ -104,19 +112,6 @@ public class ProjectBuildTask extends AbstractLogFileTask {
git.close();
return true;
} catch (Exception e) {
- if (e instanceof InvalidRemoteException) {
- if (project.isHttpRepositoryUrl()) {
- String url =
- project
- .getUrl()
- .replaceAll(
- "(https://|http://)(.*?)/(.*?)/(.*?)(\\.git|)\\s*$",
"git@$2:$3/$4.git");
- project.setUrl(url);
- fileLogger.info(
- "clone project by https(http) failed, Now try to clone project
by ssh...");
- return cloneSourceCode(project);
- }
- }
fileLogger.error(
String.format(
"[StreamPark] project [%s] branch [%s] git clone failed, err:
%s",
diff --git
a/streampark-console/streampark-console-service/src/test/java/org/apache/streampark/console/core/entity/ProjectTest.java
b/streampark-console/streampark-console-service/src/test/java/org/apache/streampark/console/core/service/ProjectServiceTest.java
similarity index 71%
rename from
streampark-console/streampark-console-service/src/test/java/org/apache/streampark/console/core/entity/ProjectTest.java
rename to
streampark-console/streampark-console-service/src/test/java/org/apache/streampark/console/core/service/ProjectServiceTest.java
index ed506667a..e82a61cd9 100644
---
a/streampark-console/streampark-console-service/src/test/java/org/apache/streampark/console/core/entity/ProjectTest.java
+++
b/streampark-console/streampark-console-service/src/test/java/org/apache/streampark/console/core/service/ProjectServiceTest.java
@@ -15,38 +15,41 @@
* limitations under the License.
*/
-package org.apache.streampark.console.core.entity;
+package org.apache.streampark.console.core.service;
+import org.apache.streampark.console.SpringTestBase;
+import org.apache.streampark.console.core.entity.Project;
import org.apache.streampark.console.core.enums.GitAuthorizedError;
-import lombok.extern.slf4j.Slf4j;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
+import org.springframework.beans.factory.annotation.Autowired;
import java.util.List;
-@Slf4j
-class ProjectTest {
+public class ProjectServiceTest extends SpringTestBase {
private final Project project = new Project();
+ @Autowired private ProjectService projectService;
+
@BeforeEach
void before() {
- project.setUrl("https://github.com/apache/incubator-streampark.git";);
+ project.setUrl("[email protected]:apache/incubator-streampark.git");
}
@Disabled("This test case can't be runnable due to external service is not
available.")
@Test
void testGitBranches() {
- List<String> branches = project.getAllBranches();
+ List<String> branches = projectService.getAllBranches(project);
branches.forEach(System.out::println);
}
@Disabled("This test case can't be runnable due to external service is not
available.")
@Test
void testGitCheckAuth() {
- GitAuthorizedError error = project.gitCheck();
- log.error("{}", error);
+ GitAuthorizedError error = projectService.gitCheck(project);
+ System.out.println(error);
}
}
