This is an automated email from the ASF dual-hosted git repository.
leonbao pushed a commit to branch dev-resource-tree
in repository https://gitbox.apache.org/repos/asf/incubator-dolphinscheduler.git
The following commit(s) were added to refs/heads/dev-resource-tree by this push:
new 1a4963e Fix github action rerun failed (#2067)
1a4963e is described below
commit 1a4963e2cb6f9ab67df178080616375106b7d6b6
Author: lgcareer <[email protected]>
AuthorDate: Tue Mar 3 16:58:10 2020 +0800
Fix github action rerun failed (#2067)
* update resource service test
* Fix github action rerun failed
* add status of PARENT_RESOURCE_NOT_EXIST
---
.github/workflows/ci_backend.yml | 6 +-
.github/workflows/ci_frontend.yml | 4 +-
.github/workflows/ci_ut.yml | 10 ++--
.../apache/dolphinscheduler/api/enums/Status.java | 1 +
.../api/service/ResourcesService.java | 14 ++++-
.../api/service/ResourcesServiceTest.java | 66 ++++++++++++++++------
6 files changed, 72 insertions(+), 29 deletions(-)
diff --git a/.github/workflows/ci_backend.yml b/.github/workflows/ci_backend.yml
index e527c3c..1ca15c2 100644
--- a/.github/workflows/ci_backend.yml
+++ b/.github/workflows/ci_backend.yml
@@ -45,17 +45,17 @@ jobs:
Compile-check:
runs-on: ubuntu-latest
steps:
- - uses: actions/checkout@v1
+ - uses: actions/checkout@v2
- name: Set up JDK 1.8
uses: actions/setup-java@v1
with:
java-version: 1.8
- name: Compile
- run: mvn -U -B -T 1C clean install -Prelease -Dmaven.compile.fork=true
-Dmaven.test.skip=true
+ run: mvn -B clean compile package -Prelease -Dmaven.test.skip=true
License-check:
runs-on: ubuntu-latest
steps:
- - uses: actions/checkout@v1
+ - uses: actions/checkout@v2
- name: Set up JDK 1.8
uses: actions/setup-java@v1
with:
diff --git a/.github/workflows/ci_frontend.yml
b/.github/workflows/ci_frontend.yml
index fab75c6..1f959b4 100644
--- a/.github/workflows/ci_frontend.yml
+++ b/.github/workflows/ci_frontend.yml
@@ -34,7 +34,7 @@ jobs:
matrix:
os: [ubuntu-latest, macos-latest]
steps:
- - uses: actions/checkout@v1
+ - uses: actions/checkout@v2
- name: Set up Node.js
uses: actions/setup-node@v1
with:
@@ -49,7 +49,7 @@ jobs:
License-check:
runs-on: ubuntu-latest
steps:
- - uses: actions/checkout@v1
+ - uses: actions/checkout@v2
- name: Set up JDK 1.8
uses: actions/setup-java@v1
with:
diff --git a/.github/workflows/ci_ut.yml b/.github/workflows/ci_ut.yml
index 1166029..13993ca 100644
--- a/.github/workflows/ci_ut.yml
+++ b/.github/workflows/ci_ut.yml
@@ -20,7 +20,7 @@ env:
DOCKER_DIR: ./docker
LOG_DIR: /tmp/dolphinscheduler
-name: Test Coveralls Parallel
+name: Unit Test
jobs:
@@ -29,9 +29,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- - uses: actions/checkout@v1
- with:
- submodules: true
+ - uses: actions/checkout@v2
- uses: actions/cache@v1
with:
path: ~/.m2/repository
@@ -47,13 +45,13 @@ jobs:
- name: Compile
run: |
export MAVEN_OPTS='-Dmaven.repo.local=.m2/repository
-XX:+TieredCompilation -XX:TieredStopAtLevel=1 -XX:+CMSClassUnloadingEnabled
-XX:+UseConcMarkSweepGC -XX:-UseGCOverheadLimit -Xmx3g'
- mvn test -B -Dmaven.test.skip=false cobertura:cobertura
+ mvn test -B -Dmaven.test.skip=false
CODECOV_TOKEN="09c2663f-b091-4258-8a47-c981827eb29a" bash <(curl -s
https://codecov.io/bash)
- name: Run SonarCloud Analysis
run: >
mvn verify --batch-mode
org.sonarsource.scanner.maven:sonar-maven-plugin:3.6.1.1688:sonar
- -Dsonar.junit.reportPaths=target/cobertura
+ -Dsonar.coverage.jacoco.xmlReportPaths=target/site/jacoco/jacoco.xml
-Dmaven.test.skip=true
-Dsonar.host.url=https://sonarcloud.io
-Dsonar.organization=apache
diff --git
a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java
b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java
index 8d785e5..b85747e 100644
---
a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java
+++
b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java
@@ -185,6 +185,7 @@ public enum Status {
RESOURCE_FILE_NOT_EXIST(20012, "resource file {0} not exists in hdfs!"),
UDF_RESOURCE_IS_BOUND(20013, "udf resource file is bound by UDF
functions:{0}"),
RESOURCE_IS_USED(20014, "resource file is used by process definition"),
+ PARENT_RESOURCE_NOT_EXIST(20015, "resource not exist"),
diff --git
a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java
b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java
index 095f532..b1dbed2 100644
---
a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java
+++
b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java
@@ -91,13 +91,19 @@ public class ResourcesService extends BaseService {
int pid,
String currentDir) {
Result result = new Result();
+ // if hdfs not startup
+ if (!PropertyUtils.getResUploadStartupState()){
+ logger.error("resource upload startup state: {}",
PropertyUtils.getResUploadStartupState());
+ putMsg(result, Status.HDFS_NOT_STARTUP);
+ return result;
+ }
String fullName = currentDir.equals("/") ?
String.format("%s%s",currentDir,name):String.format("%s/%s",currentDir,name);
if (pid != -1) {
Resource parentResource = resourcesMapper.selectById(pid);
if (parentResource == null) {
- putMsg(result, Status.RESOURCE_NOT_EXIST);
+ putMsg(result, Status.PARENT_RESOURCE_NOT_EXIST);
return result;
}
@@ -406,6 +412,10 @@ public class ResourcesService extends BaseService {
/**
* create direcoty
+ * @param loginUser login user
+ * @param fullName full name
+ * @param type resource type
+ * @param result Result
*/
private void createDirecotry(User loginUser,String fullName,ResourceType
type,Result result){
// query tenant
@@ -554,7 +564,7 @@ public class ResourcesService extends BaseService {
resourceIdSet.retainAll(allChildren);
if (CollectionUtils.isNotEmpty(resourceIdSet)) {
logger.error("can't be deleted,because it is used of process
definition");
- putMsg(result, Status.USER_NO_OPERATION_PERM);
+ putMsg(result, Status.RESOURCE_IS_USED);
return result;
}
diff --git
a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ResourcesServiceTest.java
b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ResourcesServiceTest.java
index 73fa903..c735317 100644
---
a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ResourcesServiceTest.java
+++
b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ResourcesServiceTest.java
@@ -71,6 +71,8 @@ public class ResourcesServiceTest {
private UserMapper userMapper;
@Mock
private UdfFuncMapper udfFunctionMapper;
+ @Mock
+ private ProcessDefinitionMapper processDefinitionMapper;
@Before
public void setUp() {
@@ -121,19 +123,30 @@ public class ResourcesServiceTest {
Assert.assertEquals(Status.UDF_RESOURCE_SUFFIX_NOT_JAR.getMsg(),result.getMsg());
- //UDF_RESOURCE_SUFFIX_NOT_JAR
- Mockito.when(tenantMapper.queryById(0)).thenReturn(getTenant());
-
Mockito.when(resourcesMapper.queryResourceList("ResourcesServiceTest.jar", 0,
1)).thenReturn(getResourceList());
- mockMultipartFile = new
MockMultipartFile("ResourcesServiceTest.jar","ResourcesServiceTest.jar","pdf",new
String("test").getBytes());
- result =
resourcesService.createResource(user,"ResourcesServiceTest.jar","ResourcesServiceTest",ResourceType.UDF,mockMultipartFile,-1,"/");
+ }
+
+ @Test
+ public void testCreateDirecotry(){
+
+
PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(false);
+ User user = new User();
+ //HDFS_NOT_STARTUP
+ Result result =
resourcesService.createDirectory(user,"directoryTest","directory
test",ResourceType.FILE,-1,"/");
logger.info(result.toString());
- Assert.assertEquals(Status.RESOURCE_EXIST.getMsg(),result.getMsg());
+ Assert.assertEquals(Status.HDFS_NOT_STARTUP.getMsg(),result.getMsg());
- //SUCCESS
-
Mockito.when(resourcesMapper.queryResourceList("ResourcesServiceTest.jar", 0,
1)).thenReturn(new ArrayList<>());
- result =
resourcesService.createResource(user,"ResourcesServiceTest.jar","ResourcesServiceTest",ResourceType.UDF,mockMultipartFile,-1,"/");
+ //PARENT_RESOURCE_NOT_EXIST
+
PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true);
+
Mockito.when(resourcesMapper.selectById(Mockito.anyInt())).thenReturn(null);
+ result =
resourcesService.createDirectory(user,"directoryTest","directory
test",ResourceType.FILE,1,"/");
logger.info(result.toString());
- Assert.assertEquals(Status.SUCCESS.getMsg(),result.getMsg());
+
Assert.assertEquals(Status.RESOURCE_NOT_EXIST.getMsg(),result.getMsg());
+ //RESOURCE_EXIST
+
PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true);
+ Mockito.when(resourcesMapper.queryResourceList("/directoryTest", 0,
0)).thenReturn(getResourceList());
+ result =
resourcesService.createDirectory(user,"directoryTest","directory
test",ResourceType.FILE,-1,"/");
+ logger.info(result.toString());
+ Assert.assertEquals(Status.RESOURCE_EXIST.getMsg(),result.getMsg());
}
@@ -174,12 +187,14 @@ public class ResourcesServiceTest {
logger.info(result.toString());
Assert.assertEquals(Status.RESOURCE_EXIST.getMsg(),result.getMsg());
//USER_NOT_EXIST
+
Mockito.when(userMapper.queryDetailsById(Mockito.anyInt())).thenReturn(null);
result =
resourcesService.updateResource(user,1,"ResourcesServiceTest1.jar","ResourcesServiceTest",ResourceType.UDF);
logger.info(result.toString());
Assert.assertTrue(Status.USER_NOT_EXIST.getCode() == result.getCode());
//TENANT_NOT_EXIST
Mockito.when(userMapper.queryDetailsById(1)).thenReturn(getUser());
+
Mockito.when(tenantMapper.queryById(Mockito.anyInt())).thenReturn(null);
result =
resourcesService.updateResource(user,1,"ResourcesServiceTest1.jar","ResourcesServiceTest",ResourceType.UDF);
logger.info(result.toString());
Assert.assertEquals(Status.TENANT_NOT_EXIST.getMsg(),result.getMsg());
@@ -199,7 +214,7 @@ public class ResourcesServiceTest {
//SUCCESS
PowerMockito.when(HadoopUtils.getHdfsResourceFileName(Mockito.any(),
Mockito.any())).thenReturn("test");
- result =
resourcesService.updateResource(user,1,"ResourcesServiceTest1.jar","ResourcesServiceTest",ResourceType.UDF);
+ result =
resourcesService.updateResource(user,1,"ResourcesServiceTest1.jar","ResourcesServiceTest1.jar",ResourceType.UDF);
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS.getMsg(),result.getMsg());
@@ -264,6 +279,7 @@ public class ResourcesServiceTest {
//TENANT_NOT_EXIST
loginUser.setUserType(UserType.ADMIN_USER);
loginUser.setTenantId(2);
+
Mockito.when(userMapper.queryDetailsById(Mockito.anyInt())).thenReturn(loginUser);
result = resourcesService.delete(loginUser,1);
logger.info(result.toString());
Assert.assertEquals(Status.TENANT_NOT_EXIST.getMsg(),
result.getMsg());
@@ -286,14 +302,20 @@ public class ResourcesServiceTest {
User user = new User();
user.setId(1);
- Mockito.when(resourcesMapper.queryResourceList("test", 0,
0)).thenReturn(getResourceList());
- Result result =
resourcesService.verifyResourceName("test",ResourceType.FILE,user);
+
Mockito.when(resourcesMapper.queryResourceList("/ResourcesServiceTest.jar", 0,
0)).thenReturn(getResourceList());
+ Result result =
resourcesService.verifyResourceName("/ResourcesServiceTest.jar",ResourceType.FILE,user);
logger.info(result.toString());
Assert.assertEquals(Status.RESOURCE_EXIST.getMsg(), result.getMsg());
//TENANT_NOT_EXIST
Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant());
- result =
resourcesService.verifyResourceName("test1",ResourceType.FILE,user);
+ String unExistFullName = "/test.jar";
+ try {
+
Mockito.when(hadoopUtils.exists(unExistFullName)).thenReturn(false);
+ } catch (IOException e) {
+ logger.error("hadoop error",e);
+ }
+ result =
resourcesService.verifyResourceName("/test.jar",ResourceType.FILE,user);
logger.info(result.toString());
Assert.assertEquals(Status.TENANT_NOT_EXIST.getMsg(), result.getMsg());
@@ -306,9 +328,9 @@ public class ResourcesServiceTest {
logger.error("hadoop error",e);
}
PowerMockito.when(HadoopUtils.getHdfsResourceFileName("123",
"test1")).thenReturn("test");
- result =
resourcesService.verifyResourceName("test1",ResourceType.FILE,user);
+ result =
resourcesService.verifyResourceName("/ResourcesServiceTest.jar",ResourceType.FILE,user);
logger.info(result.toString());
-
Assert.assertTrue(Status.RESOURCE_FILE_EXIST.getCode()==result.getCode());
+ Assert.assertTrue(Status.RESOURCE_EXIST.getCode()==result.getCode());
//SUCCESS
result =
resourcesService.verifyResourceName("test2",ResourceType.FILE,user);
@@ -585,6 +607,7 @@ public class ResourcesServiceTest {
private Resource getResource(){
Resource resource = new Resource();
+ resource.setPid(-1);
resource.setUserId(1);
resource.setDescription("ResourcesServiceTest.jar");
resource.setAlias("ResourcesServiceTest.jar");
@@ -593,6 +616,17 @@ public class ResourcesServiceTest {
return resource;
}
+ private Resource getUdfResource(){
+
+ Resource resource = new Resource();
+ resource.setUserId(1);
+ resource.setDescription("udfTest");
+ resource.setAlias("udfTest.jar");
+ resource.setFullName("/udfTest.jar");
+ resource.setType(ResourceType.UDF);
+ return resource;
+ }
+
private UdfFunc getUdfFunc(){
UdfFunc udfFunc = new UdfFunc();
