This is an automated email from the ASF dual-hosted git repository.
journey pushed a commit to branch dev-resource-tree
in repository https://gitbox.apache.org/repos/asf/incubator-dolphinscheduler.git
The following commit(s) were added to refs/heads/dev-resource-tree by this push:
new 3e29b12 Fix issue #2234 and #2228 (#2246)
3e29b12 is described below
commit 3e29b12d92f5302597ecf698cfac681a186bba52
Author: lgcareer <[email protected]>
AuthorDate: Fri Mar 20 10:09:38 2020 +0800
Fix issue #2234 and #2228 (#2246)
* update resource service test
* Fix github action rerun failed
* add status of PARENT_RESOURCE_NOT_EXIST
* build resource process definition map
* update resource name also need update all the children full name
* need add queryResource
* add queryResourceJarList
* add queryResourceJarList
* add queryResourceJarList
* add resource filter in order to get filtered resource
* add comments of resource filter
* update list children by resource
* Return null if query resource list is empty
* update queryResource method change parameter pid to id
* revert .env
* remove parameter pid
* Fix issue #2234 and #2228
* change resource name with full name
---
.../api/controller/ResourcesController.java | 8 +--
.../api/dto/resources/filter/ResourceFilter.java | 15 ++---
.../api/service/ResourcesService.java | 68 ++++++++++++++--------
.../dto/resources/filter/ResourceFilterTest.java | 4 +-
.../dao/mapper/ResourceMapper.java | 3 +-
.../dao/mapper/ProcessDefinitionMapper.xml | 2 +-
.../dolphinscheduler/dao/mapper/ResourceMapper.xml | 7 +--
dolphinscheduler-ui/.env | 2 +-
8 files changed, 63 insertions(+), 46 deletions(-)
diff --git
a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java
b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java
index f9d3b1a..1d27f11 100644
---
a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java
+++
b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java
@@ -344,14 +344,14 @@ public class ResourcesController extends BaseController{
@ResponseStatus(HttpStatus.OK)
public Result queryResource(@ApiIgnore @RequestAttribute(value =
Constants.SESSION_USER) User loginUser,
@RequestParam(value ="fullName",required
= false) String fullName,
- @RequestParam(value ="pid",required =
false) Integer pid,
+ @RequestParam(value ="id",required =
false) Integer id,
@RequestParam(value ="type") ResourceType
type
) {
try {
- logger.info("login user {}, query resource by full name: {} or
pid: {},resource type: {}",
- loginUser.getUserName(), fullName,pid,type);
+ logger.info("login user {}, query resource by full name: {} or id:
{},resource type: {}",
+ loginUser.getUserName(), fullName,id,type);
- return resourceService.queryResource(fullName,pid,type);
+ return resourceService.queryResource(fullName,id,type);
} catch (Exception e) {
logger.error(RESOURCE_NOT_EXIST.getMsg(), e);
return error(Status.RESOURCE_NOT_EXIST.getCode(),
Status.RESOURCE_NOT_EXIST.getMsg());
diff --git
a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/filter/ResourceFilter.java
b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/filter/ResourceFilter.java
index 53f3d78..c918a16 100644
---
a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/filter/ResourceFilter.java
+++
b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/filter/ResourceFilter.java
@@ -38,6 +38,11 @@ public class ResourceFilter implements IFilter {
private List<Resource> resourceList;
/**
+ * parent list
+ */
+ //Set<Resource> parentList = new HashSet<>();
+
+ /**
* constructor
* @param suffix resource suffix
* @param resourceList resource list
@@ -68,8 +73,7 @@ public class ResourceFilter implements IFilter {
Set<Resource> filterFileList = fileFilter();
for(Resource file:filterFileList){
parentList.add(file);
- parentList.addAll(listAllParent(file));
- //listAllParent(file);
+ setAllParent(file,parentList);
}
return parentList;
@@ -80,16 +84,13 @@ public class ResourceFilter implements IFilter {
* @param resource resource
* @return parent resource dir set
*/
- Set<Resource> listAllParent(Resource resource){
- Set<Resource> parentList = new HashSet<>();
+ private void setAllParent(Resource resource,Set<Resource> parentList){
for (Resource resourceTemp : resourceList) {
if (resourceTemp.getId() == resource.getPid()) {
parentList.add(resourceTemp);
- listAllParent(resourceTemp);
+ setAllParent(resourceTemp,parentList);
}
}
- return parentList;
-
}
@Override
diff --git
a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java
b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java
index 3490061..6cfa3ba 100644
---
a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java
+++
b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java
@@ -336,7 +336,7 @@ public class ResourcesService extends BaseService {
// updateResource data
- List<Integer> integers = resourcesMapper.listChildren(resourceId);
+ List<Integer> childrenResource =
resourcesMapper.listChildren(resourceId);
String oldFullName = resource.getFullName();
Date now = new Date();
@@ -347,9 +347,9 @@ public class ResourcesService extends BaseService {
try {
resourcesMapper.updateById(resource);
- if (resource.isDirectory()) {
+ if (resource.isDirectory() &&
CollectionUtils.isNotEmpty(childrenResource)) {
List<Resource> childResourceList = new ArrayList<>();
- List<Resource> resourceList =
resourcesMapper.listResourceByIds(integers.toArray(new
Integer[integers.size()]));
+ List<Resource> resourceList =
resourcesMapper.listResourceByIds(childrenResource.toArray(new
Integer[childrenResource.size()]));
childResourceList = resourceList.stream().map(t -> {
t.setFullName(t.getFullName().replaceFirst(oldFullName,
fullName));
t.setUpdateTime(now);
@@ -378,8 +378,8 @@ public class ResourcesService extends BaseService {
// get file hdfs path
// delete hdfs file by type
- String originHdfsFileName =
HadoopUtils.getHdfsFileName(resource.getType(),tenantCode,originResourceName);
- String destHdfsFileName =
HadoopUtils.getHdfsFileName(resource.getType(),tenantCode,name);
+ String originHdfsFileName =
HadoopUtils.getHdfsFileName(resource.getType(),tenantCode,originFullName);
+ String destHdfsFileName =
HadoopUtils.getHdfsFileName(resource.getType(),tenantCode,fullName);
try {
if (HadoopUtils.getInstance().exists(originHdfsFileName)) {
@@ -664,26 +664,42 @@ public class ResourcesService extends BaseService {
/**
* verify resource by full name or pid and type
* @param fullName resource full name
- * @param pid parent id
+ * @param id resource id
* @param type resource type
* @return true if the resource full name or pid not exists, otherwise
return false
*/
- public Result queryResource(String fullName,Integer pid,ResourceType type)
{
+ public Result queryResource(String fullName,Integer id,ResourceType type) {
Result result = new Result();
- if (StringUtils.isBlank(fullName) && pid == null) {
+ if (StringUtils.isBlank(fullName) && id == null) {
logger.error("You must input one of fullName and pid");
putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR);
return result;
}
- List<Resource> resourceList =
resourcesMapper.queryResource(fullName,pid,type.ordinal());
- if (CollectionUtils.isEmpty(resourceList)) {
- logger.error("resource file not exist, resource full name {}",
fullName);
- putMsg(result, Status.RESOURCE_NOT_EXIST);
- return result;
+ if (StringUtils.isNotBlank(fullName)) {
+ List<Resource> resourceList =
resourcesMapper.queryResource(fullName,type.ordinal());
+ if (CollectionUtils.isEmpty(resourceList)) {
+ logger.error("resource file not exist, resource full name {}
", fullName);
+ putMsg(result, Status.RESOURCE_NOT_EXIST);
+ return result;
+ }
+ putMsg(result, Status.SUCCESS);
+ result.setData(resourceList.get(0));
+ } else {
+ Resource resource = resourcesMapper.selectById(id);
+ if (resource == null) {
+ logger.error("resource file not exist, resource id {}", id);
+ putMsg(result, Status.RESOURCE_NOT_EXIST);
+ return result;
+ }
+ Resource parentResource =
resourcesMapper.selectById(resource.getPid());
+ if (parentResource == null) {
+ logger.error("parent resource file not exist, resource id
{}", id);
+ putMsg(result, Status.RESOURCE_NOT_EXIST);
+ return result;
+ }
+ putMsg(result, Status.SUCCESS);
+ result.setData(parentResource);
}
- putMsg(result, Status.SUCCESS);
- result.setData(resourceList.get(0));
-
return result;
}
@@ -865,7 +881,7 @@ public class ResourcesService extends BaseService {
resourcesMapper.updateById(resource);
- result = uploadContentToHdfs(resource.getAlias(), tenantCode, content);
+ result = uploadContentToHdfs(resource.getFullName(), tenantCode,
content);
if (!result.getCode().equals(Status.SUCCESS.getCode())) {
throw new RuntimeException(result.getMsg());
}
@@ -1106,24 +1122,28 @@ public class ResourcesService extends BaseService {
*/
List<Integer> listAllChildren(Resource resource){
List<Integer> childList = new ArrayList<>();
- childList.add(resource.getId());
+ if (resource.getId() != -1) {
+ childList.add(resource.getId());
+ }
+
if(resource.isDirectory()){
- childList.addAll(listAllChildren(resource.getId()));
+ listAllChildren(resource.getId(),childList);
}
return childList;
}
/**
* list all children id
- * @param resourceId resource id
- * @return all children id
+ * @param resourceId resource id
+ * @param childList child list
*/
- List<Integer> listAllChildren(int resourceId){
+ void listAllChildren(int resourceId,List<Integer> childList){
+
List<Integer> children = resourcesMapper.listChildren(resourceId);
for(int chlidId:children){
- listAllChildren(chlidId);
+ childList.add(chlidId);
+ listAllChildren(chlidId,childList);
}
- return children;
}
/**
diff --git
a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/dto/resources/filter/ResourceFilterTest.java
b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/dto/resources/filter/ResourceFilterTest.java
index 955fa23..8a4a16c 100644
---
a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/dto/resources/filter/ResourceFilterTest.java
+++
b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/dto/resources/filter/ResourceFilterTest.java
@@ -39,13 +39,15 @@ public class ResourceFilterTest {
Resource resource3 = new Resource(5,3,"b1.txt","/b/b1.txt",false);
Resource resource4 = new Resource(6,3,"b2.jar","/b/b2.jar",false);
Resource resource5 = new Resource(7,-1,"b2","/b2",true);
- Resource resource6 = new Resource(8,3,"c2.jar","/b/c2.jar",true);
+ Resource resource6 = new Resource(8,-1,"b2","/b/b2",true);
+ Resource resource7 = new Resource(9,8,"c2.jar","/b/b2/c2.jar",false);
allList.add(resource1);
allList.add(resource2);
allList.add(resource3);
allList.add(resource4);
allList.add(resource5);
allList.add(resource6);
+ allList.add(resource7);
ResourceFilter resourceFilter = new ResourceFilter(".jar",allList);
diff --git
a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ResourceMapper.java
b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ResourceMapper.java
index b7b4447..f07a92c 100644
---
a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ResourceMapper.java
+++
b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ResourceMapper.java
@@ -118,11 +118,10 @@ public interface ResourceMapper extends
BaseMapper<Resource> {
/**
* query resource by full name or pid
* @param fullName full name
- * @param pid parent id
* @param type resource type
* @return resource
*/
- List<Resource> queryResource(@Param("fullName") String
fullName,@Param("pid") Integer pid,@Param("type") int type);
+ List<Resource> queryResource(@Param("fullName") String
fullName,@Param("type") int type);
/**
* list resource by id array
diff --git
a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapper.xml
b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapper.xml
index 377de30..c9086b9 100644
---
a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapper.xml
+++
b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapper.xml
@@ -92,6 +92,6 @@
<select id="listResources" resultType="java.util.HashMap">
SELECT id,resource_ids
FROM t_ds_process_definition
- WHERE release_state = 1 and resource_ids is not null
+ WHERE release_state = 1 and resource_ids is not null and resource_ids
!= ''
</select>
</mapper>
\ No newline at end of file
diff --git
a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ResourceMapper.xml
b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ResourceMapper.xml
index 57383ab..46fef9e 100644
---
a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ResourceMapper.xml
+++
b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ResourceMapper.xml
@@ -116,12 +116,7 @@
select *
from t_ds_resources
where type = #{type}
- <if test="fullName != null and fullName != ''">
- and full_name = #{fullName}
- </if>
- <if test="pid != null and pid != 0">
- and pid = #{pid}
- </if>
+ and full_name = #{fullName}
</select>
<update id="batchUpdateResource" parameterType="java.util.List">
diff --git a/dolphinscheduler-ui/.env b/dolphinscheduler-ui/.env
index 4c7e96e..e676be6 100644
--- a/dolphinscheduler-ui/.env
+++ b/dolphinscheduler-ui/.env
@@ -17,4 +17,4 @@
API_BASE = http://192.168.xx.xx:12345
# If IP access is required for local development, remove the "#"
-#DEV_HOST = 192.168.xx.xx
\ No newline at end of file
+#DEV_HOST = 192.168.xx.xx
