This is an automated email from the ASF dual-hosted git repository.
kirs pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/incubator-dolphinscheduler.git
The following commit(s) were added to refs/heads/dev by this push:
new ba039dc [Improvement][dao] Resource features SQL slow query
optimization. (#4947)
ba039dc is described below
commit ba039dc252a2ddfe316297a2955f0ea9f966432a
Author: zhuangchong <[email protected]>
AuthorDate: Sat Mar 20 12:50:26 2021 +0800
[Improvement][dao] Resource features SQL slow query optimization. (#4947)
* Resourse features SQL slow query optimization.
* update resource mapper queryResourcePaging method.
* split resource and resource_uesr sql statement.
* determine the size of the collection.
* update process service class code style.
* update ResourceUserMapper java class code style.
* add resourceUserMapper in the test config file.
* solve code smell
* split paging query join statement.
* update ResourceUserMapper class.
* remove sql join statement.
* update resource test class code style.
* update ResourcesServiceTest test class.
* rerun ut.
* update user service test class.
* solve query resource list method code smell.
---
.../api/service/impl/ResourcesServiceImpl.java | 66 +++++++----
.../api/service/impl/UsersServiceImpl.java | 3 +-
.../api/service/ResourcesServiceTest.java | 11 +-
.../api/service/UsersServiceTest.java | 1 -
.../dao/mapper/ResourceMapper.java | 43 +++----
.../dao/mapper/ResourceUserMapper.java | 16 ++-
.../dolphinscheduler/dao/mapper/ResourceMapper.xml | 123 +++++++++++----------
.../dao/mapper/ResourceUserMapper.xml | 14 +++
.../dao/mapper/ResourceMapperTest.java | 60 +++++-----
.../server/registry/DependencyConfig.java | 6 +
.../processor/TaskCallbackServiceTestConfig.java | 6 +
.../service/process/ProcessService.java | 39 ++++++-
12 files changed, 252 insertions(+), 136 deletions(-)
diff --git
a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java
b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java
index f4da1dc..4b7cc59 100644
---
a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java
+++
b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java
@@ -529,8 +529,10 @@ public class ResourcesServiceImpl extends BaseServiceImpl
implements ResourcesSe
}
}
- IPage<Resource> resourceIPage =
resourcesMapper.queryResourcePaging(page,
- userId,directoryId, type.ordinal(), searchVal);
+ List<Integer> resourcesIds =
resourceUserMapper.queryResourcesIdListByUserIdAndPerm(userId, 0);
+
+ IPage<Resource> resourceIPage =
resourcesMapper.queryResourcePaging(page, userId, directoryId, type.ordinal(),
searchVal,resourcesIds);
+
PageInfo<Resource> pageInfo = new PageInfo<>(pageNo, pageSize);
pageInfo.setTotalCount((int)resourceIPage.getTotal());
pageInfo.setLists(resourceIPage.getRecords());
@@ -615,15 +617,10 @@ public class ResourcesServiceImpl extends BaseServiceImpl
implements ResourcesSe
@Override
public Map<String, Object> queryResourceList(User loginUser, ResourceType
type) {
Map<String, Object> result = new HashMap<>();
-
- int userId = loginUser.getId();
- if (isAdmin(loginUser)) {
- userId = 0;
- }
- List<Resource> allResourceList =
resourcesMapper.queryResourceListAuthored(userId, type.ordinal(),0);
+ List<Resource> allResourceList = queryAuthoredResourceList(loginUser,
type);
Visitor resourceTreeVisitor = new ResourceTreeVisitor(allResourceList);
result.put(Constants.DATA_LIST,
resourceTreeVisitor.visit().getChildren());
- putMsg(result,Status.SUCCESS);
+ putMsg(result, Status.SUCCESS);
return result;
}
@@ -638,11 +635,10 @@ public class ResourcesServiceImpl extends BaseServiceImpl
implements ResourcesSe
@Override
public Map<String, Object> queryResourceByProgramType(User loginUser,
ResourceType type, ProgramType programType) {
Map<String, Object> result = new HashMap<>();
+
+ List<Resource> allResourceList = queryAuthoredResourceList(loginUser,
type);
+
String suffix = ".jar";
- int userId = loginUser.getId();
- if (isAdmin(loginUser)) {
- userId = 0;
- }
if (programType != null) {
switch (programType) {
case JAVA:
@@ -654,11 +650,10 @@ public class ResourcesServiceImpl extends BaseServiceImpl
implements ResourcesSe
default:
}
}
- List<Resource> allResourceList =
resourcesMapper.queryResourceListAuthored(userId, type.ordinal(),0);
- List<Resource> resources = new ResourceFilter(suffix,new
ArrayList<>(allResourceList)).filter();
+ List<Resource> resources = new ResourceFilter(suffix, new
ArrayList<>(allResourceList)).filter();
Visitor resourceTreeVisitor = new ResourceTreeVisitor(resources);
result.put(Constants.DATA_LIST,
resourceTreeVisitor.visit().getChildren());
- putMsg(result,Status.SUCCESS);
+ putMsg(result, Status.SUCCESS);
return result;
}
@@ -1173,8 +1168,7 @@ public class ResourcesServiceImpl extends BaseServiceImpl
implements ResourcesSe
List<Resource> list;
if (resourceList != null && !resourceList.isEmpty()) {
Set<Resource> resourceSet = new HashSet<>(resourceList);
- List<Resource> authedResourceList =
resourcesMapper.queryAuthorizedResourceList(userId);
-
+ List<Resource> authedResourceList = queryResourceList(userId,
Constants.AUTHORIZE_WRITABLE_PERM);
getAuthorizedResourceList(resourceSet, authedResourceList);
list = new ArrayList<>(resourceSet);
} else {
@@ -1249,7 +1243,7 @@ public class ResourcesServiceImpl extends BaseServiceImpl
implements ResourcesSe
if (isNotAdmin(loginUser, result)) {
return result;
}
- List<Resource> authedResources =
resourcesMapper.queryAuthorizedResourceList(userId);
+ List<Resource> authedResources = queryResourceList(userId,
Constants.AUTHORIZE_WRITABLE_PERM);
Visitor visitor = new ResourceTreeVisitor(authedResources);
String visit = JSONUtils.toJsonString(visitor.visit(),
SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS);
logger.info(visit);
@@ -1329,4 +1323,38 @@ public class ResourcesServiceImpl extends
BaseServiceImpl implements ResourcesSe
}
}
+ /**
+ * query authored resource list (own and authorized)
+ * @param loginUser login user
+ * @param type ResourceType
+ * @return all authored resource list
+ */
+ private List<Resource> queryAuthoredResourceList(User loginUser,
ResourceType type) {
+ List<Resource> relationResources;
+ int userId = loginUser.getId();
+ if (isAdmin(loginUser)) {
+ userId = 0;
+ relationResources = new ArrayList<>();
+ } else {
+ // query resource relation
+ relationResources = queryResourceList(userId, 0);
+ }
+
+ List<Resource> ownResourceList =
resourcesMapper.queryResourceListAuthored(userId, type.ordinal());
+ ownResourceList.addAll(relationResources);
+
+ return ownResourceList;
+ }
+
+ /**
+ * query resource list by userId and perm
+ * @param userId userId
+ * @param perm perm
+ * @return resource list
+ */
+ private List<Resource> queryResourceList(Integer userId, int perm) {
+ List<Integer> resIds =
resourceUserMapper.queryResourcesIdListByUserIdAndPerm(userId, perm);
+ return CollectionUtils.isEmpty(resIds) ? new ArrayList<>() :
resourcesMapper.queryResourceListById(resIds);
+ }
+
}
diff --git
a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UsersServiceImpl.java
b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UsersServiceImpl.java
index 865f4c3..62561ae 100644
---
a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UsersServiceImpl.java
+++
b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UsersServiceImpl.java
@@ -586,7 +586,8 @@ public class UsersServiceImpl extends BaseServiceImpl
implements UsersService {
}
//get the authorized resource id list by user id
- List<Resource> oldAuthorizedRes =
resourceMapper.queryAuthorizedResourceList(userId);
+ List<Integer> resIds =
resourceUserMapper.queryResourcesIdListByUserIdAndPerm(userId,
Constants.AUTHORIZE_WRITABLE_PERM);
+ List<Resource> oldAuthorizedRes = CollectionUtils.isEmpty(resIds) ?
new ArrayList<>() : resourceMapper.queryResourceListById(resIds);
//if resource type is UDF,need check whether it is bound by UDF
function
Set<Integer> oldAuthorizedResIds =
oldAuthorizedRes.stream().map(Resource::getId).collect(Collectors.toSet());
diff --git
a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ResourcesServiceTest.java
b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ResourcesServiceTest.java
index 89bd3df..931a419 100644
---
a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ResourcesServiceTest.java
+++
b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ResourcesServiceTest.java
@@ -266,8 +266,9 @@ public class ResourcesServiceTest {
IPage<Resource> resourcePage = new Page<>(1, 10);
resourcePage.setTotal(1);
resourcePage.setRecords(getResourceList());
+
Mockito.when(resourcesMapper.queryResourcePaging(Mockito.any(Page.class),
- Mockito.eq(0), Mockito.eq(-1), Mockito.eq(0),
Mockito.eq("test"))).thenReturn(resourcePage);
+ Mockito.eq(0), Mockito.eq(-1), Mockito.eq(0),
Mockito.eq("test"), Mockito.any())).thenReturn(resourcePage);
Map<String, Object> result =
resourcesService.queryResourceListPaging(loginUser, -1, ResourceType.FILE,
"test", 1, 10);
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS));
@@ -281,7 +282,7 @@ public class ResourcesServiceTest {
User loginUser = new User();
loginUser.setId(0);
loginUser.setUserType(UserType.ADMIN_USER);
- Mockito.when(resourcesMapper.queryResourceListAuthored(0, 0,
0)).thenReturn(getResourceList());
+ Mockito.when(resourcesMapper.queryResourceListAuthored(0,
0)).thenReturn(getResourceList());
Map<String, Object> result =
resourcesService.queryResourceList(loginUser, ResourceType.FILE);
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS));
@@ -615,7 +616,11 @@ public class ResourcesServiceTest {
Assert.assertEquals(Status.USER_NO_OPERATION_PERM,
result.get(Constants.STATUS));
//SUCCESS
user.setUserType(UserType.ADMIN_USER);
-
Mockito.when(resourcesMapper.queryAuthorizedResourceList(1)).thenReturn(getResourceList());
+
+ List<Integer> resIds = new ArrayList<>();
+ resIds.add(1);
+
Mockito.when(resourceUserMapper.queryResourcesIdListByUserIdAndPerm(Mockito.anyInt(),
Mockito.anyInt())).thenReturn(resIds);
+
Mockito.when(resourcesMapper.queryResourceListById(Mockito.any())).thenReturn(getResourceList());
result = resourcesService.authorizedFile(user, 1);
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS));
diff --git
a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UsersServiceTest.java
b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UsersServiceTest.java
index 39b8cda..e8707fc 100644
---
a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UsersServiceTest.java
+++
b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UsersServiceTest.java
@@ -325,7 +325,6 @@ public class UsersServiceTest {
logger.info(result.toString());
Assert.assertEquals(Status.USER_NOT_EXIST,
result.get(Constants.STATUS));
//success
- when(resourceMapper.queryAuthorizedResourceList(1)).thenReturn(new
ArrayList<Resource>());
when(resourceMapper.selectById(Mockito.anyInt())).thenReturn(getResource());
when(resourceUserMapper.deleteResourceUser(1, 0)).thenReturn(1);
result = usersService.grantResources(loginUser, 1, resourceIds);
diff --git
a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ResourceMapper.java
b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ResourceMapper.java
index 96b2039..8fbd9a3 100644
---
a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ResourceMapper.java
+++
b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ResourceMapper.java
@@ -14,15 +14,18 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.dolphinscheduler.dao.mapper;
import org.apache.dolphinscheduler.dao.entity.Resource;
-import com.baomidou.mybatisplus.core.mapper.BaseMapper;
-import com.baomidou.mybatisplus.core.metadata.IPage;
+
import org.apache.ibatis.annotations.Param;
import java.util.List;
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+import com.baomidou.mybatisplus.core.metadata.IPage;
+
/**
* resource mapper interface
*/
@@ -43,14 +46,10 @@ public interface ResourceMapper extends
BaseMapper<Resource> {
* query resource list
* @param userId userId
* @param type type
- * @param perm perm
* @return resource list
*/
- List<Resource> queryResourceListAuthored(
- @Param("userId") int userId,
- @Param("type") int type,
- @Param("perm") int perm);
-
+ List<Resource> queryResourceListAuthored(@Param("userId") int userId,
+ @Param("type") int type);
/**
* resource page
@@ -59,20 +58,15 @@ public interface ResourceMapper extends
BaseMapper<Resource> {
* @param id id
* @param type type
* @param searchVal searchVal
+ * @param resIds resIds
* @return resource page
*/
IPage<Resource> queryResourcePaging(IPage<Resource> page,
@Param("userId") int userId,
@Param("id") int id,
@Param("type") int type,
- @Param("searchVal") String searchVal);
-
- /**
- * query Authed resource list
- * @param userId userId
- * @return resource list
- */
- List<Resource> queryAuthorizedResourceList(@Param("userId") int userId);
+ @Param("searchVal") String searchVal,
+ @Param("resIds") List<Integer> resIds);
/**
* query resource except userId
@@ -82,23 +76,20 @@ public interface ResourceMapper extends
BaseMapper<Resource> {
List<Resource> queryResourceExceptUserId(@Param("userId") int userId);
/**
- * query tenant code by name
- * @param resName resource name
- * @param resType resource type
- * @return tenant code
- */
- String queryTenantCodeByResourceName(@Param("resName") String
resName,@Param("resType") int resType);
-
- /**
* list authorized resource
* @param userId userId
* @param resNames resNames
* @param <T> T
* @return resource list
*/
- <T> List<Resource> listAuthorizedResource(@Param("userId") int
userId,@Param("resNames")T[] resNames);
-
+ <T> List<Resource> listAuthorizedResource(@Param("userId") int userId,
@Param("resNames") T[] resNames);
+ /**
+ * list resources by id
+ * @param resIds resIds
+ * @return resource list
+ */
+ List<Resource> queryResourceListById(@Param("resIds") List<Integer>
resIds);
/**
* list authorized resource
diff --git
a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ResourceUserMapper.java
b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ResourceUserMapper.java
index 176f7d8..dbaca25 100644
---
a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ResourceUserMapper.java
+++
b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ResourceUserMapper.java
@@ -14,18 +14,32 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.dolphinscheduler.dao.mapper;
import org.apache.dolphinscheduler.dao.entity.ResourcesUser;
-import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+
import org.apache.ibatis.annotations.Param;
+import java.util.List;
+
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+
/**
* resource user relation mapper interface
*/
public interface ResourceUserMapper extends BaseMapper<ResourcesUser> {
/**
+ * query resourcesId list by userId and perm
+ * @param userId userId
+ * @param perm perm
+ * @return resourcesId list result
+ */
+ List<Integer> queryResourcesIdListByUserIdAndPerm(@Param("userId") int
userId,
+ @Param("perm") int perm);
+
+ /**
* delete resource user relation
* @param userId userId
* @param resourceId resourceId
diff --git
a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ResourceMapper.xml
b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ResourceMapper.xml
index c67fd72..7a8b043 100644
---
a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ResourceMapper.xml
+++
b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ResourceMapper.xml
@@ -18,90 +18,82 @@
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
"http://mybatis.org/dtd/mybatis-3-mapper.dtd"; >
<mapper namespace="org.apache.dolphinscheduler.dao.mapper.ResourceMapper">
- <sql id="baseSql">
- id, alias, file_name, description, user_id, type, size, create_time,
update_time,
- pid, full_name, is_directory
- </sql>
<sql id="baseSqlV2">
${alias}.id, ${alias}.alias, ${alias}.file_name, ${alias}.description,
${alias}.user_id, ${alias}.type, ${alias}.size, ${alias}.create_time,
${alias}.update_time,
${alias}.pid, ${alias}.full_name, ${alias}.is_directory
</sql>
<select id="queryResourceList"
resultType="org.apache.dolphinscheduler.dao.entity.Resource">
select
- <include refid="baseSql"/>
- from t_ds_resources
+ <include refid="baseSqlV2">
+ <property name="alias" value="r"/>
+ </include>
+ from t_ds_resources r
where 1= 1
<if test="fullName != null and fullName != ''">
- and full_name = #{fullName}
+ and r.full_name = #{fullName}
</if>
<if test="type != -1">
- and type = #{type}
+ and r.type = #{type}
</if>
<if test="userId != 0">
- and user_id = #{userId}
+ and r.user_id = #{userId}
</if>
</select>
<select id="queryResourceListAuthored"
resultType="org.apache.dolphinscheduler.dao.entity.Resource">
select
- <include refid="baseSql"/>
- from t_ds_resources
+ <include refid="baseSqlV2">
+ <property name="alias" value="r"/>
+ </include>
+ from t_ds_resources r
where 1 = 1
<if test="type != -1">
- and type=#{type}
+ and r.type=#{type}
</if>
- <if test="userId != 0 and perm != 0">
- and id in (select resources_id from t_ds_relation_resources_user
where user_id=#{userId} and perm=#{perm}
- union select id as resources_id from t_ds_resources where
user_id=#{userId})
- </if>
- <if test="userId != 0 and perm == 0">
- and id in (select resources_id from t_ds_relation_resources_user
where user_id=#{userId}
- union select id as resources_id from t_ds_resources where
user_id=#{userId})
+ <if test="userId != 0">
+ and r.user_id=#{userId}
</if>
</select>
<select id="queryResourcePaging"
resultType="org.apache.dolphinscheduler.dao.entity.Resource">
select
- d.id, d.alias, d.file_name, d.description, d.user_id, d.type, d.size,
d.create_time, d.update_time,
- d.pid, d.full_name, d.is_directory,
- u.user_name as user_name
+ <include refid="baseSqlV2">
+ <property name="alias" value="d"/>
+ </include>
from t_ds_resources d
- join t_ds_user u on d.user_id = u.id
where d.type=#{type} and d.pid=#{id}
<if test="userId != 0">
- and d.id in (select resources_id from t_ds_relation_resources_user
where user_id=#{userId}
- union select id as resources_id from t_ds_resources where
user_id=#{userId})
+ and (
+ <if test="resIds != null and resIds.size() > 0">
+ and d.id in
+ <foreach collection="resIds" item="i" open="(" close=") or"
separator=",">
+ #{i}
+ </foreach>
+ </if>
+ d.user_id=#{userId} )
</if>
<if test="searchVal != null and searchVal != ''">
and d.alias like concat('%', #{searchVal}, '%')
</if>
order by d.update_time desc
</select>
- <select id="queryAuthorizedResourceList"
resultType="org.apache.dolphinscheduler.dao.entity.Resource">
+
+ <select id="queryResourceExceptUserId"
resultType="org.apache.dolphinscheduler.dao.entity.Resource">
select
<include refid="baseSqlV2">
<property name="alias" value="r"/>
</include>
- from t_ds_resources r,t_ds_relation_resources_user rel
- where r.id = rel.resources_id AND rel.user_id = #{userId} and perm=7
- </select>
- <select id="queryResourceExceptUserId"
resultType="org.apache.dolphinscheduler.dao.entity.Resource">
- select
- <include refid="baseSql"/>
- from t_ds_resources
- where user_id <![CDATA[ <> ]]> #{userId}
- </select>
- <select id="queryTenantCodeByResourceName" resultType="java.lang.String">
- select tenant_code
- from t_ds_tenant t, t_ds_user u, t_ds_resources res
- where t.id = u.tenant_id and u.id = res.user_id and res.type=#{resType}
- and res.full_name= #{resName}
+ from t_ds_resources r
+ where r.user_id <![CDATA[ <> ]]> #{userId}
</select>
+
<select id="listAuthorizedResource"
resultType="org.apache.dolphinscheduler.dao.entity.Resource">
select
- <include refid="baseSql"/>
- from t_ds_resources
- where type=0
- and id in (select resources_id from t_ds_relation_resources_user where
user_id=#{userId} and perm=7
- union select id as resources_id from t_ds_resources where
user_id=#{userId})
+ <include refid="baseSqlV2">
+ <property name="alias" value="r"/>
+ </include>
+ from t_ds_resources r
+ where r.type = 0
+ and r.user_id=#{userId}
+
<if test="resNames != null and resNames.length > 0">
and full_name in
<foreach collection="resNames" item="i" open="(" close=")"
separator=",">
@@ -109,12 +101,27 @@
</foreach>
</if>
</select>
+ <select id="queryResourceListById"
resultType="org.apache.dolphinscheduler.dao.entity.Resource">
+ select
+ <include refid="baseSqlV2">
+ <property name="alias" value="r"/>
+ </include>
+ from t_ds_resources r
+ where 1 = 1
+ <if test="resIds != null and resIds.size() > 0">
+ and r.id in
+ <foreach collection="resIds" item="i" open="(" close=")"
separator=",">
+ #{i}
+ </foreach>
+ </if>
+ </select>
<select id="listAuthorizedResourceById"
resultType="org.apache.dolphinscheduler.dao.entity.Resource">
select
- <include refid="baseSql"/>
- from t_ds_resources
- where id in (select resources_id from t_ds_relation_resources_user
where user_id=#{userId} and perm=7
- union select id as resources_id from t_ds_resources where
user_id=#{userId})
+ <include refid="baseSqlV2">
+ <property name="alias" value="r"/>
+ </include>
+ from t_ds_resources r
+ where r.user_id=#{userId}
<if test="resIds != null and resIds.length > 0">
and id in
<foreach collection="resIds" item="i" open="(" close=")"
separator=",">
@@ -138,10 +145,12 @@
<select id="queryResource"
resultType="org.apache.dolphinscheduler.dao.entity.Resource">
select
- <include refid="baseSql"/>
- from t_ds_resources
- where type = #{type}
- and full_name = #{fullName}
+ <include refid="baseSqlV2">
+ <property name="alias" value="r"/>
+ </include>
+ from t_ds_resources r
+ where r.type = #{type}
+ and r.full_name = #{fullName}
</select>
<update id="batchUpdateResource" parameterType="java.util.List">
@@ -159,9 +168,11 @@
<select id="listResourceByIds"
resultType="org.apache.dolphinscheduler.dao.entity.Resource">
select
- <include refid="baseSql"/>
- from t_ds_resources
- where id in
+ <include refid="baseSqlV2">
+ <property name="alias" value="r"/>
+ </include>
+ from t_ds_resources r
+ where r.id in
<foreach collection="resIds" item="i" open="(" close=")" separator=",">
#{i}
</foreach>
diff --git
a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ResourceUserMapper.xml
b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ResourceUserMapper.xml
index 7fdd09f..d5500b4 100644
---
a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ResourceUserMapper.xml
+++
b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ResourceUserMapper.xml
@@ -18,6 +18,20 @@
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
"http://mybatis.org/dtd/mybatis-3-mapper.dtd"; >
<mapper namespace="org.apache.dolphinscheduler.dao.mapper.ResourceUserMapper">
+
+ <select id="queryResourcesIdListByUserIdAndPerm"
resultType="java.lang.Integer">
+ select
+ resources_id
+ from t_ds_relation_resources_user rel
+ where 1 = 1
+ <if test="userId != 0 and perm != 0">
+ and rel.user_id=#{userId} and rel.perm=#{perm}
+ </if>
+ <if test="userId != 0 and perm == 0">
+ and rel.user_id=#{userId}
+ </if>
+ </select>
+
<delete id="deleteResourceUser">
delete
from t_ds_relation_resources_user
diff --git
a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ResourceMapperTest.java
b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ResourceMapperTest.java
index f06e4fc..011d69c 100644
---
a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ResourceMapperTest.java
+++
b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ResourceMapperTest.java
@@ -14,18 +14,29 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.dolphinscheduler.dao.mapper;
+import static java.util.stream.Collectors.toList;
+
+import static org.hamcrest.Matchers.greaterThan;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertThat;
-import com.baomidou.mybatisplus.core.metadata.IPage;
-import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.ResourceType;
import org.apache.dolphinscheduler.common.enums.UserType;
+import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.dao.entity.Resource;
import org.apache.dolphinscheduler.dao.entity.ResourcesUser;
import org.apache.dolphinscheduler.dao.entity.Tenant;
import org.apache.dolphinscheduler.dao.entity.User;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Date;
+import java.util.List;
+
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -35,15 +46,8 @@ import org.springframework.test.annotation.Rollback;
import org.springframework.test.context.junit4.SpringRunner;
import org.springframework.transaction.annotation.Transactional;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Date;
-import java.util.List;
-
-import static java.util.stream.Collectors.toList;
-import static org.hamcrest.Matchers.greaterThan;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertThat;
+import com.baomidou.mybatisplus.core.metadata.IPage;
+import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
@RunWith(SpringRunner.class)
@SpringBootTest
@@ -238,14 +242,16 @@ public class ResourceMapperTest {
0,
-1,
resource.getType().ordinal(),
- ""
+ "",
+ new ArrayList<>()
);
IPage<Resource> resourceIPage1 = resourceMapper.queryResourcePaging(
page,
1110,
-1,
resource.getType().ordinal(),
- ""
+ "",
+ null
);
Assert.assertNotEquals(resourceIPage.getTotal(), 0);
Assert.assertNotEquals(resourceIPage1.getTotal(), 0);
@@ -259,7 +265,8 @@ public class ResourceMapperTest {
public void testQueryResourceListAuthored() {
Resource resource = insertOne();
- List<Resource> resources =
resourceMapper.queryAuthorizedResourceList(resource.getUserId());
+ List<Integer> resIds =
resourceUserMapper.queryResourcesIdListByUserIdAndPerm(resource.getUserId(),
Constants.AUTHORIZE_WRITABLE_PERM);
+ List<Resource> resources = CollectionUtils.isEmpty(resIds) ? new
ArrayList<>() : resourceMapper.queryResourceListById(resIds);
ResourcesUser resourcesUser = new ResourcesUser();
@@ -268,7 +275,8 @@ public class ResourceMapperTest {
resourcesUser.setPerm(Constants.AUTHORIZE_WRITABLE_PERM);
resourceUserMapper.insert(resourcesUser);
- List<Resource> resources1 =
resourceMapper.queryAuthorizedResourceList(1110);
+ List<Integer> resIds1 =
resourceUserMapper.queryResourcesIdListByUserIdAndPerm(1110,
Constants.AUTHORIZE_WRITABLE_PERM);
+ List<Resource> resources1 = CollectionUtils.isEmpty(resIds1) ? new
ArrayList<>() : resourceMapper.queryResourceListById(resIds1);
Assert.assertEquals(0, resources.size());
Assert.assertNotEquals(0, resources1.size());
@@ -282,7 +290,8 @@ public class ResourceMapperTest {
public void testQueryAuthorizedResourceList() {
Resource resource = insertOne();
- List<Resource> resources =
resourceMapper.queryAuthorizedResourceList(resource.getUserId());
+ List<Integer> resIds =
resourceUserMapper.queryResourcesIdListByUserIdAndPerm(resource.getUserId(),
Constants.AUTHORIZE_WRITABLE_PERM);
+ List<Resource> resources = CollectionUtils.isEmpty(resIds) ? new
ArrayList<>() : resourceMapper.queryResourceListById(resIds);
resourceMapper.deleteById(resource.getId());
Assert.assertEquals(0, resources.size());
@@ -306,7 +315,6 @@ public class ResourceMapperTest {
@Test
public void testQueryTenantCodeByResourceName() {
-
Tenant tenant = new Tenant();
tenant.setTenantCode("ut tenant code for resource");
int tenantInsertStatus = tenantMapper.insert(tenant);
@@ -324,7 +332,6 @@ public class ResourceMapperTest {
Assert.fail("insert user data error");
}
-
Resource resource = insertOne();
resource.setUserId(user.getId());
int userUpdateStatus = resourceMapper.updateById(resource);
@@ -332,12 +339,13 @@ public class ResourceMapperTest {
Assert.fail("update user data error");
}
- String resource1 = resourceMapper.queryTenantCodeByResourceName(
- resource.getFullName(), ResourceType.FILE.ordinal()
- );
+ List<Resource> resourceList =
resourceMapper.queryResource(resource.getFullName(),
ResourceType.FILE.ordinal());
+ int resourceUserId = resourceList.get(0).getUserId();
+ User resourceUser = userMapper.selectById(resourceUserId);
+ Tenant resourceTenant =
tenantMapper.selectById(resourceUser.getTenantId());
- Assert.assertEquals("ut tenant code for resource", resource1);
+ Assert.assertEquals("ut tenant code for resource",
resourceTenant.getTenantCode());
}
@@ -358,7 +366,6 @@ public class ResourceMapperTest {
Assert.assertEquals(generalUser2.getId(), resource.getUserId());
Assert.assertFalse(resources.stream().map(t ->
t.getFullName()).collect(toList()).containsAll(Arrays.asList(resNames)));
-
// authorize object unauthorizedResource to generalUser
createResourcesUser(unauthorizedResource, generalUser2);
List<Resource> authorizedResources =
resourceMapper.listAuthorizedResource(generalUser2.getId(), resNames);
@@ -390,10 +397,10 @@ public class ResourceMapperTest {
Resource resource = createResource(generalUser1);
createResourcesUser(resource, generalUser2);
- List<Resource> resourceList =
resourceMapper.queryResourceListAuthored(generalUser2.getId(),
ResourceType.FILE.ordinal(), 0);
+ List<Resource> resourceList =
resourceMapper.queryResourceListAuthored(generalUser2.getId(),
ResourceType.FILE.ordinal());
Assert.assertNotNull(resourceList);
- resourceList =
resourceMapper.queryResourceListAuthored(generalUser2.getId(),
ResourceType.FILE.ordinal(), 4);
+ resourceList =
resourceMapper.queryResourceListAuthored(generalUser2.getId(),
ResourceType.FILE.ordinal());
Assert.assertFalse(resourceList.contains(resource));
}
@@ -422,4 +429,5 @@ public class ResourceMapperTest {
insertOne();
Assert.assertTrue(resourceMapper.existResource(fullName, userId,
type));
}
-}
\ No newline at end of file
+}
+
diff --git
a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/registry/DependencyConfig.java
b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/registry/DependencyConfig.java
index d51d64d..f237bca 100644
---
a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/registry/DependencyConfig.java
+++
b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/registry/DependencyConfig.java
@@ -30,6 +30,7 @@ import
org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapMapper;
import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapper;
import org.apache.dolphinscheduler.dao.mapper.ProjectMapper;
import org.apache.dolphinscheduler.dao.mapper.ResourceMapper;
+import org.apache.dolphinscheduler.dao.mapper.ResourceUserMapper;
import org.apache.dolphinscheduler.dao.mapper.ScheduleMapper;
import org.apache.dolphinscheduler.dao.mapper.TaskInstanceMapper;
import org.apache.dolphinscheduler.dao.mapper.TenantMapper;
@@ -131,6 +132,11 @@ public class DependencyConfig {
}
@Bean
+ public ResourceUserMapper resourceUserMapper() {
+ return Mockito.mock(ResourceUserMapper.class);
+ }
+
+ @Bean
public ErrorCommandMapper errorCommandMapper() {
return Mockito.mock(ErrorCommandMapper.class);
}
diff --git
a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/processor/TaskCallbackServiceTestConfig.java
b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/processor/TaskCallbackServiceTestConfig.java
index 224764c..f4876a6 100644
---
a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/processor/TaskCallbackServiceTestConfig.java
+++
b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/processor/TaskCallbackServiceTestConfig.java
@@ -30,6 +30,7 @@ import
org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapMapper;
import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapper;
import org.apache.dolphinscheduler.dao.mapper.ProjectMapper;
import org.apache.dolphinscheduler.dao.mapper.ResourceMapper;
+import org.apache.dolphinscheduler.dao.mapper.ResourceUserMapper;
import org.apache.dolphinscheduler.dao.mapper.ScheduleMapper;
import org.apache.dolphinscheduler.dao.mapper.TaskInstanceMapper;
import org.apache.dolphinscheduler.dao.mapper.TenantMapper;
@@ -119,6 +120,11 @@ public class TaskCallbackServiceTestConfig {
}
@Bean
+ public ResourceUserMapper resourceUserMapper() {
+ return Mockito.mock(ResourceUserMapper.class);
+ }
+
+ @Bean
public ErrorCommandMapper errorCommandMapper() {
return Mockito.mock(ErrorCommandMapper.class);
}
diff --git
a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java
b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java
index fe73eae..55b1d0f 100644
---
a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java
+++
b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java
@@ -76,6 +76,7 @@ import
org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapMapper;
import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapper;
import org.apache.dolphinscheduler.dao.mapper.ProjectMapper;
import org.apache.dolphinscheduler.dao.mapper.ResourceMapper;
+import org.apache.dolphinscheduler.dao.mapper.ResourceUserMapper;
import org.apache.dolphinscheduler.dao.mapper.ScheduleMapper;
import org.apache.dolphinscheduler.dao.mapper.TaskInstanceMapper;
import org.apache.dolphinscheduler.dao.mapper.TenantMapper;
@@ -156,6 +157,9 @@ public class ProcessService {
private ResourceMapper resourceMapper;
@Autowired
+ private ResourceUserMapper resourceUserMapper;
+
+ @Autowired
private ErrorCommandMapper errorCommandMapper;
@Autowired
@@ -1794,7 +1798,21 @@ public class ProcessService {
public String queryTenantCodeByResName(String resName, ResourceType
resourceType) {
// in order to query tenant code successful although the version is
older
String fullName = resName.startsWith("/") ? resName :
String.format("/%s", resName);
- return resourceMapper.queryTenantCodeByResourceName(fullName,
resourceType.ordinal());
+
+ List<Resource> resourceList = resourceMapper.queryResource(fullName,
resourceType.ordinal());
+ if (CollectionUtils.isEmpty(resourceList)) {
+ return StringUtils.EMPTY;
+ }
+ int userId = resourceList.get(0).getUserId();
+ User user = userMapper.selectById(userId);
+ if (Objects.isNull(user)) {
+ return StringUtils.EMPTY;
+ }
+ Tenant tenant = tenantMapper.selectById(user.getTenantId());
+ if (Objects.isNull(tenant)) {
+ return StringUtils.EMPTY;
+ }
+ return tenant.getTenantCode();
}
/**
@@ -2036,11 +2054,15 @@ public class ProcessService {
switch (authorizationType) {
case RESOURCE_FILE_ID:
case UDF_FILE:
- Set<Integer> authorizedResourceFiles =
resourceMapper.listAuthorizedResourceById(userId,
needChecks).stream().map(Resource::getId).collect(toSet());
+ List<Resource> ownUdfResources =
resourceMapper.listAuthorizedResourceById(userId, needChecks);
+ addAuthorizedResources(ownUdfResources, userId);
+ Set<Integer> authorizedResourceFiles =
ownUdfResources.stream().map(Resource::getId).collect(toSet());
originResSet.removeAll(authorizedResourceFiles);
break;
case RESOURCE_FILE_NAME:
- Set<String> authorizedResources =
resourceMapper.listAuthorizedResource(userId,
needChecks).stream().map(Resource::getFullName).collect(toSet());
+ List<Resource> ownResources =
resourceMapper.listAuthorizedResource(userId, needChecks);
+ addAuthorizedResources(ownResources, userId);
+ Set<String> authorizedResources =
ownResources.stream().map(Resource::getFullName).collect(toSet());
originResSet.removeAll(authorizedResources);
break;
case DATASOURCE:
@@ -2163,4 +2185,15 @@ public class ProcessService {
}
return JSONUtils.toJsonString(processData);
}
+
+ /**
+ * add authorized resources
+ * @param ownResources own resources
+ * @param userId userId
+ */
+ private void addAuthorizedResources(List<Resource> ownResources, int
userId) {
+ List<Integer> relationResourceIds =
resourceUserMapper.queryResourcesIdListByUserIdAndPerm(userId, 7);
+ List<Resource> relationResources =
CollectionUtils.isNotEmpty(relationResourceIds) ?
resourceMapper.queryResourceListById(relationResourceIds) : new ArrayList<>();
+ ownResources.addAll(relationResources);
+ }
}
