This is an automated email from the ASF dual-hosted git repository. alexkun pushed a commit to branch dev-1.2.0 in repository https://gitbox.apache.org/repos/asf/incubator-linkis.git
commit 2f9e9bd9b9309a154dbf04d124e5c9a156f12d76 Author: peacewong <[email protected]> AuthorDate: Fri Jul 22 16:30:49 2022 +0800 undonetask interface performance optimization close #2504 --- .../impl/DefaultLabelManagerPersistence.java | 2 +- .../linkis/jobhistory/dao/JobHistoryMapper.java | 38 +++++++++++-- .../jobhistory/dao/impl/JobHistoryMapper.xml | 40 ++++++++++++++ .../jobhistory/restful/api/QueryRestfulApi.java | 49 ++++++++++++++++- .../jobhistory/service/JobHistoryQueryService.java | 4 +- .../service/impl/JobHistoryQueryServiceImpl.scala | 64 +++++++++++++++++++--- 6 files changed, 180 insertions(+), 17 deletions(-) diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/persistence/impl/DefaultLabelManagerPersistence.java b/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/persistence/impl/DefaultLabelManagerPersistence.java index 2e4798099..da46fa111 100644 --- a/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/persistence/impl/DefaultLabelManagerPersistence.java +++ b/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/persistence/impl/DefaultLabelManagerPersistence.java @@ -139,7 +139,7 @@ public class DefaultLabelManagerPersistence implements LabelManagerPersistence { @Override public PersistenceLabel getLabel(int id) { - PersistenceLabel persistenceLabel = labelManagerMapper.getLabel(id); + PersistenceLabel persistenceLabel = labelManagerMapper.getLabel(id); PersistenceUtils.setValue(persistenceLabel); return labelManagerMapper.getLabel(id); } diff --git a/linkis-public-enhancements/linkis-publicservice/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/dao/JobHistoryMapper.java b/linkis-public-enhancements/linkis-publicservice/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/dao/JobHistoryMapper.java index 12214723e..3c72f6e96 100644 --- a/linkis-public-enhancements/linkis-publicservice/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/dao/JobHistoryMapper.java +++ b/linkis-public-enhancements/linkis-publicservice/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/dao/JobHistoryMapper.java @@ -32,22 +32,22 @@ public interface JobHistoryMapper { void updateJobHistory(JobHistory jobReq); - List<JobHistory> search( + List<JobHistory> searchWithIdOrderAsc( @Param("id") Long id, @Param("umUser") String username, @Param("status") List<String> status, @Param("startDate") Date startDate, @Param("endDate") Date endDate, - @Param("engineType") String engineType, - @Param("startId") Long startId); + @Param("engineType") String engineType); - List<JobHistory> searchWithIdOrderAsc( + List<JobHistory> search( @Param("id") Long id, @Param("umUser") String username, @Param("status") List<String> status, @Param("startDate") Date startDate, @Param("endDate") Date endDate, - @Param("engineType") String engineType); + @Param("engineType") String engineType, + @Param("startId") Long startId); List<JobHistory> searchWithUserCreator( @Param("id") Long id, @@ -71,5 +71,33 @@ public interface JobHistoryMapper { @Param("engineType") String engineType, @Param("startId") Long startId); + Integer countUndoneTaskNoCreator( + @Param("umUser") String username, + @Param("status") List<String> status, + @Param("startDate") Date startDate, + @Param("endDate") Date endDate, + @Param("engineType") String engineType, + @Param("startId") Long startId); + + Integer countUndoneTaskWithUserCreator( + @Param("umUser") String username, + @Param("userCreatorKey") String userCreatorKey, + @Param("userCreatorValue") String userCreator, + @Param("status") List<String> status, + @Param("startDate") Date startDate, + @Param("endDate") Date endDate, + @Param("engineType") String engineType, + @Param("startId") Long startId); + + Integer countUndoneTaskWithCreatorOnly( + @Param("umUser") String username, + @Param("userCreatorKey") String userCreatorKey, + @Param("creator") String userCreator, + @Param("status") List<String> status, + @Param("startDate") Date startDate, + @Param("endDate") Date endDate, + @Param("engineType") String engineType, + @Param("startId") Long startId); + String selectJobHistoryStatusForUpdate(Long jobId); } diff --git a/linkis-public-enhancements/linkis-publicservice/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/dao/impl/JobHistoryMapper.xml b/linkis-public-enhancements/linkis-publicservice/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/dao/impl/JobHistoryMapper.xml index 2a2823c1f..9dc88cd2e 100644 --- a/linkis-public-enhancements/linkis-publicservice/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/dao/impl/JobHistoryMapper.xml +++ b/linkis-public-enhancements/linkis-publicservice/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/dao/impl/JobHistoryMapper.xml @@ -173,4 +173,44 @@ WHERE id = #{jobId} FOR UPDATE </select> + + <select id="countUndoneTaskNoCreator" useCache="true" resultType="java.lang.Integer" > + /*slave*/ SELECT COUNT(1) FROM linkis_ps_job_history_group_history + <where> + <if test="startId != null"> id >= #{startId}</if> + <if test="umUser != null">and submit_user = #{umUser}</if> + <if test="engineType != null">and engine_type = #{engineType}</if> + <if test="startDate != null">and created_time >= #{startDate} AND created_time <![CDATA[<=]]> #{endDate}</if> + <if test="status != null">and <foreach collection="status" item="element" close=")" separator="," open="status in (">#{element}</foreach></if> + </where> + </select> + + <select id="countUndoneTaskWithUserCreator" useCache="true" resultType="java.lang.Integer" > + /*slave*/ SELECT COUNT(1) FROM linkis_ps_job_history_group_history + <where> + <if test="startId != null"> id >= #{startId}</if> + <if test="umUser != null">and submit_user = #{umUser}</if> + <if test="engineType != null">and engine_type = #{engineType}</if> + <if test="startDate != null">and created_time >= #{startDate} AND created_time <![CDATA[<=]]> #{endDate}</if> + <if test="status != null">and <foreach collection="status" item="element" close=")" separator="," open="status in (">#{element}</foreach></if> + <if test="userCreatorKey != null and userCreatorValue != null"> + and LOCATE('"${userCreatorKey}":"${userCreatorValue}', labels) > 0 + </if> + </where> + </select> + + <select id="countUndoneTaskWithCreatorOnly" useCache="true" resultType="java.lang.Integer" > + /*slave*/ SELECT COUNT(1) FROM linkis_ps_job_history_group_history + <where> + <if test="startId != null"> id >= #{startId}</if> + <if test="umUser != null">and submit_user = #{umUser}</if> + <if test="engineType != null">and engine_type = #{engineType}</if> + <if test="startDate != null">and created_time >= #{startDate} AND created_time <![CDATA[<=]]> #{endDate}</if> + <if test="status != null">and <foreach collection="status" item="element" close=")" separator="," open="status in (">#{element}</foreach></if> + <if test="userCreatorKey != null and creator != null"> + and labels like '%"${userCreatorKey}":"%-${creator}%' + </if> + </where> + </select> + </mapper> diff --git a/linkis-public-enhancements/linkis-publicservice/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/restful/api/QueryRestfulApi.java b/linkis-public-enhancements/linkis-publicservice/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/restful/api/QueryRestfulApi.java index 117b92bb1..89bb49f26 100644 --- a/linkis-public-enhancements/linkis-publicservice/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/restful/api/QueryRestfulApi.java +++ b/linkis-public-enhancements/linkis-publicservice/linkis-jobhistory/src/main/java/org/apache/linkis/jobhistory/restful/api/QueryRestfulApi.java @@ -209,8 +209,8 @@ public class QueryRestfulApi { } /** Method list should not contain subjob, which may cause performance problems. */ - @RequestMapping(path = "/listundone", method = RequestMethod.GET) - public Message listundone( + @RequestMapping(path = "/listundonetasks", method = RequestMethod.GET) + public Message listundonetasks( HttpServletRequest req, @RequestParam(value = "startDate", required = false) Long startDate, @RequestParam(value = "endDate", required = false) Long endDate, @@ -281,4 +281,49 @@ public class QueryRestfulApi { .data(TaskConstant.TASKS, vos) .data(JobRequestConstants.TOTAL_PAGE(), total); } + + /** Method list should not contain subjob, which may cause performance problems. */ + @RequestMapping(path = "/listundone", method = RequestMethod.GET) + public Message listundone( + HttpServletRequest req, + @RequestParam(value = "startDate", required = false) Long startDate, + @RequestParam(value = "endDate", required = false) Long endDate, + @RequestParam(value = "pageNow", required = false) Integer pageNow, + @RequestParam(value = "pageSize", required = false) Integer pageSize, + @RequestParam(value = "startTaskID", required = false) Long taskID, + @RequestParam(value = "engineType", required = false) String engineType, + @RequestParam(value = "creator", required = false) String creator) + throws IOException, QueryException { + String username = SecurityFilter.getLoginUsername(req); + if (endDate == null) { + endDate = System.currentTimeMillis(); + } + if (startDate == null) { + startDate = 0L; + } + if (StringUtils.isEmpty(creator)) { + creator = null; + } + Date sDate = new Date(startDate); + Date eDate = new Date(endDate); + if (startDate == 0L) { + sDate = DateUtils.addDays(eDate, -1); + } + if (sDate.getTime() == eDate.getTime()) { + Calendar instance = Calendar.getInstance(); + instance.setTimeInMillis(endDate); + instance.add(Calendar.DAY_OF_MONTH, 1); + eDate = new Date(instance.getTime().getTime()); + } + Integer total = + jobHistoryQueryService.countUndoneTasks( + username, + creator, + sDate, + eDate, + engineType, + queryCacheManager.getUndoneTaskMinId()); + + return Message.ok().data(JobRequestConstants.TOTAL_PAGE(), total); + } } diff --git a/linkis-public-enhancements/linkis-publicservice/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/service/JobHistoryQueryService.java b/linkis-public-enhancements/linkis-publicservice/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/service/JobHistoryQueryService.java index cd41a0f7f..ff309ebaa 100644 --- a/linkis-public-enhancements/linkis-publicservice/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/service/JobHistoryQueryService.java +++ b/linkis-public-enhancements/linkis-publicservice/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/service/JobHistoryQueryService.java @@ -38,7 +38,9 @@ public interface JobHistoryQueryService { JobHistory getJobHistoryByIdAndName(Long jobID, String userName); - List<JobHistory> search(Long jobId, String username, String creator, String status, Date sDate, Date eDate, String executionApplicationName, Long startJobId); + List<JobHistory> search(Long jobId, String username, String creator, String status, Date sDate, Date eDate, String engineType, Long startJobId); + + Integer countUndoneTasks(String username, String creator, Date sDate, Date eDate, String engineType, Long startJobId); JobHistory searchOne(Long jobId, Date sDate, Date eDate); diff --git a/linkis-public-enhancements/linkis-publicservice/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/service/impl/JobHistoryQueryServiceImpl.scala b/linkis-public-enhancements/linkis-publicservice/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/service/impl/JobHistoryQueryServiceImpl.scala index 39972617c..edf868564 100644 --- a/linkis-public-enhancements/linkis-publicservice/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/service/impl/JobHistoryQueryServiceImpl.scala +++ b/linkis-public-enhancements/linkis-publicservice/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/service/impl/JobHistoryQueryServiceImpl.scala @@ -17,6 +17,7 @@ package org.apache.linkis.jobhistory.service.impl +import com.google.common.cache.{Cache, CacheBuilder} import com.google.common.collect.Iterables import org.apache.commons.lang3.StringUtils import org.apache.commons.lang3.exception.ExceptionUtils @@ -38,6 +39,7 @@ import org.springframework.stereotype.Service import java.sql.Timestamp import java.{lang, util} import java.util.Date +import java.util.concurrent.{Callable, TimeUnit} import scala.collection.JavaConversions._ import scala.collection.JavaConverters.asScalaBufferConverter @@ -49,8 +51,13 @@ class JobHistoryQueryServiceImpl extends JobHistoryQueryService with Logging { private var jobHistoryMapper: JobHistoryMapper = _ @Autowired private var jobDetailMapper: JobDetailMapper = _ -// @Autowired -// private var queryCacheService: QueryCacheService = _ + + private val unDoneTaskCache: Cache[String, Integer] = CacheBuilder.newBuilder().concurrencyLevel(5) + .expireAfterWrite(1, TimeUnit.MINUTES) + .initialCapacity(20) + .maximumSize(1000) + .recordStats() + .build() @Receiver override def add(jobReqInsert: JobReqInsert): JobRespProtocol = { @@ -210,11 +217,11 @@ class JobHistoryQueryServiceImpl extends JobHistoryQueryService with Logging { } override def search(jobId: java.lang.Long, username: String, status: String, creator: String, sDate: Date, eDate: Date, engineType: String, startJobId: java.lang.Long): util.List[JobHistory] = { - import scala.collection.JavaConversions._ + val split: util.List[String] = if (status != null) status.split(",").toList else null val result = if (StringUtils.isBlank(creator)) { jobHistoryMapper.search(jobId, username, split, sDate, eDate, engineType, startJobId) - } else if(StringUtils.isBlank(username)) { + } else if (StringUtils.isBlank(username)) { val fakeLabel = new UserCreatorLabel jobHistoryMapper.searchWithCreatorOnly(jobId, username, fakeLabel.getLabelKey, creator, split, sDate, eDate, engineType, startJobId) } else { @@ -223,7 +230,7 @@ class JobHistoryQueryServiceImpl extends JobHistoryQueryService with Logging { fakeLabel.setCreator(creator) val userCreator = fakeLabel.getStringValue Utils.tryCatch(fakeLabel.valueCheck(userCreator)) { - t => logger.info("input user or creator is not correct", t) + t => info("input user or creator is not correct", t) throw t } jobHistoryMapper.searchWithUserCreator(jobId, username, fakeLabel.getLabelKey, userCreator, split, sDate, eDate, engineType, startJobId) @@ -231,14 +238,16 @@ class JobHistoryQueryServiceImpl extends JobHistoryQueryService with Logging { result } + + override def getQueryVOList(list: java.util.List[JobHistory]): java.util.List[JobRequest] = { jobHistory2JobRequest(list) } - private def shouldUpdate(oldStatus: String, newStatus: String): Boolean = { - if(TaskStatus.valueOf(oldStatus) == TaskStatus.valueOf(newStatus)){ + private def shouldUpdate(oldStatus: String, newStatus: String): Boolean = { + if (TaskStatus.valueOf(oldStatus) == TaskStatus.valueOf(newStatus)) { true - }else{ + } else { TaskStatus.valueOf(oldStatus).ordinal <= TaskStatus.valueOf(newStatus).ordinal && !TaskStatus.isComplete(TaskStatus.valueOf(oldStatus)) } } @@ -256,5 +265,44 @@ class JobHistoryQueryServiceImpl extends JobHistoryQueryService with Logging { }) } + override def countUndoneTasks(username: String, creator: String, sDate: Date, eDate: Date, engineType: String, startJobId: lang.Long): Integer = { + val cacheKey = if (StringUtils.isNoneBlank(username, creator, engineType)) "" else { + s"${username}_${creator}_${engineType}" + } + if (StringUtils.isBlank(cacheKey)) { + getCountUndoneTasks(username, creator, sDate, eDate, engineType, startJobId) + } else { + unDoneTaskCache.get(cacheKey, new Callable[Integer]{ + override def call(): Integer = { + getCountUndoneTasks(username, creator, sDate, eDate, engineType, startJobId) + } + }) + } + } + + private def getCountUndoneTasks(username: String, creator: String, sDate: Date, eDate: Date, engineType: String, startJobId: lang.Long): Integer = { + val statusList: util.List[String] = new util.ArrayList[String]() + statusList.add(TaskStatus.Running.toString) + statusList.add(TaskStatus.Inited.toString) + statusList.add(TaskStatus.Scheduled.toString) + + val count = if (StringUtils.isBlank(creator)) { + jobHistoryMapper.countUndoneTaskNoCreator(username, statusList, sDate, eDate, engineType, startJobId) + } else if (StringUtils.isBlank(username)) { + val fakeLabel = new UserCreatorLabel + jobHistoryMapper.countUndoneTaskWithCreatorOnly(username, fakeLabel.getLabelKey, creator, statusList, sDate, eDate, engineType, startJobId) + } else { + val fakeLabel = new UserCreatorLabel + fakeLabel.setUser(username) + fakeLabel.setCreator(creator) + val userCreator = fakeLabel.getStringValue + Utils.tryCatch(fakeLabel.valueCheck(userCreator)) { + t => logger.info("input user or creator is not correct", t) + throw t + } + jobHistoryMapper.countUndoneTaskWithUserCreator(username, fakeLabel.getLabelKey, userCreator, statusList, sDate, eDate, engineType, startJobId) + } + count + } } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
