This is an automated email from the ASF dual-hosted git repository.
casion pushed a commit to branch dev-1.2.0
in repository https://gitbox.apache.org/repos/asf/incubator-linkis.git
The following commit(s) were added to refs/heads/dev-1.2.0 by this push:
new 0ea0a0ad8 Fix issue that querying ECM history, permission management
fails (#2417)
0ea0a0ad8 is described below
commit 0ea0a0ad8a86bdc3071029c637a41bd60a1997b9
Author: peacewong <[email protected]>
AuthorDate: Thu Jul 14 22:17:47 2022 +0800
Fix issue that querying ECM history, permission management fails (#2417)
* fix isues #2412
Co-authored-by: v_kkhuang <[email protected]>
---
.../am/restful/ECResourceInfoRestfulApi.java | 32 +++++++---------
.../manager/am/service/ECResourceInfoService.java | 2 +-
.../am/service/impl/ECResourceInfoServiceImpl.java | 4 +-
.../manager/am/util/ECResourceInfoUtils.java | 43 +++++++++++++++++++---
.../manager/am/vo/ECResourceInfoRecordVo.java | 19 +++++-----
.../ResourceVo.java} | 36 +++++++++++-------
.../linkis/manager/rm/restful/RMMonitorRest.scala | 1 +
.../linkis/manager/dao/ECResourceRecordMapper.java | 3 +-
.../manager/dao/impl/ECResourceRecordMapper.xml | 1 +
9 files changed, 89 insertions(+), 52 deletions(-)
diff --git
a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/restful/ECResourceInfoRestfulApi.java
b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/restful/ECResourceInfoRestfulApi.java
index 5311bdf29..5ecba6768 100644
---
a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/restful/ECResourceInfoRestfulApi.java
+++
b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/restful/ECResourceInfoRestfulApi.java
@@ -25,6 +25,7 @@ import org.apache.linkis.manager.am.util.ECResourceInfoUtils;
import org.apache.linkis.manager.am.vo.ECResourceInfoRecordVo;
import
org.apache.linkis.manager.common.entity.persistence.ECResourceInfoRecord;
import org.apache.linkis.server.Message;
+import org.apache.linkis.server.security.SecurityFilter;
import org.apache.linkis.server.utils.ModuleUserUtils;
import org.apache.commons.lang.StringUtils;
@@ -40,7 +41,6 @@ import com.github.pagehelper.PageHelper;
import com.github.pagehelper.PageInfo;
import java.util.*;
-import java.util.stream.Collectors;
@RequestMapping(
path = "/linkisManager/ecinfo",
@@ -99,14 +99,13 @@ public class ECResourceInfoRestfulApi {
@RequestParam(value = "pageNow", required = false, defaultValue =
"1") Integer pageNow,
@RequestParam(value = "pageSize", required = false, defaultValue =
"20")
Integer pageSize) {
- // String username = SecurityFilter.getLoginUsername(req);
- String username = "hadoop";
+ String username = SecurityFilter.getLoginUsername(req);
// Parameter judgment
instance = ECResourceInfoUtils.strCheckAndDef(instance, null);
- creator = ECResourceInfoUtils.strCheckAndDef(creator, null);
+ String creatorUser = ECResourceInfoUtils.strCheckAndDef(creator, null);
engineType = ECResourceInfoUtils.strCheckAndDef(engineType, null);
- if (null != creator && !ECResourceInfoUtils.checkNameValid(creator)) {
- return Message.error("Invalid creator : " + creator);
+ if (null != creatorUser &&
!ECResourceInfoUtils.checkNameValid(creatorUser)) {
+ return Message.error("Invalid creator : " + creatorUser);
}
if (null == startDate) {
Calendar calendar = Calendar.getInstance();
@@ -117,8 +116,8 @@ public class ECResourceInfoRestfulApi {
}
if (Configuration.isAdmin(username)) {
username = null;
- if (StringUtils.isNotBlank(creator)) {
- username = creator;
+ if (StringUtils.isNotBlank(creatorUser)) {
+ username = creatorUser;
}
}
List<ECResourceInfoRecordVo> list = new ArrayList<>();
@@ -128,14 +127,7 @@ public class ECResourceInfoRestfulApi {
try {
queryTasks =
ecResourceInfoService.getECResourceInfoRecordList(
- instance, endDate, startDate, username);
- if (StringUtils.isNotBlank(engineType)) {
- String finalEngineType = engineType;
- queryTasks =
- queryTasks.stream()
- .filter(info ->
info.getLabelValue().contains(finalEngineType))
- .collect(Collectors.toList());
- }
+ instance, endDate, startDate, username,
engineType);
queryTasks.forEach(
info -> {
ECResourceInfoRecordVo ecrHistroryListVo = new
ECResourceInfoRecordVo();
@@ -143,11 +135,13 @@ public class ECResourceInfoRestfulApi {
ecrHistroryListVo.setEngineType(
info.getLabelValue().split(",")[1].split("-")[0]);
ecrHistroryListVo.setUsedResource(
-
ECResourceInfoUtils.getStringToMap(info.getUsedResource()));
+
ECResourceInfoUtils.getStringToMap(info.getUsedResource(), info));
ecrHistroryListVo.setReleasedResource(
-
ECResourceInfoUtils.getStringToMap(info.getReleasedResource()));
+ ECResourceInfoUtils.getStringToMap(
+ info.getReleasedResource(), info));
ecrHistroryListVo.setRequestResource(
-
ECResourceInfoUtils.getStringToMap(info.getRequestResource()));
+ ECResourceInfoUtils.getStringToMap(
+ info.getRequestResource(), info));
list.add(ecrHistroryListVo);
});
} finally {
diff --git
a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/ECResourceInfoService.java
b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/ECResourceInfoService.java
index 014e31e15..22b4ce24a 100644
---
a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/ECResourceInfoService.java
+++
b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/ECResourceInfoService.java
@@ -31,7 +31,7 @@ public interface ECResourceInfoService {
void deleteECResourceInfoRecord(Integer id);
List<ECResourceInfoRecord> getECResourceInfoRecordList(
- String instance, Date endDate, Date startDate, String username);
+ String instance, Date endDate, Date startDate, String username,
String engineType);
// TODO add search method
diff --git
a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/impl/ECResourceInfoServiceImpl.java
b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/impl/ECResourceInfoServiceImpl.java
index 899295799..a00ac60a4 100644
---
a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/impl/ECResourceInfoServiceImpl.java
+++
b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/impl/ECResourceInfoServiceImpl.java
@@ -64,8 +64,8 @@ public class ECResourceInfoServiceImpl implements
ECResourceInfoService {
@Override
public List<ECResourceInfoRecord> getECResourceInfoRecordList(
- String instance, Date endDate, Date startDate, String username) {
+ String instance, Date endDate, Date startDate, String username,
String engineType) {
return ecResourceRecordMapper.getECResourceInfoHistory(
- username, instance, endDate, startDate);
+ username, instance, endDate, startDate, engineType);
}
}
diff --git
a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/util/ECResourceInfoUtils.java
b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/util/ECResourceInfoUtils.java
index 0871913ff..980389f9a 100644
---
a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/util/ECResourceInfoUtils.java
+++
b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/util/ECResourceInfoUtils.java
@@ -16,9 +16,16 @@
*/
package org.apache.linkis.manager.am.util;
+import org.apache.linkis.common.utils.ByteTimeUtils;
+import org.apache.linkis.manager.am.vo.ResourceVo;
+import
org.apache.linkis.manager.common.entity.persistence.ECResourceInfoRecord;
+import org.apache.linkis.server.BDPJettyServerHelper;
+
+import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang.StringUtils;
-import com.google.gson.Gson;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.util.HashMap;
import java.util.Map;
@@ -26,6 +33,8 @@ import java.util.regex.Pattern;
public class ECResourceInfoUtils {
+ private static Logger logger =
LoggerFactory.getLogger(ECResourceInfoUtils.class);
+
public static String NAME_REGEX = "^[a-zA-Z\\d_\\.]+$";
public static boolean checkNameValid(String creator) {
@@ -36,10 +45,32 @@ public class ECResourceInfoUtils {
return StringUtils.isBlank(str) ? def : str;
}
- public static Map<String, Object> getStringToMap(String str) {
- Gson gson = new Gson();
- Map<String, Object> map = new HashMap<>();
- map = gson.fromJson(str, map.getClass());
- return map;
+ public static ResourceVo getStringToMap(String str, ECResourceInfoRecord
info) {
+ ResourceVo resourceVo = null;
+ Map<String, Object> map =
+ BDPJettyServerHelper.gson().fromJson(str, new
HashMap<>().getClass());
+ if (MapUtils.isNotEmpty(map)) {
+ resourceVo = new ResourceVo();
+ if (info.getLabelValue().contains("spark")
+ || (info.getLabelValue().contains("flink"))) {
+ if (null != map.get("driver")) {
+ Map<String, Object> divermap = MapUtils.getMap(map,
"driver");
+ resourceVo.setInstance(((Double)
divermap.get("instance")).intValue());
+ resourceVo.setCores(((Double)
divermap.get("cpu")).intValue());
+ resourceVo.setMemory(
+
ByteTimeUtils.byteStringAsBytes(divermap.get("memory").toString()));
+ return resourceVo;
+ } else {
+ logger.warn("Compatible with old data ,{},{}",
info.getLabelValue(), info);
+ return null; // Compatible with old data
+ }
+ }
+ resourceVo.setInstance(((Double) map.get("instance")).intValue());
+
resourceVo.setMemory(ByteTimeUtils.byteStringAsBytes((map.get("memory").toString())));
+ Double core =
+ null == map.get("cpu") ? (Double) map.get("cores") :
(Double) map.get("cpu");
+ resourceVo.setCores(core.intValue());
+ }
+ return resourceVo;
}
}
diff --git
a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/vo/ECResourceInfoRecordVo.java
b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/vo/ECResourceInfoRecordVo.java
index fe8c8e77b..dedb09116 100644
---
a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/vo/ECResourceInfoRecordVo.java
+++
b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/vo/ECResourceInfoRecordVo.java
@@ -17,7 +17,6 @@
package org.apache.linkis.manager.am.vo;
import java.util.Date;
-import java.util.Map;
public class ECResourceInfoRecordVo {
@@ -31,11 +30,11 @@ public class ECResourceInfoRecordVo {
private String serviceInstance;
- private Map usedResource;
+ private ResourceVo usedResource;
- private Map requestResource;
+ private ResourceVo requestResource;
- private Map releasedResource;
+ private ResourceVo releasedResource;
private String ecmInstance;
@@ -97,27 +96,27 @@ public class ECResourceInfoRecordVo {
this.serviceInstance = serviceInstance;
}
- public Map getUsedResource() {
+ public ResourceVo getUsedResource() {
return usedResource;
}
- public void setUsedResource(Map usedResource) {
+ public void setUsedResource(ResourceVo usedResource) {
this.usedResource = usedResource;
}
- public Map getRequestResource() {
+ public ResourceVo getRequestResource() {
return requestResource;
}
- public void setRequestResource(Map requestResource) {
+ public void setRequestResource(ResourceVo requestResource) {
this.requestResource = requestResource;
}
- public Map getReleasedResource() {
+ public ResourceVo getReleasedResource() {
return releasedResource;
}
- public void setReleasedResource(Map releasedResource) {
+ public void setReleasedResource(ResourceVo releasedResource) {
this.releasedResource = releasedResource;
}
diff --git
a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/ECResourceInfoService.java
b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/vo/ResourceVo.java
similarity index 58%
copy from
linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/ECResourceInfoService.java
copy to
linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/vo/ResourceVo.java
index 014e31e15..035938103 100644
---
a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/service/ECResourceInfoService.java
+++
b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/java/org/apache/linkis/manager/am/vo/ResourceVo.java
@@ -14,25 +14,35 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+package org.apache.linkis.manager.am.vo;
-package org.apache.linkis.manager.am.service;
+public class ResourceVo {
-import
org.apache.linkis.manager.common.entity.persistence.ECResourceInfoRecord;
+ private int cores;
+ private int instance;
+ private long memory;
-import java.util.Date;
-import java.util.List;
+ public int getCores() {
+ return cores;
+ }
-public interface ECResourceInfoService {
+ public void setCores(int cores) {
+ this.cores = cores;
+ }
- ECResourceInfoRecord getECResourceInfoRecord(String ticketId);
+ public int getInstance() {
+ return instance;
+ }
- void deleteECResourceInfoRecordByTicketId(String ticketId);
+ public void setInstance(int instance) {
+ this.instance = instance;
+ }
- void deleteECResourceInfoRecord(Integer id);
-
- List<ECResourceInfoRecord> getECResourceInfoRecordList(
- String instance, Date endDate, Date startDate, String username);
-
- // TODO add search method
+ public long getMemory() {
+ return memory;
+ }
+ public void setMemory(long memory) {
+ this.memory = memory;
+ }
}
diff --git
a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/restful/RMMonitorRest.scala
b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/restful/RMMonitorRest.scala
index c41c4eede..682f55419 100644
---
a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/restful/RMMonitorRest.scala
+++
b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/restful/RMMonitorRest.scala
@@ -142,6 +142,7 @@ class RMMonitorRest extends Logging {
engineInstance.put("engineType", engineTypeLabel.getEngineType)
engineInstance.put("instance", node.getServiceInstance.getInstance)
engineInstance.put("label", engineTypeLabel.getStringValue)
+ node.setNodeResource(ResourceUtils.convertTo(node.getNodeResource,
ResourceType.LoadInstance))
engineInstance.put("resource", node.getNodeResource)
if (node.getNodeStatus == null) {
engineInstance.put("status", "Busy")
diff --git
a/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/dao/ECResourceRecordMapper.java
b/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/dao/ECResourceRecordMapper.java
index 4d8bb7178..89bbbef47 100644
---
a/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/dao/ECResourceRecordMapper.java
+++
b/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/dao/ECResourceRecordMapper.java
@@ -42,5 +42,6 @@ public interface ECResourceRecordMapper {
@Param("username") String username,
@Param("instance") String instance,
@Param("endDate") Date endDate,
- @Param("startDate") Date startDate);
+ @Param("startDate") Date startDate,
+ @Param("engineType") String engineType);
}
diff --git
a/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/dao/impl/ECResourceRecordMapper.xml
b/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/dao/impl/ECResourceRecordMapper.xml
index 0154fad45..0e2b6956f 100644
---
a/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/dao/impl/ECResourceRecordMapper.xml
+++
b/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/dao/impl/ECResourceRecordMapper.xml
@@ -66,6 +66,7 @@
<where>
<if test="instance != null"> service_instance = #{instance}</if>
<if test="username != null"> and create_user = #{username}</if>
+ <if test="engineType !=null">and label_value like
concat('%',#{engineType},'%')</if>
<if test="startDate != null">and create_time BETWEEN #{startDate}
AND #{endDate}</if>
</where>
ORDER BY linkis_cg_ec_resource_info_record.create_time DESC
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
