This is an automated email from the ASF dual-hosted git repository. qiaojialin pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/iotdb-web-workbench.git
commit 893c93408bce2de4663dc4560ba212473fbd681d Author: loveher147 <[email protected]> AuthorDate: Mon May 30 15:19:22 2022 +0800 新增监控指标展示功能 - 解决大数据量场景的卡顿问题。 --- .../iotdb/admin/controller/IotDBController.java | 62 +++++--- .../iotdb/admin/model/dto/DataModelDetailDTO.java | 3 +- .../apache/iotdb/admin/model/vo/NodeTreeVO.java | 9 ++ .../apache/iotdb/admin/service/IotDBService.java | 28 ++-- .../iotdb/admin/service/impl/IotDBServiceImpl.java | 162 +++++++++++++++------ 5 files changed, 188 insertions(+), 76 deletions(-) diff --git a/backend/src/main/java/org/apache/iotdb/admin/controller/IotDBController.java b/backend/src/main/java/org/apache/iotdb/admin/controller/IotDBController.java index 8930913..67e4289 100644 --- a/backend/src/main/java/org/apache/iotdb/admin/controller/IotDBController.java +++ b/backend/src/main/java/org/apache/iotdb/admin/controller/IotDBController.java @@ -98,12 +98,12 @@ public class IotDBController { @GetMapping("/dataModel/detail") @ApiOperation("Get IoTDB data model in detail") public BaseVO<DataModelVO> getDataModelDetail( - @PathVariable("serverId") Integer serverId, - @RequestParam(value = "path", required = false, defaultValue = "root") String path, - @RequestParam(value = "pageSize", required = false, defaultValue = "10") Integer pageSize, - @RequestParam(value = "pageNum", required = false, defaultValue = "1") Integer pageNum, - HttpServletRequest request) - throws BaseException { + @PathVariable("serverId") Integer serverId, + @RequestParam(value = "path", required = false, defaultValue = "root") String path, + @RequestParam(value = "pageSize", required = false, defaultValue = "10") Integer pageSize, + @RequestParam(value = "pageNum", required = false, defaultValue = "1") Integer pageNum, + HttpServletRequest request) + throws BaseException { check(request, serverId); Connection connection = connectionService.getById(serverId); DataModelVO dataModelVO = iotDBService.getDataModelDetail(connection, path, pageSize, pageNum); @@ -113,11 +113,11 @@ public class IotDBController { @GetMapping("/storageGroups/info") @ApiOperation("Get information of the storage group list") public BaseVO<GroupInfoVO> getAllStorageGroupsInfo( - @PathVariable("serverId") Integer serverId, - @RequestParam(value = "pageSize", required = false, defaultValue = "15") Integer pageSize, - @RequestParam(value = "pageNum", required = false, defaultValue = "1") Integer pageNum, - HttpServletRequest request) - throws BaseException { + @PathVariable("serverId") Integer serverId, + @RequestParam(value = "pageSize", required = false, defaultValue = "15") Integer pageSize, + @RequestParam(value = "pageNum", required = false, defaultValue = "1") Integer pageNum, + HttpServletRequest request) + throws BaseException { check(request, serverId); Connection connection = connectionService.getById(serverId); List<String> groupNames = iotDBService.getAllStorageGroups(connection); @@ -162,8 +162,6 @@ public class IotDBController { String host = connection.getHost(); for (String groupName : groupNames) { StorageGroupVO storageGroupVO = new StorageGroupVO(); - Integer id = groupService.getGroupId(host, groupName); - storageGroupVO.setGroupId(id); storageGroupVO.setGroupName(groupName); storageGroupVOList.add(storageGroupVO); } @@ -193,9 +191,9 @@ public class IotDBController { Connection connection = connectionService.getById(serverId); Long ttl = groupDTO.getTtl(); String ttlUnit = groupDTO.getTtlUnit(); - checkTtl(ttl, ttlUnit); Integer groupId = groupDTO.getGroupId(); groupDTO.setGroupName(groupName); + List<String> groupNames = iotDBService.getAllStorageGroups(connection); if (groupId == null) { if (!groupNames.contains(groupDTO.getGroupName())) { @@ -206,6 +204,7 @@ public class IotDBController { groupService.updateStorageGroupInfo(connection, groupDTO); } if (ttl != null && ttlUnit != null) { + checkTtl(ttl, ttlUnit); if (ttl >= 0) { Long times = switchTime(ttlUnit); iotDBService.saveGroupTtl(connection, groupName, ttl * times); @@ -354,12 +353,19 @@ public class IotDBController { public BaseVO<NodeTreeVO> getDevicesTreeByGroup( @PathVariable("serverId") Integer serverId, @PathVariable("groupName") String groupName, + @RequestParam(value = "pageSize", required = false, defaultValue = "10") Integer pageSize, + @RequestParam(value = "pageNum", required = false, defaultValue = "1") Integer pageNum, HttpServletRequest request) throws BaseException { checkParameter(groupName); check(request, serverId); Connection connection = connectionService.getById(serverId); - NodeTreeVO deviceList = iotDBService.getDeviceList(connection, groupName); + NodeTreeVO deviceList = iotDBService.getDeviceList(connection, groupName, pageSize, pageNum); + if (deviceList == null) { + deviceList = new NodeTreeVO(groupName); + } + deviceList.setPageNum(pageNum); + deviceList.setPageSize(pageSize); return BaseVO.success("Get successfully", deviceList); } @@ -461,8 +467,8 @@ public class IotDBController { @PathVariable("serverId") Integer serverId, @PathVariable("groupName") String groupName, @PathVariable("deviceName") String deviceName, - @RequestParam("pageSize") Integer pageSize, - @RequestParam("pageNum") Integer pageNum, + @RequestParam(value = "pageSize", required = false, defaultValue = "10") Integer pageSize, + @RequestParam(value = "pageNum", required = false, defaultValue = "1") Integer pageNum, @RequestParam(value = "keyword", required = false) String keyword, HttpServletRequest request) throws BaseException { @@ -475,17 +481,26 @@ public class IotDBController { List<MeasurementVO> measurementVOList = new ArrayList<>(); String host = connection.getHost(); if (measurementDTOList != null) { + List<String> timeseriesList = new ArrayList<>(); + for (MeasurementDTO measurementDTO : measurementDTOList) { + timeseriesList.add(measurementDTO.getTimeseries()); + } + List<String> batchNewValue = + iotDBService.getBatchLastMeasurementValue(connection, timeseriesList); + List<String> batchDataCount = + iotDBService.getBatchDataCount(connection, deviceName, timeseriesList); + int index = 0; for (MeasurementDTO measurementDTO : measurementDTOList) { MeasurementVO measurementVO = new MeasurementVO(); BeanUtils.copyProperties(measurementDTO, measurementVO); String description = measurementService.getDescription(host, measurementDTO.getTimeseries()); - String newValue = - iotDBService.getLastMeasurementValue(connection, measurementDTO.getTimeseries()); - Integer dataCount = - iotDBService.getOneDataCount(connection, deviceName, measurementDTO.getTimeseries()); - measurementVO.setDataCount(dataCount); - measurementVO.setNewValue(newValue); + if (batchNewValue.size() != 0) { + measurementVO.setNewValue(batchNewValue.get(index)); + } + if (batchDataCount.size() != 0) { + measurementVO.setDataCount(Integer.parseInt(batchDataCount.get(index))); + } measurementVO.setDescription(description); ObjectMapper mapper = new ObjectMapper(); List<List<String>> tags = new ArrayList<>(); @@ -519,6 +534,7 @@ public class IotDBController { throw new BaseException(ErrorCode.GET_MSM_FAIL, ErrorCode.GET_MSM_FAIL_MSG); } measurementVOList.add(measurementVO); + index++; } } MeasuremtnInfoVO measuremtnInfoVO = new MeasuremtnInfoVO(); diff --git a/backend/src/main/java/org/apache/iotdb/admin/model/dto/DataModelDetailDTO.java b/backend/src/main/java/org/apache/iotdb/admin/model/dto/DataModelDetailDTO.java index b33cce1..58463b0 100644 --- a/backend/src/main/java/org/apache/iotdb/admin/model/dto/DataModelDetailDTO.java +++ b/backend/src/main/java/org/apache/iotdb/admin/model/dto/DataModelDetailDTO.java @@ -1,8 +1,9 @@ package org.apache.iotdb.admin.model.dto; -import lombok.Data; import org.apache.iotdb.admin.model.vo.DataModelVO; +import lombok.Data; + import java.io.Serializable; import java.util.List; diff --git a/backend/src/main/java/org/apache/iotdb/admin/model/vo/NodeTreeVO.java b/backend/src/main/java/org/apache/iotdb/admin/model/vo/NodeTreeVO.java index cfa4acc..46a476b 100644 --- a/backend/src/main/java/org/apache/iotdb/admin/model/vo/NodeTreeVO.java +++ b/backend/src/main/java/org/apache/iotdb/admin/model/vo/NodeTreeVO.java @@ -31,10 +31,19 @@ public class NodeTreeVO implements Serializable { private List<NodeTreeVO> children; + private Integer pageSize; + + private Integer pageNum; + + private Integer total; + // private List<String> childrenName; + public NodeTreeVO(String name) { this.name = name; } + public NodeTreeVO() {} + public List<NodeTreeVO> initChildren() { if (children == null) { children = new ArrayList<>(); diff --git a/backend/src/main/java/org/apache/iotdb/admin/service/IotDBService.java b/backend/src/main/java/org/apache/iotdb/admin/service/IotDBService.java index 2a97117..4c5eff4 100644 --- a/backend/src/main/java/org/apache/iotdb/admin/service/IotDBService.java +++ b/backend/src/main/java/org/apache/iotdb/admin/service/IotDBService.java @@ -62,9 +62,6 @@ public interface IotDBService { void setIotDBRole(Connection connection, IotDBRole iotDBRole) throws BaseException; - DataModelVO getDataModelDetail( - Connection connection, String path, Integer pageSize, Integer pageNum) throws BaseException; - UserRolesVO getRolesOfUser(Connection connection, String userName) throws BaseException; void userGrant(Connection connection, String userName, UserGrantDTO userGrantDTO) @@ -122,7 +119,9 @@ public interface IotDBService { List<NodeTreeVO> getDeviceNodeTree(Connection connection, String groupName) throws BaseException; - NodeTreeVO getDeviceList(Connection connection, String groupName) throws BaseException; + NodeTreeVO getDeviceList( + Connection connection, String groupName, Integer pageSize, Integer pageNum) + throws BaseException; List<String> getDeviceParents(Connection connection, String groupName, String deviceName) throws BaseException; @@ -155,6 +154,8 @@ public interface IotDBService { Connection connection, String userOrRole, String name, PrivilegeInfoDTO privilegeInfoDTO) throws BaseException; + public List<QueryMetricsVO> getSlowQueryMetricsData(); + RecordVO getRecords( Connection connection, String deviceName, String timeseriesName, String dataType) throws BaseException; @@ -164,9 +165,7 @@ public interface IotDBService { void updatePwd(Connection connection, IotDBUser iotDBUser) throws BaseException; - void stopQuery(Integer serverId, Long timestamp) throws BaseException; - - QueryInfoDTO getQueryInfoListByQueryClassificationId( + public QueryInfoDTO getQueryInfoListByQueryClassificationId( Connection connection, Integer queryClassificationId, Integer pageSize, @@ -177,10 +176,19 @@ public interface IotDBService { Integer executionResult) throws BaseException; - MetricsDataForDiagramVO getMetricDataByMetricId(Connection connection, Integer metricId) + public List<QueryMetricsVO> getTopQueryMetricsData(); + + public MetricsDataForDiagramVO getMetricDataByMetricId(Connection connection, Integer metricId) throws BaseException; - List<QueryMetricsVO> getTopQueryMetricsData(); + void stopQuery(Integer serverId, Long timestamp) throws BaseException; + + DataModelVO getDataModelDetail( + Connection connection, String path, Integer pageSize, Integer pageNum) throws BaseException; + + List<String> getBatchLastMeasurementValue(Connection connection, List<String> timeseriesList) + throws BaseException; - List<QueryMetricsVO> getSlowQueryMetricsData(); + List<String> getBatchDataCount( + Connection connection, String deviceName, List<String> timeseriesList) throws BaseException; } diff --git a/backend/src/main/java/org/apache/iotdb/admin/service/impl/IotDBServiceImpl.java b/backend/src/main/java/org/apache/iotdb/admin/service/impl/IotDBServiceImpl.java index 4318825..c174d0e 100644 --- a/backend/src/main/java/org/apache/iotdb/admin/service/impl/IotDBServiceImpl.java +++ b/backend/src/main/java/org/apache/iotdb/admin/service/impl/IotDBServiceImpl.java @@ -151,10 +151,11 @@ public class IotDBServiceImpl implements IotDBService { sessionPool = getSessionPool(connection); DataModelVO root = new DataModelVO(path); setNodeInfo(root, sessionPool, path); - List<DataModelVO> childrenDataModel = getChildrenDataModel(root, path, sessionPool); + List<DataModelVO> childrenDataModel = getChildrenDataModel(root, path, sessionPool, 20); root.setChildren(childrenDataModel); root.setGroupCount(path.equals("root") ? getGroupCount(sessionPool) : null); root.setPath(path); + root.setShowNum(20); return root; } finally { closeSessionPool(sessionPool); @@ -162,15 +163,21 @@ public class IotDBServiceImpl implements IotDBService { } private List<DataModelVO> getChildrenDataModel( - DataModelVO root, String path, SessionPool sessionPool) throws BaseException { + DataModelVO root, String path, SessionPool sessionPool, Integer showNum) + throws BaseException { Set<String> childrenNode = getChildrenNode(path, sessionPool); if (childrenNode == null) { return null; } + List<String> childrenNodeList = new ArrayList<>(childrenNode); + List<String> childrenNodeSubList = new ArrayList<>(); + if (childrenNodeList.size() > showNum) { + childrenNodeSubList = childrenNodeList.subList(0, showNum); + } else { + childrenNodeSubList = childrenNodeList; + } List<DataModelVO> childrenlist = new ArrayList<>(); - - // TODO: 大量IO - for (String child : childrenNode) { + for (String child : childrenNodeSubList) { DataModelVO childNode = new DataModelVO(child); setNodeInfo(childNode, sessionPool, path + "." + child); childrenlist.add(childNode); @@ -561,7 +568,15 @@ public class IotDBServiceImpl implements IotDBService { Connection connection, String deviceName, Integer pageSize, Integer pageNum, String keyword) throws BaseException { SessionPool sessionPool = getSessionPool(connection); + String queryCountSql = "count timeseries " + deviceName; + String s = executeQueryOneValue(sessionPool, queryCountSql); + int size = Integer.parseInt(s); String sql = "show timeseries " + deviceName; + int pageStart = pageNum == 1 ? 0 : (pageNum - 1) * pageSize; + int pageEnd = size < pageNum * pageSize ? size : pageNum * pageSize; + if (size > pageStart) { + sql = "show timeseries " + deviceName + " limit " + pageSize + " offset " + pageStart; + } SessionDataSetWrapper sessionDataSetWrapper = null; try { sessionDataSetWrapper = sessionPool.executeQueryStatement(sql); @@ -582,37 +597,39 @@ public class IotDBServiceImpl implements IotDBService { } else { continue; } - if (count >= pageSize * (pageNum - 1) + 1 && count <= pageSize * pageNum) { - MeasurementDTO t = new MeasurementDTO(); - List<String> columnNames = sessionDataSetWrapper.getColumnNames(); - for (int i = 0; i < fields.size(); i++) { - Field field = - MeasurementDTO.class.getDeclaredField(columnNames.get(i).replaceAll(" ", "")); - field.setAccessible(true); - field.set(t, fields.get(i).toString()); - } - results.add(t); + // if (count >= pageSize * (pageNum - 1) + 1 && count <= pageSize * pageNum) + // { + MeasurementDTO t = new MeasurementDTO(); + List<String> columnNames = sessionDataSetWrapper.getColumnNames(); + for (int i = 0; i < fields.size(); i++) { + Field field = + MeasurementDTO.class.getDeclaredField(columnNames.get(i).replaceAll(" ", "")); + field.setAccessible(true); + field.set(t, fields.get(i).toString()); } + results.add(t); + // } } else { count++; - if (count >= pageSize * (pageNum - 1) + 1 && count <= pageSize * pageNum) { - MeasurementDTO t = new MeasurementDTO(); - List<String> columnNames = sessionDataSetWrapper.getColumnNames(); - for (int i = 0; i < fields.size(); i++) { - Field field = - MeasurementDTO.class.getDeclaredField(columnNames.get(i).replaceAll(" ", "")); - field.setAccessible(true); - field.set(t, fields.get(i).toString()); - } - results.add(t); + // if (count >= pageSize * (pageNum - 1) + 1 && count <= pageSize * pageNum) + // { + MeasurementDTO t = new MeasurementDTO(); + List<String> columnNames = sessionDataSetWrapper.getColumnNames(); + for (int i = 0; i < fields.size(); i++) { + Field field = + MeasurementDTO.class.getDeclaredField(columnNames.get(i).replaceAll(" ", "")); + field.setAccessible(true); + field.set(t, fields.get(i).toString()); } + results.add(t); + // } } } } CountDTO countDTO = new CountDTO(); countDTO.setObjects(results); - countDTO.setTotalCount(count); - Integer totalPage = count % pageSize == 0 ? count / pageSize : count / pageSize + 1; + countDTO.setTotalCount(size); + Integer totalPage = size % pageSize == 0 ? size / pageSize : size / pageSize + 1; countDTO.setTotalPage(totalPage); return countDTO; } catch (IoTDBConnectionException e) { @@ -767,7 +784,7 @@ public class IotDBServiceImpl implements IotDBService { @Override public DataModelVO getDataModelDetail( - Connection connection, String path, Integer pageSize, Integer pageNum) throws BaseException { + Connection connection, String path, Integer pageSize, Integer pageNum) throws BaseException { SessionPool sessionPool = null; try { sessionPool = getSessionPool(connection); @@ -775,9 +792,9 @@ public class IotDBServiceImpl implements IotDBService { setNodeInfo(root, sessionPool, path); List<DataModelVO> childrenDataModel = null; DataModelDetailDTO childrenDataModelDetail = - getChildrenDataModelDetail(root, path, sessionPool, pageSize, pageNum); + getChildrenDataModelDetail(root, path, sessionPool, pageSize, pageNum); childrenDataModel = - childrenDataModelDetail == null ? null : childrenDataModelDetail.getDataModelVOList(); + childrenDataModelDetail == null ? null : childrenDataModelDetail.getDataModelVOList(); if (childrenDataModelDetail != null) { root.setPageNum(childrenDataModelDetail.getPageNum()); root.setPageSize(childrenDataModelDetail.getPageSize()); @@ -785,9 +802,9 @@ public class IotDBServiceImpl implements IotDBService { } root.setChildren(childrenDataModel); root.setTotalSonNodeCount( - getChildrenNode(path, sessionPool) == null - ? 0 - : getChildrenNode(path, sessionPool).size()); + getChildrenNode(path, sessionPool) == null + ? 0 + : getChildrenNode(path, sessionPool).size()); root.setGroupCount(path.equals("root") ? getGroupCount(sessionPool) : null); root.setPath(path); return root; @@ -796,9 +813,57 @@ public class IotDBServiceImpl implements IotDBService { } } + @Override + public List<String> getBatchLastMeasurementValue( + Connection connection, List<String> timeseriesList) throws BaseException { + SessionPool sessionPool = getSessionPool(connection); + List<Integer> indexList = new ArrayList<>(); + for (String timeseries : timeseriesList) { + indexList.add(timeseries.lastIndexOf(".")); + } + String sql = "select "; + for (int i = 0; i < timeseriesList.size(); i++) { + sql += "last_value(" + timeseriesList.get(i).substring(indexList.get(i) + 1) + ")" + ", "; + } + sql = sql.substring(0, sql.length() - 2); + sql += " from "; + sql += timeseriesList.get(0).substring(0, indexList.get(0)); + List<String> values; + try { + values = executeQueryOneLine(sessionPool, sql); + } finally { + closeSessionPool(sessionPool); + } + return values; + } + + @Override + public List<String> getBatchDataCount( + Connection connection, String deviceName, List<String> timeseriesList) throws BaseException { + SessionPool sessionPool = getSessionPool(connection); + List<Integer> indexList = new ArrayList<>(); + for (String timeseries : timeseriesList) { + indexList.add(timeseries.lastIndexOf(".")); + } + String sql = "select "; + for (int i = 0; i < timeseriesList.size(); i++) { + sql += "count(" + timeseriesList.get(i).substring(indexList.get(i) + 1) + ")" + ", "; + } + sql = sql.substring(0, sql.length() - 2); + sql += " from "; + sql += timeseriesList.get(0).substring(0, indexList.get(0)); + List<String> values; + try { + values = executeQueryOneLine(sessionPool, sql); + } finally { + closeSessionPool(sessionPool); + } + return values; + } + private DataModelDetailDTO getChildrenDataModelDetail( - DataModelVO root, String path, SessionPool sessionPool, Integer pageSize, Integer pageNum) - throws BaseException { + DataModelVO root, String path, SessionPool sessionPool, Integer pageSize, Integer pageNum) + throws BaseException { Set<String> childrenNode = getChildrenNode(path, sessionPool); if (childrenNode == null) { return null; @@ -1633,9 +1698,7 @@ public class IotDBServiceImpl implements IotDBService { private void upsertMeasurementAlias(SessionPool sessionPool, String timeseries, String alias) throws BaseException { - // 需要改为" "值。 - if (alias == null || "null".equals(alias)) { - // if (alias == null || "null".equals(alias) || StringUtils.isBlank(alias)) { + if (alias == null || "null".equals(alias) || StringUtils.isBlank(alias)) { return; } if (alias.matches("^as$") || alias.matches("^\\d+$") || alias.matches("^like$")) { @@ -1850,7 +1913,9 @@ public class IotDBServiceImpl implements IotDBService { } @Override - public NodeTreeVO getDeviceList(Connection connection, String groupName) throws BaseException { + public NodeTreeVO getDeviceList( + Connection connection, String groupName, Integer pageSize, Integer pageNum) + throws BaseException { SessionPool sessionPool = null; try { sessionPool = getSessionPool(connection); @@ -1863,7 +1928,9 @@ public class IotDBServiceImpl implements IotDBService { ancestryName = groupName; } NodeTreeVO ancestry = new NodeTreeVO(ancestryName); - assembleDeviceList(ancestry, groupName, sessionPool); + assembleDeviceList(ancestry, groupName, sessionPool, pageSize, pageNum); + ancestry.setName(groupName); + ancestry.setTotal(devices.size()); return ancestry; } finally { closeSessionPool(sessionPool); @@ -1891,17 +1958,28 @@ public class IotDBServiceImpl implements IotDBService { } } - private void assembleDeviceList(NodeTreeVO node, String deviceName, SessionPool sessionPool) + private void assembleDeviceList( + NodeTreeVO node, + String deviceName, + SessionPool sessionPool, + Integer pageSize, + Integer pageNum) throws BaseException { List<String> descendants = findDescendants(deviceName, sessionPool); if (descendants.size() == 0) { return; } List<String> children = findChildren(descendants); + int size = children.size(); + int pageStart = pageNum == 1 ? 0 : (pageNum - 1) * pageSize; + int pageEnd = size < pageNum * pageSize ? size : pageNum * pageSize; + if (size > pageStart) { + children = children.subList(pageStart, pageEnd); + } for (String child : children) { NodeTreeVO childNode = new NodeTreeVO(child); node.initChildren().add(childNode); - assembleDeviceList(childNode, child, sessionPool); + // assembleDeviceList(childNode, child, sessionPool); } }
