This is an automated email from the ASF dual-hosted git repository.
siyao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 7b5a6b100c HDDS-9087. Ozone Recon Heatmap code refactoring and Bucket
level access count fix (#5125)
7b5a6b100c is described below
commit 7b5a6b100c093e2c5f4d4417243d81d6fd81420f
Author: devmadhuu <[email protected]>
AuthorDate: Fri Jul 28 05:23:29 2023 +0530
HDDS-9087. Ozone Recon Heatmap code refactoring and Bucket level access
count fix (#5125)
---
.../ozone/recon/heatmap/HeatMapServiceImpl.java | 444 +--------------------
.../{HeatMapServiceImpl.java => HeatMapUtil.java} | 339 +++++++---------
.../ozone/recon/heatmap/TestHeatMapInfo.java | 10 +-
3 files changed, 160 insertions(+), 633 deletions(-)
diff --git
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/heatmap/HeatMapServiceImpl.java
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/heatmap/HeatMapServiceImpl.java
index b3d2904f86..73868077e5 100644
---
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/heatmap/HeatMapServiceImpl.java
+++
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/heatmap/HeatMapServiceImpl.java
@@ -20,26 +20,15 @@
package org.apache.hadoop.ozone.recon.heatmap;
import com.google.inject.Inject;
-import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager;
-import org.apache.hadoop.ozone.recon.api.handlers.EntityHandler;
-import org.apache.hadoop.ozone.recon.api.types.DUResponse;
-import org.apache.hadoop.ozone.recon.api.types.EntityMetaData;
import org.apache.hadoop.ozone.recon.api.types.EntityReadAccessHeatMapResponse;
-import org.apache.hadoop.ozone.recon.api.types.ResponseStatus;
import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager;
import org.apache.hadoop.ozone.recon.spi.ReconNamespaceSummaryManager;
-import org.jetbrains.annotations.NotNull;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.List;
-import java.util.stream.Collectors;
-
import static
org.apache.hadoop.hdds.recon.ReconConfigKeys.OZONE_RECON_HEATMAP_PROVIDER_KEY;
import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX;
@@ -55,6 +44,7 @@ public class HeatMapServiceImpl extends HeatMapService {
private final ReconOMMetadataManager omMetadataManager;
private final OzoneStorageContainerManager reconSCM;
private IHeatMapProvider heatMapProvider;
+ private HeatMapUtil heatMapUtil;
@Inject
public HeatMapServiceImpl(OzoneConfiguration ozoneConfiguration,
@@ -66,6 +56,9 @@ public class HeatMapServiceImpl extends HeatMapService {
this.reconNamespaceSummaryManager = namespaceSummaryManager;
this.omMetadataManager = omMetadataManager;
this.reconSCM = reconSCM;
+ heatMapUtil =
+ new HeatMapUtil(reconNamespaceSummaryManager, omMetadataManager,
+ reconSCM, ozoneConfiguration);
initializeProvider();
}
@@ -75,7 +68,7 @@ public class HeatMapServiceImpl extends HeatMapService {
LOG.info("HeatMapProvider: {}", heatMapProviderCls);
if (!StringUtils.isEmpty(heatMapProviderCls)) {
try {
- heatMapProvider = loadHeatMapProvider(heatMapProviderCls);
+ heatMapProvider = heatMapUtil.loadHeatMapProvider(heatMapProviderCls);
} catch (Exception e) {
LOG.error("Loading HeatMapProvider fails!!! : {}", e);
return;
@@ -99,8 +92,9 @@ public class HeatMapServiceImpl extends HeatMapService {
String path,
String entityType,
String startDate) throws Exception {
- return retrieveDataAndGenerateHeatMap(validatePath(path), entityType,
- startDate);
+ return heatMapUtil.retrieveDataAndGenerateHeatMap(heatMapProvider,
+ validatePath(path),
+ entityType, startDate);
}
private String validatePath(String path) {
@@ -110,426 +104,4 @@ public class HeatMapServiceImpl extends HeatMapService {
return path;
}
- private void addBucketData(
- EntityReadAccessHeatMapResponse rootEntity,
- EntityReadAccessHeatMapResponse volumeEntity, String[] split,
- int readAccessCount, long keySize) {
- List<EntityReadAccessHeatMapResponse> children =
- volumeEntity.getChildren();
- EntityReadAccessHeatMapResponse bucketEntity = null;
- List<EntityReadAccessHeatMapResponse> bucketList =
- children.stream().filter(entity -> entity.getLabel().
- equalsIgnoreCase(split[1])).collect(Collectors.toList());
- if (bucketList.size() > 0) {
- bucketEntity = bucketList.get(0);
- }
- if (children.contains(bucketEntity)) {
- addPrefixPathInfoToBucket(rootEntity, split, bucketEntity,
- readAccessCount, keySize);
- } else {
- addBucketAndPrefixPath(split, rootEntity, volumeEntity, readAccessCount,
- keySize);
- }
- }
-
- private void addVolumeData(
- EntityReadAccessHeatMapResponse rootEntity,
- String[] split, int readAccessCount, long entitySize) {
- List<EntityReadAccessHeatMapResponse> children =
- rootEntity.getChildren();
- EntityReadAccessHeatMapResponse volumeInfo =
- new EntityReadAccessHeatMapResponse();
- volumeInfo.setLabel(split[0]);
- volumeInfo.setPath(split[0]);
- children.add(volumeInfo);
- if (split.length < 2) {
- volumeInfo.setSize(entitySize);
- volumeInfo.setAccessCount(readAccessCount);
- volumeInfo.setMinAccessCount(readAccessCount);
- volumeInfo.setMaxAccessCount(readAccessCount);
- return;
- }
- addBucketAndPrefixPath(split, rootEntity, volumeInfo, readAccessCount,
- entitySize);
- }
-
- private void updateVolumeSize(
- EntityReadAccessHeatMapResponse volumeInfo) {
- List<EntityReadAccessHeatMapResponse> children =
- volumeInfo.getChildren();
- children.stream().forEach(bucket -> {
- volumeInfo.setSize(volumeInfo.getSize() + bucket.getSize());
- updateBucketLevelMinMaxAccessCount(bucket);
- });
- }
-
- private void updateEntityAccessRatio(EntityReadAccessHeatMapResponse entity)
{
- long delta = entity.getMaxAccessCount() - entity.getMinAccessCount();
- List<EntityReadAccessHeatMapResponse> children =
- entity.getChildren();
- children.stream().forEach(path -> {
- if (path.getChildren().size() != 0) {
- updateEntityAccessRatio(path);
- } else {
- path.setColor(1.000);
- long accessCount = path.getAccessCount();
- accessCount =
- (accessCount == 0 ? path.getMinAccessCount() : accessCount);
- if (delta >= 0) {
- if (accessCount > 0) {
- double truncatedValue = truncate(
- ((double) accessCount /
- (double) entity.getMaxAccessCount()), 3);
- path.setColor(truncatedValue);
- }
- }
- }
- });
- }
-
- private static double truncate(double value, int decimalPlaces) {
- if (decimalPlaces < 0) {
- throw new IllegalArgumentException();
- }
- value = value * Math.pow(10, decimalPlaces);
- value = Math.floor(value);
- value = value / Math.pow(10, decimalPlaces);
- return value;
- }
-
- private void updateRootEntitySize(
- EntityReadAccessHeatMapResponse rootEntity) {
- List<EntityReadAccessHeatMapResponse> children =
- rootEntity.getChildren();
- children.stream().forEach(volume -> {
- updateVolumeSize(volume);
- updateVolumeLevelMinMaxAccessCount(volume);
- setEntityLevelAccessCount(volume);
- rootEntity.setSize(rootEntity.getSize() + volume.getSize());
- });
- }
-
- private void addBucketAndPrefixPath(
- String[] split, EntityReadAccessHeatMapResponse rootEntity,
- EntityReadAccessHeatMapResponse volumeEntity,
- long readAccessCount, long keySize) {
- List<EntityReadAccessHeatMapResponse> bucketEntities =
- volumeEntity.getChildren();
- EntityReadAccessHeatMapResponse bucket =
- new EntityReadAccessHeatMapResponse();
- bucket.setLabel(split[1]);
- bucket.setPath(omMetadataManager.getBucketKey(split[0], split[1]));
- bucketEntities.add(bucket);
- bucket.setMinAccessCount(readAccessCount);
- if (split.length > 2) {
- addPrefixPathInfoToBucket(rootEntity, split, bucket, readAccessCount,
- keySize);
- } else {
- updateBucketSize(bucket, keySize);
- }
- }
-
- private void setEntityLevelAccessCount(
- EntityReadAccessHeatMapResponse entity) {
- List<EntityReadAccessHeatMapResponse> children = entity.getChildren();
- children.stream().forEach(child -> {
- entity.setAccessCount(entity.getAccessCount() + child.getAccessCount());
- });
- // This is being taken as whole number
- if (entity.getAccessCount() > 0 && children.size() > 0) {
- entity.setAccessCount(entity.getAccessCount() / children.size());
- }
- }
-
- private void addPrefixPathInfoToBucket(
- EntityReadAccessHeatMapResponse rootEntity, String[] split,
- EntityReadAccessHeatMapResponse bucket,
- long readAccessCount, long keySize) {
- List<EntityReadAccessHeatMapResponse> prefixes = bucket.getChildren();
- updateBucketSize(bucket, keySize);
- String path = Arrays.stream(split)
- .skip(2).collect(Collectors.joining("/"));
- EntityReadAccessHeatMapResponse prefixPathInfo =
- new EntityReadAccessHeatMapResponse();
- prefixPathInfo.setLabel(path);
- prefixPathInfo.setPath(bucket.getPath() + OM_KEY_PREFIX + path);
- prefixPathInfo.setAccessCount(readAccessCount);
- prefixPathInfo.setSize(keySize);
- prefixes.add(prefixPathInfo);
- // This is done for specific ask by UI treemap to render and provide
- // varying color shades based on varying ranges of access count.
- updateRootLevelMinMaxAccessCount(readAccessCount, rootEntity);
- }
-
- private void updateVolumeLevelMinMaxAccessCount(
- EntityReadAccessHeatMapResponse volume) {
- List<EntityReadAccessHeatMapResponse>
- children = initializeEntityMinMaxCount(volume);
- children.stream().forEach(child -> {
- long bucketMinAccessCount = child.getMinAccessCount();
- long bucketMaxAccessCount = child.getMaxAccessCount();
- volume.setMinAccessCount(
- bucketMinAccessCount < volume.getMinAccessCount() ?
- bucketMinAccessCount :
- volume.getMinAccessCount());
- volume.setMaxAccessCount(
- bucketMaxAccessCount > volume.getMaxAccessCount() ?
- bucketMaxAccessCount :
- volume.getMaxAccessCount());
- });
- }
-
- @NotNull
- private static List<EntityReadAccessHeatMapResponse>
- initializeEntityMinMaxCount(
- EntityReadAccessHeatMapResponse entity) {
- List<EntityReadAccessHeatMapResponse> children =
- entity.getChildren();
- if (children.size() == 0) {
- entity.setMaxAccessCount(entity.getMinAccessCount());
- }
- if (children.size() > 0) {
- entity.setMinAccessCount(Long.MAX_VALUE);
- }
- return children;
- }
-
- private void updateBucketLevelMinMaxAccessCount(
- EntityReadAccessHeatMapResponse bucket) {
- List<EntityReadAccessHeatMapResponse>
- children = initializeEntityMinMaxCount(bucket);
- children.stream().forEach(path -> {
- long readAccessCount = path.getAccessCount();
- bucket.setMinAccessCount(
- path.getAccessCount() < bucket.getMinAccessCount() ? readAccessCount
:
- bucket.getMinAccessCount());
- bucket.setMaxAccessCount(
- readAccessCount > bucket.getMaxAccessCount() ? readAccessCount :
- bucket.getMaxAccessCount());
- });
- }
-
- private void updateRootLevelMinMaxAccessCount(
- long readAccessCount,
- EntityReadAccessHeatMapResponse rootEntity) {
- rootEntity.setMinAccessCount(
- readAccessCount < rootEntity.getMinAccessCount() ? readAccessCount :
- rootEntity.getMinAccessCount());
- rootEntity.setMaxAccessCount(
- readAccessCount > rootEntity.getMaxAccessCount() ? readAccessCount :
- rootEntity.getMaxAccessCount());
- }
-
- private void updateBucketSize(EntityReadAccessHeatMapResponse bucket,
- long keySize) {
- bucket.setSize(bucket.getSize() + keySize);
- }
-
- /**
- * Transforms the access metadata of entities (provided by heatmap provider)
- * by grouping them in their respective buckets and volumes in tree
structure
- * format which is easy for heatmap UI to consume.
- * Sample Data Format:
- * {
- * "label": "root",
- * "children": [
- * {
- * "label": "hivevol1676574631",
- * "children": [
- * {
- * "label": "hiveencbuck1676574631",
- * "children": [
- * {
- * "label": "enc_path/hive_tpcds/store_sales/store_sales.dat",
- * "size": 256,
- * "path": "/hivevol1676574631/hiveencbuck1676574631/enc_path/
- * hive_tpcds/store_sales/store_sales.dat",
- * "accessCount": 155074,
- * "color": 1
- * },
- * {
- * "label": "enc_path/hive_tpcds/catalog_sales/
- * catalog_sales.dat",
- * "size": 256,
- * "path": "/hivevol1676574631/hiveencbuck1676574631/enc_path/
- * hive_tpcds/catalog_sales/catalog_sales.dat",
- * "accessCount": 68567,
- * "color": 0.442
- * }
- * ],
- * "size": 3584,
- * "path": "/hivevol1676574631/hiveencbuck1676574631",
- * "minAccessCount": 2924,
- * "maxAccessCount": 155074
- * },
- * {
- * "label": "hivebuck1676574631",
- * "children": [
- * {
- * "label": "reg_path/hive_tpcds/store_sales/store_sales.dat",
- * "size": 256,
- * "path": "/hivevol1676574631/hivebuck1676574631/reg_path/
- * hive_tpcds/store_sales/store_sales.dat",
- * "accessCount": 155069,
- * "color": 1
- * },
- * {
- * "label": "reg_path/hive_tpcds/catalog_sales/
- * catalog_sales.dat",
- * "size": 256,
- * "path": "/hivevol1676574631/hivebuck1676574631/reg_path/
- * hive_tpcds/catalog_sales/catalog_sales.dat",
- * "accessCount": 68566,
- * "color": 0.442
- * }
- * ],
- * "size": 3584,
- * "path": "/hivevol1676574631/hivebuck1676574631",
- * "minAccessCount": 2924,
- * "maxAccessCount": 155069
- * }
- * ],
- * "size": 7168,
- * "path": "/hivevol1676574631"
- * },
- * {
- * "label": "hivevol1675429570",
- * "children": [
- * {
- * "label": "hivebuck1675429570",
- * "children": [
- * {
- * "label": "reg_path/hive_tpcds/store_sales/store_sales.dat",
- * "size": 256,
- * "path": "/hivevol1675429570/hivebuck1675429570/reg_path/
- * hive_tpcds/store_sales/store_sales.dat",
- * "accessCount": 129977,
- * "color": 1
- * } ],
- * "size": 3072,
- * "path": "/hivevol1675429570/hivebuck1675429570",
- * "minAccessCount": 3195,
- * "maxAccessCount": 129977
- * }
- * ],
- * "size": 6144,
- * "path": "/hivevol1675429570"
- * }
- * ],
- * "size": 25600,
- * "minAccessCount": 2924,
- * "maxAccessCount": 155074
- * }
- * @param entityMetaDataList list of entity's access metadata objects
- * @return transformed data to be consumed by heatmap UI.
- */
- public EntityReadAccessHeatMapResponse generateHeatMap(
- List<EntityMetaData> entityMetaDataList) {
- EntityReadAccessHeatMapResponse rootEntity =
- new EntityReadAccessHeatMapResponse();
- rootEntity.setMinAccessCount(
- entityMetaDataList.get(0).getReadAccessCount());
- rootEntity.setLabel("root");
- rootEntity.setPath(OM_KEY_PREFIX);
- List<EntityReadAccessHeatMapResponse> children =
- rootEntity.getChildren();
- entityMetaDataList.forEach(entityMetaData -> {
- String path = entityMetaData.getVal();
- String[] split = path.split("/");
- if (split.length == 0) {
- return;
- }
- long entitySize = 0;
- try {
- entitySize = getEntitySize(path);
- } catch (IOException e) {
- LOG.error("IOException while getting key size for key : " +
- "{} - {}", path, e);
- }
- EntityReadAccessHeatMapResponse volumeEntity = null;
- List<EntityReadAccessHeatMapResponse> volumeList =
- children.stream().filter(entity -> entity.getLabel().
- equalsIgnoreCase(split[0])).collect(Collectors.toList());
- if (volumeList.size() > 0) {
- volumeEntity = volumeList.get(0);
- }
- if (null != volumeEntity) {
- if (validateLength(split, 2)) {
- return;
- }
- addBucketData(rootEntity, volumeEntity, split,
- entityMetaData.getReadAccessCount(), entitySize);
- } else {
- if (validateLength(split, 1)) {
- return;
- }
- addVolumeData(rootEntity, split,
- entityMetaData.getReadAccessCount(), entitySize);
- }
- });
- updateRootEntitySize(rootEntity);
- updateVolumeLevelMinMaxAccessCount(rootEntity);
- updateEntityAccessRatio(rootEntity);
- return rootEntity;
- }
-
- private static boolean validateLength(String[] split, int minLength) {
- return (split.length < minLength);
- }
-
- private long getEntitySize(String path) throws IOException {
- long entitySize = 0;
- LOG.info("Getting entity size for {}: ", path);
- EntityHandler entityHandler =
- EntityHandler.getEntityHandler(reconNamespaceSummaryManager,
- omMetadataManager, reconSCM, path);
- if (null != entityHandler) {
- DUResponse duResponse = entityHandler.getDuResponse(false, false);
- if (null != duResponse && duResponse.getStatus() == ResponseStatus.OK) {
- return duResponse.getSize();
- }
- }
- // returning some default value due to some issue
- return 256L;
- }
-
- public EntityReadAccessHeatMapResponse retrieveDataAndGenerateHeatMap(
- String normalizePath,
- String entityType,
- String startDate) throws Exception {
- if (null != heatMapProvider) {
- List<EntityMetaData> entityMetaDataList = heatMapProvider
- .retrieveData(normalizePath, entityType, startDate);
- if (null != entityMetaDataList &&
- (CollectionUtils.isNotEmpty(entityMetaDataList))) {
- // Transforms and return heatmap data by grouping access metadata of
- // entities in their respective buckets and volumes in tree structure.
- // Refer the javadoc for more details.ß
- return generateHeatMap(entityMetaDataList);
- }
- }
- return new EntityReadAccessHeatMapResponse();
- }
-
- /**
- * This method loads heatMapProvider implementation class.
- *
- * @param className - load the class and instantiate object.
- * @return the implementation class object of IHeatMapProvider
- * @throws Exception
- */
- public static IHeatMapProvider loadHeatMapProvider(String className)
- throws Exception {
- try {
- Class<?> clazz = Class.forName(className);
- Object o = clazz.newInstance();
- if (o instanceof IHeatMapProvider) {
- return (IHeatMapProvider) o;
- }
- return null;
- } catch (ClassNotFoundException | InstantiationException |
- IllegalAccessException e) {
- throw new Exception(e);
- }
- }
}
diff --git
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/heatmap/HeatMapServiceImpl.java
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/heatmap/HeatMapUtil.java
similarity index 89%
copy from
hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/heatmap/HeatMapServiceImpl.java
copy to
hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/heatmap/HeatMapUtil.java
index b3d2904f86..b885ea5ada 100644
---
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/heatmap/HeatMapServiceImpl.java
+++
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/heatmap/HeatMapUtil.java
@@ -21,7 +21,6 @@ package org.apache.hadoop.ozone.recon.heatmap;
import com.google.inject.Inject;
import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager;
import org.apache.hadoop.ozone.recon.api.handlers.EntityHandler;
@@ -40,74 +39,49 @@ import java.util.Arrays;
import java.util.List;
import java.util.stream.Collectors;
-import static
org.apache.hadoop.hdds.recon.ReconConfigKeys.OZONE_RECON_HEATMAP_PROVIDER_KEY;
import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX;
/**
- * This class is an implementation of abstract class for retrieving
- * data through HeatMapService.
+ * This class is general utility class for keeping heatmap utility functions.
*/
-public class HeatMapServiceImpl extends HeatMapService {
+public class HeatMapUtil {
private static final Logger LOG =
- LoggerFactory.getLogger(HeatMapServiceImpl.class);
- private final OzoneConfiguration ozoneConfiguration;
+ LoggerFactory.getLogger(HeatMapUtil.class);
+ private OzoneConfiguration ozoneConfiguration;
private final ReconNamespaceSummaryManager reconNamespaceSummaryManager;
private final ReconOMMetadataManager omMetadataManager;
private final OzoneStorageContainerManager reconSCM;
- private IHeatMapProvider heatMapProvider;
@Inject
- public HeatMapServiceImpl(OzoneConfiguration ozoneConfiguration,
- ReconNamespaceSummaryManager
- namespaceSummaryManager,
- ReconOMMetadataManager omMetadataManager,
- OzoneStorageContainerManager reconSCM) {
- this.ozoneConfiguration = ozoneConfiguration;
+ public HeatMapUtil(ReconNamespaceSummaryManager
+ namespaceSummaryManager,
+ ReconOMMetadataManager omMetadataManager,
+ OzoneStorageContainerManager reconSCM,
+ OzoneConfiguration ozoneConfiguration) {
this.reconNamespaceSummaryManager = namespaceSummaryManager;
this.omMetadataManager = omMetadataManager;
this.reconSCM = reconSCM;
- initializeProvider();
+ this.ozoneConfiguration = ozoneConfiguration;
}
- private void initializeProvider() {
- String heatMapProviderCls =
- ozoneConfiguration.get(OZONE_RECON_HEATMAP_PROVIDER_KEY);
- LOG.info("HeatMapProvider: {}", heatMapProviderCls);
- if (!StringUtils.isEmpty(heatMapProviderCls)) {
- try {
- heatMapProvider = loadHeatMapProvider(heatMapProviderCls);
- } catch (Exception e) {
- LOG.error("Loading HeatMapProvider fails!!! : {}", e);
- return;
- }
- if (null != heatMapProvider) {
- try {
- heatMapProvider.init(ozoneConfiguration, omMetadataManager,
- reconNamespaceSummaryManager, reconSCM);
- } catch (Exception e) {
- LOG.error("Initializing HeatMapProvider fails!!! : {}", e);
- heatMapProvider = null;
- }
- } else {
- LOG.error("Loading HeatMapProvider fails!!!");
+ private long getEntitySize(String path) throws IOException {
+ long entitySize = 0;
+ LOG.info("Getting entity size for {}: ", path);
+ EntityHandler entityHandler =
+ EntityHandler.getEntityHandler(reconNamespaceSummaryManager,
+ omMetadataManager, reconSCM, path);
+ if (null != entityHandler) {
+ DUResponse duResponse = entityHandler.getDuResponse(false, false);
+ if (null != duResponse && duResponse.getStatus() == ResponseStatus.OK) {
+ return duResponse.getSize();
}
}
+ // returning some default value due to some issue
+ return 256L;
}
- @Override
- public EntityReadAccessHeatMapResponse retrieveData(
- String path,
- String entityType,
- String startDate) throws Exception {
- return retrieveDataAndGenerateHeatMap(validatePath(path), entityType,
- startDate);
- }
-
- private String validatePath(String path) {
- if (null != path && path.startsWith(OM_KEY_PREFIX)) {
- path = path.substring(1);
- }
- return path;
+ private static boolean validateLength(String[] split, int minLength) {
+ return (split.length < minLength);
}
private void addBucketData(
@@ -132,6 +106,63 @@ public class HeatMapServiceImpl extends HeatMapService {
}
}
+ private void updateRootLevelMinMaxAccessCount(
+ long readAccessCount,
+ EntityReadAccessHeatMapResponse rootEntity) {
+ rootEntity.setMinAccessCount(
+ readAccessCount < rootEntity.getMinAccessCount() ? readAccessCount :
+ rootEntity.getMinAccessCount());
+ rootEntity.setMaxAccessCount(
+ readAccessCount > rootEntity.getMaxAccessCount() ? readAccessCount :
+ rootEntity.getMaxAccessCount());
+ }
+
+ private void updateBucketSize(EntityReadAccessHeatMapResponse bucket,
+ long keySize) {
+ bucket.setSize(bucket.getSize() + keySize);
+ }
+
+
+ private void addPrefixPathInfoToBucket(
+ EntityReadAccessHeatMapResponse rootEntity, String[] split,
+ EntityReadAccessHeatMapResponse bucket,
+ long readAccessCount, long keySize) {
+ List<EntityReadAccessHeatMapResponse> prefixes = bucket.getChildren();
+ updateBucketSize(bucket, keySize);
+ String path = Arrays.stream(split)
+ .skip(2).collect(Collectors.joining("/"));
+ EntityReadAccessHeatMapResponse prefixPathInfo =
+ new EntityReadAccessHeatMapResponse();
+ prefixPathInfo.setLabel(path);
+ prefixPathInfo.setPath(bucket.getPath() + OM_KEY_PREFIX + path);
+ prefixPathInfo.setAccessCount(readAccessCount);
+ prefixPathInfo.setSize(keySize);
+ prefixes.add(prefixPathInfo);
+ // This is done for specific ask by UI treemap to render and provide
+ // varying color shades based on varying ranges of access count.
+ updateRootLevelMinMaxAccessCount(readAccessCount, rootEntity);
+ }
+
+ private void addBucketAndPrefixPath(
+ String[] split, EntityReadAccessHeatMapResponse rootEntity,
+ EntityReadAccessHeatMapResponse volumeEntity,
+ long readAccessCount, long keySize) {
+ List<EntityReadAccessHeatMapResponse> bucketEntities =
+ volumeEntity.getChildren();
+ EntityReadAccessHeatMapResponse bucket =
+ new EntityReadAccessHeatMapResponse();
+ bucket.setLabel(split[1]);
+ bucket.setPath(omMetadataManager.getBucketKey(split[0], split[1]));
+ bucketEntities.add(bucket);
+ bucket.setMinAccessCount(readAccessCount);
+ if (split.length > 2) {
+ addPrefixPathInfoToBucket(rootEntity, split, bucket, readAccessCount,
+ keySize);
+ } else {
+ updateBucketSize(bucket, keySize);
+ }
+ }
+
private void addVolumeData(
EntityReadAccessHeatMapResponse rootEntity,
String[] split, int readAccessCount, long entitySize) {
@@ -153,6 +184,33 @@ public class HeatMapServiceImpl extends HeatMapService {
entitySize);
}
+ private void setEntityLevelAccessCount(
+ EntityReadAccessHeatMapResponse entity) {
+ List<EntityReadAccessHeatMapResponse> children = entity.getChildren();
+ children.stream().forEach(child -> {
+ entity.setAccessCount(entity.getAccessCount() + child.getAccessCount());
+ });
+ // This is being taken as whole number
+ if (entity.getAccessCount() > 0 && children.size() > 0) {
+ entity.setAccessCount(entity.getAccessCount() / children.size());
+ }
+ }
+
+ private void updateBucketLevelMinMaxAccessCount(
+ EntityReadAccessHeatMapResponse bucket) {
+ List<EntityReadAccessHeatMapResponse>
+ children = initializeEntityMinMaxCount(bucket);
+ children.stream().forEach(path -> {
+ long readAccessCount = path.getAccessCount();
+ bucket.setMinAccessCount(
+ path.getAccessCount() < bucket.getMinAccessCount() ? readAccessCount
:
+ bucket.getMinAccessCount());
+ bucket.setMaxAccessCount(
+ readAccessCount > bucket.getMaxAccessCount() ? readAccessCount :
+ bucket.getMaxAccessCount());
+ });
+ }
+
private void updateVolumeSize(
EntityReadAccessHeatMapResponse volumeInfo) {
List<EntityReadAccessHeatMapResponse> children =
@@ -163,40 +221,6 @@ public class HeatMapServiceImpl extends HeatMapService {
});
}
- private void updateEntityAccessRatio(EntityReadAccessHeatMapResponse entity)
{
- long delta = entity.getMaxAccessCount() - entity.getMinAccessCount();
- List<EntityReadAccessHeatMapResponse> children =
- entity.getChildren();
- children.stream().forEach(path -> {
- if (path.getChildren().size() != 0) {
- updateEntityAccessRatio(path);
- } else {
- path.setColor(1.000);
- long accessCount = path.getAccessCount();
- accessCount =
- (accessCount == 0 ? path.getMinAccessCount() : accessCount);
- if (delta >= 0) {
- if (accessCount > 0) {
- double truncatedValue = truncate(
- ((double) accessCount /
- (double) entity.getMaxAccessCount()), 3);
- path.setColor(truncatedValue);
- }
- }
- }
- });
- }
-
- private static double truncate(double value, int decimalPlaces) {
- if (decimalPlaces < 0) {
- throw new IllegalArgumentException();
- }
- value = value * Math.pow(10, decimalPlaces);
- value = Math.floor(value);
- value = value / Math.pow(10, decimalPlaces);
- return value;
- }
-
private void updateRootEntitySize(
EntityReadAccessHeatMapResponse rootEntity) {
List<EntityReadAccessHeatMapResponse> children =
@@ -209,56 +233,19 @@ public class HeatMapServiceImpl extends HeatMapService {
});
}
- private void addBucketAndPrefixPath(
- String[] split, EntityReadAccessHeatMapResponse rootEntity,
- EntityReadAccessHeatMapResponse volumeEntity,
- long readAccessCount, long keySize) {
- List<EntityReadAccessHeatMapResponse> bucketEntities =
- volumeEntity.getChildren();
- EntityReadAccessHeatMapResponse bucket =
- new EntityReadAccessHeatMapResponse();
- bucket.setLabel(split[1]);
- bucket.setPath(omMetadataManager.getBucketKey(split[0], split[1]));
- bucketEntities.add(bucket);
- bucket.setMinAccessCount(readAccessCount);
- if (split.length > 2) {
- addPrefixPathInfoToBucket(rootEntity, split, bucket, readAccessCount,
- keySize);
- } else {
- updateBucketSize(bucket, keySize);
- }
- }
-
- private void setEntityLevelAccessCount(
+ @NotNull
+ private static List<EntityReadAccessHeatMapResponse>
+ initializeEntityMinMaxCount(
EntityReadAccessHeatMapResponse entity) {
- List<EntityReadAccessHeatMapResponse> children = entity.getChildren();
- children.stream().forEach(child -> {
- entity.setAccessCount(entity.getAccessCount() + child.getAccessCount());
- });
- // This is being taken as whole number
- if (entity.getAccessCount() > 0 && children.size() > 0) {
- entity.setAccessCount(entity.getAccessCount() / children.size());
+ List<EntityReadAccessHeatMapResponse> children =
+ entity.getChildren();
+ if (children.size() == 0) {
+ entity.setMaxAccessCount(entity.getMinAccessCount());
}
- }
-
- private void addPrefixPathInfoToBucket(
- EntityReadAccessHeatMapResponse rootEntity, String[] split,
- EntityReadAccessHeatMapResponse bucket,
- long readAccessCount, long keySize) {
- List<EntityReadAccessHeatMapResponse> prefixes = bucket.getChildren();
- updateBucketSize(bucket, keySize);
- String path = Arrays.stream(split)
- .skip(2).collect(Collectors.joining("/"));
- EntityReadAccessHeatMapResponse prefixPathInfo =
- new EntityReadAccessHeatMapResponse();
- prefixPathInfo.setLabel(path);
- prefixPathInfo.setPath(bucket.getPath() + OM_KEY_PREFIX + path);
- prefixPathInfo.setAccessCount(readAccessCount);
- prefixPathInfo.setSize(keySize);
- prefixes.add(prefixPathInfo);
- // This is done for specific ask by UI treemap to render and provide
- // varying color shades based on varying ranges of access count.
- updateRootLevelMinMaxAccessCount(readAccessCount, rootEntity);
+ if (children.size() > 0) {
+ entity.setMinAccessCount(Long.MAX_VALUE);
+ }
+ return children;
}
private void updateVolumeLevelMinMaxAccessCount(
@@ -279,52 +266,40 @@ public class HeatMapServiceImpl extends HeatMapService {
});
}
- @NotNull
- private static List<EntityReadAccessHeatMapResponse>
- initializeEntityMinMaxCount(
- EntityReadAccessHeatMapResponse entity) {
- List<EntityReadAccessHeatMapResponse> children =
- entity.getChildren();
- if (children.size() == 0) {
- entity.setMaxAccessCount(entity.getMinAccessCount());
- }
- if (children.size() > 0) {
- entity.setMinAccessCount(Long.MAX_VALUE);
+ private static double truncate(double value, int decimalPlaces) {
+ if (decimalPlaces < 0) {
+ throw new IllegalArgumentException();
}
- return children;
+ value = value * Math.pow(10, decimalPlaces);
+ value = Math.floor(value);
+ value = value / Math.pow(10, decimalPlaces);
+ return value;
}
- private void updateBucketLevelMinMaxAccessCount(
- EntityReadAccessHeatMapResponse bucket) {
- List<EntityReadAccessHeatMapResponse>
- children = initializeEntityMinMaxCount(bucket);
+ private void updateEntityAccessRatio(EntityReadAccessHeatMapResponse entity)
{
+ long delta = entity.getMaxAccessCount() - entity.getMinAccessCount();
+ List<EntityReadAccessHeatMapResponse> children =
+ entity.getChildren();
children.stream().forEach(path -> {
- long readAccessCount = path.getAccessCount();
- bucket.setMinAccessCount(
- path.getAccessCount() < bucket.getMinAccessCount() ? readAccessCount
:
- bucket.getMinAccessCount());
- bucket.setMaxAccessCount(
- readAccessCount > bucket.getMaxAccessCount() ? readAccessCount :
- bucket.getMaxAccessCount());
+ if (path.getChildren().size() != 0) {
+ updateEntityAccessRatio(path);
+ } else {
+ path.setColor(1.000);
+ long accessCount = path.getAccessCount();
+ accessCount =
+ (accessCount == 0 ? path.getMinAccessCount() : accessCount);
+ if (delta >= 0) {
+ if (accessCount > 0) {
+ double truncatedValue = truncate(
+ ((double) accessCount /
+ (double) entity.getMaxAccessCount()), 3);
+ path.setColor(truncatedValue);
+ }
+ }
+ }
});
}
- private void updateRootLevelMinMaxAccessCount(
- long readAccessCount,
- EntityReadAccessHeatMapResponse rootEntity) {
- rootEntity.setMinAccessCount(
- readAccessCount < rootEntity.getMinAccessCount() ? readAccessCount :
- rootEntity.getMinAccessCount());
- rootEntity.setMaxAccessCount(
- readAccessCount > rootEntity.getMaxAccessCount() ? readAccessCount :
- rootEntity.getMaxAccessCount());
- }
-
- private void updateBucketSize(EntityReadAccessHeatMapResponse bucket,
- long keySize) {
- bucket.setSize(bucket.getSize() + keySize);
- }
-
/**
* Transforms the access metadata of entities (provided by heatmap provider)
* by grouping them in their respective buckets and volumes in tree
structure
@@ -473,28 +448,8 @@ public class HeatMapServiceImpl extends HeatMapService {
return rootEntity;
}
- private static boolean validateLength(String[] split, int minLength) {
- return (split.length < minLength);
- }
-
- private long getEntitySize(String path) throws IOException {
- long entitySize = 0;
- LOG.info("Getting entity size for {}: ", path);
- EntityHandler entityHandler =
- EntityHandler.getEntityHandler(reconNamespaceSummaryManager,
- omMetadataManager, reconSCM, path);
- if (null != entityHandler) {
- DUResponse duResponse = entityHandler.getDuResponse(false, false);
- if (null != duResponse && duResponse.getStatus() == ResponseStatus.OK) {
- return duResponse.getSize();
- }
- }
- // returning some default value due to some issue
- return 256L;
- }
-
public EntityReadAccessHeatMapResponse retrieveDataAndGenerateHeatMap(
- String normalizePath,
+ IHeatMapProvider heatMapProvider, String normalizePath,
String entityType,
String startDate) throws Exception {
if (null != heatMapProvider) {
diff --git
a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/heatmap/TestHeatMapInfo.java
b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/heatmap/TestHeatMapInfo.java
index fb973ac227..8a8f63fe34 100644
---
a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/heatmap/TestHeatMapInfo.java
+++
b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/heatmap/TestHeatMapInfo.java
@@ -59,7 +59,7 @@ public class TestHeatMapInfo {
private boolean isSetupDone = false;
private ReconOMMetadataManager reconOMMetadataManager;
private String auditRespStr;
- private HeatMapServiceImpl heatMapService;
+ private HeatMapUtil heatMapUtil;
@SuppressWarnings("checkstyle:methodlength")
private void initializeInjector() throws Exception {
@@ -81,7 +81,7 @@ public class TestHeatMapInfo {
mock(StorageContainerServiceProviderImpl.class))
.addBinding(ContainerHealthSchemaManager.class)
.build();
- heatMapService = reconTestInjector.getInstance(HeatMapServiceImpl.class);
+ heatMapUtil = reconTestInjector.getInstance(HeatMapUtil.class);
auditRespStr = "{\n" +
" \"responseHeader\": {\n" +
" \"zkConnected\": true,\n" +
@@ -756,7 +756,7 @@ public class TestHeatMapInfo {
List<EntityMetaData> entityMetaDataList =
Arrays.stream(entities).collect(Collectors.toList());
EntityReadAccessHeatMapResponse entityReadAccessHeatMapResponse =
- heatMapService.generateHeatMap(entityMetaDataList);
+ heatMapUtil.generateHeatMap(entityMetaDataList);
Assertions.assertTrue(
entityReadAccessHeatMapResponse.getChildren().size() > 0);
Assertions.assertEquals(12,
@@ -872,7 +872,7 @@ public class TestHeatMapInfo {
// "minAccessCount": 19263
//}
EntityReadAccessHeatMapResponse entityReadAccessHeatMapResponse =
- heatMapService.generateHeatMap(entityMetaDataList);
+ heatMapUtil.generateHeatMap(entityMetaDataList);
Assertions.assertTrue(
entityReadAccessHeatMapResponse.getChildren().size() > 0);
Assertions.assertEquals(2,
@@ -1107,7 +1107,7 @@ public class TestHeatMapInfo {
// "maxAccessCount": 701
//}
EntityReadAccessHeatMapResponse entityReadAccessHeatMapResponse =
- heatMapService.generateHeatMap(entityMetaDataList);
+ heatMapUtil.generateHeatMap(entityMetaDataList);
Assertions.assertTrue(
entityReadAccessHeatMapResponse.getChildren().size() > 0);
Assertions.assertEquals(2,
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
