This is an automated email from the ASF dual-hosted git repository.
devesh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 925cc080ee HDDS-9626. [Recon] Disk Usage page with high number of
key/bucket/volume (#6535)
925cc080ee is described below
commit 925cc080ee97597dbb780dd8502d97badf94e6ba
Author: Smita <[email protected]>
AuthorDate: Mon Jun 10 18:53:56 2024 +0530
HDDS-9626. [Recon] Disk Usage page with high number of key/bucket/volume
(#6535)
---
.../webapps/recon/ozone-recon-web/api/db.json | 245 ++++++++++++++++++---
.../webapps/recon/ozone-recon-web/api/routes.json | 28 +--
.../src/components/rightDrawer/rightDrawer.tsx | 4 +-
.../src/views/diskUsage/diskUsage.tsx | 41 ++--
4 files changed, 253 insertions(+), 65 deletions(-)
diff --git
a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/api/db.json
b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/api/db.json
index b535746b46..271e70d741 100644
---
a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/api/db.json
+++
b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/api/db.json
@@ -1398,37 +1398,222 @@
"root": {
"status": "OK",
"path": "/",
- "size": 100000,
- "sizeWithReplica": 300000,
- "subPathCount": 5,
+ "size": 1709108,
+ "sizeWithReplica": -1,
+ "subPathCount": 30,
"subPaths": [
{
+ "key": false,
+ "path": "/vol-0-30461",
+ "size": 33096,
+ "sizeWithReplica": -1,
+ "isKey": false
+ },
+ {
+ "key": false,
+ "path": "/vol-1-82078",
+ "size": 33096,
+ "sizeWithReplica": -1,
+ "isKey": false
+ },
+ {
+ "key": false,
+ "path": "/vol-2-96509",
+ "size": 330496,
+ "sizeWithReplica": -1,
+ "isKey": false
+ },
+ {
+ "key": false,
+ "path": "/vol-3-75124",
+ "size": 330496,
+ "sizeWithReplica": -1,
+ "isKey": false
+ },
+ {
+ "key": false,
+ "path": "/vol-4-53147",
+ "size": 330240,
+ "sizeWithReplica": -1,
+ "isKey": false
+ },
+ {
+ "key": false,
"path": "/vol1",
- "size": 50000,
- "sizeWithReplica": 150000
+ "size": 12204,
+ "sizeWithReplica": -1,
+ "isKey": false
},
{
- "path": "/vol2",
- "size": 20000,
- "sizeWithReplica": 60000
+ "key": false,
+ "path": "/vol17",
+ "size": 8136,
+ "sizeWithReplica": -1,
+ "isKey": false
},
{
- "path": "/vol3",
- "size": 10000,
- "sizeWithReplica": 30000
+ "key": false,
+ "path": "/vol21",
+ "size": 8136,
+ "sizeWithReplica": -1,
+ "isKey": false
},
{
+ "key": false,
+ "path": "/vol22",
+ "size": 8136,
+ "sizeWithReplica": -1,
+ "isKey": false
+ },
+ {
+ "key": false,
"path": "/vol4",
- "size": 10000,
- "sizeWithReplica": 30000
+ "size": 8136,
+ "sizeWithReplica": -1,
+ "isKey": false
+ },
+ {
+ "key": false,
+ "path": "/vol18",
+ "size": 4068,
+ "sizeWithReplica": -1,
+ "isKey": false
+ },
+ {
+ "key": false,
+ "path": "/vol19",
+ "size": 4068,
+ "sizeWithReplica": -1,
+ "isKey": false
+ },
+ {
+ "key": false,
+ "path": "/s3v",
+ "size": 0,
+ "sizeWithReplica": -1,
+ "isKey": false
+ },
+ {
+ "key": false,
+ "path": "/vol10",
+ "size": 0,
+ "sizeWithReplica": -1,
+ "isKey": false
+ },
+ {
+ "key": false,
+ "path": "/vol11",
+ "size": 0,
+ "sizeWithReplica": -1,
+ "isKey": false
+ },
+ {
+ "key": false,
+ "path": "/vol12",
+ "size": 0,
+ "sizeWithReplica": -1,
+ "isKey": false
+ },
+ {
+ "key": false,
+ "path": "/vol13",
+ "size": 0,
+ "sizeWithReplica": -1,
+ "isKey": false
+ },
+ {
+ "key": false,
+ "path": "/vol14",
+ "size": 0,
+ "sizeWithReplica": -1,
+ "isKey": false
+ },
+ {
+ "key": false,
+ "path": "/vol15",
+ "size": 0,
+ "sizeWithReplica": -1,
+ "isKey": false
+ },
+ {
+ "key": false,
+ "path": "/vol16",
+ "size": 0,
+ "sizeWithReplica": -1,
+ "isKey": false
+ },
+ {
+ "key": false,
+ "path": "/vol2",
+ "size": 0,
+ "sizeWithReplica": -1,
+ "isKey": false
},
{
+ "key": false,
+ "path": "/vol20",
+ "size": 0,
+ "sizeWithReplica": -1,
+ "isKey": false
+ },
+ {
+ "key": false,
+ "path": "/vol3",
+ "size": 0,
+ "sizeWithReplica": -1,
+ "isKey": false
+ },
+ {
+ "key": false,
"path": "/vol5",
- "size": 10000,
- "sizeWithReplica": 30000
+ "size": 0,
+ "sizeWithReplica": -1,
+ "isKey": false
+ },
+ {
+ "key": false,
+ "path": "/vol6",
+ "size": 0,
+ "sizeWithReplica": -1,
+ "isKey": false
+ },
+ {
+ "key": false,
+ "path": "/vol7",
+ "size": 0,
+ "sizeWithReplica": -1,
+ "isKey": false
+ },
+ {
+ "key": false,
+ "path": "/vol8",
+ "size": 0,
+ "sizeWithReplica": -1,
+ "isKey": false
+ },
+ {
+ "key": false,
+ "path": "/vol9",
+ "size": 0,
+ "sizeWithReplica": -1,
+ "isKey": false
+ },
+ {
+ "key": false,
+ "path": "/vol10",
+ "size": 10,
+ "sizeWithReplica": -1,
+ "isKey": false
+ },
+ {
+ "key": false,
+ "path": "/vol11",
+ "size": 20,
+ "sizeWithReplica": -1,
+ "isKey": false
}
],
- "sizeDirectKey": 0
+ "sizeDirectKey": -1
},
"volume": {
"status": "OK",
@@ -1519,37 +1704,41 @@
},
"dir": {
"status": "OK",
- "path": "/dummyVolume/dummyBucket/dummyDir",
- "size": 200000,
+ "path": "/volumetest/buckettest/dir1",
+ "size": 111010000,
"sizeWithReplica": -1,
- "subPathCount": 5,
+ "subPathCount": 4,
"subPaths": [
{
- "path": "/dummyVolume/dummyBucket/dummyDir/key1",
- "size": 100000,
+ "key": true,
+ "path": "/volumetest/buckettest/dir1/key100MB",
+ "size": 100000000,
"sizeWithReplica": -1,
"isKey": true
},
{
- "path": "/dummyVolume/dummyBucket/dummyDir/key2",
- "size": 30000,
+ "key": true,
+ "path": "/volumetest/buckettest/dir1/key10mb",
+ "size": 10000000,
"sizeWithReplica": -1,
"isKey": true
},
{
- "path": "/dummyVolume/dummyBucket/dummyDir/key3",
- "size": 30000,
+ "key": true,
+ "path": "/volumetest/buckettest/dir1/key1MB",
+ "size": 1000000,
"sizeWithReplica": -1,
"isKey": true
},
{
- "path": "/dummyVolume/dummyBucket/dummyDir/key4",
- "size": 40000,
+ "key": true,
+ "path": "/volumetest/buckettest/dir1/key10kb",
+ "size": 10000,
"sizeWithReplica": -1,
"isKey": true
}
],
- "sizeDirectKey": 50000
+ "sizeDirectKey": 111010000
},
"empty": {
"status": "OK",
diff --git
a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/api/routes.json
b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/api/routes.json
index 1e1f79d187..cb6cea4c84 100644
---
a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/api/routes.json
+++
b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/api/routes.json
@@ -3,24 +3,24 @@
"/containers/:id/keys": "/keys",
"/utilization/fileCount": "/fileSizeCounts",
"/utilization/containerCount": "/containerCount",
- "/namespace/du?path=/&files=true": "/root",
- "/namespace/du?path=/vol:id&files=true": "/volume",
- "/namespace/du?path=/vol:id/&files=true": "/volume",
- "/namespace/du?path=/dummyVolume&files=true": "/volume",
- "/namespace/du?path=/vol:id/bucket:id&files=true": "/bucket",
- "/namespace/du?path=/dummyVolume/bucket:id&files=true": "/bucket",
- "/namespace/du?path=/dummyVolume/bucket:id/&files=true": "/bucket",
- "/namespace/du?path=/dummyVolume/dummyBucket&files=true": "/bucket",
- "/namespace/du?path=/vol:id/bucket:id/dir:id*&files=true": "/dir",
- "/namespace/du?path=/dummyVolume/dummyBucket/dir:id&files=true": "/dir",
- "/namespace/du?path=/dummyVolume/dummyBucket/dir:id/&files=true": "/dir",
- "/namespace/du?path=/dummyVolume/dummyBucket/dummyDir&files=true": "/dir",
+ "/namespace/du?path=/&files=true&sortSubpaths=true": "/root",
+ "/namespace/du?path=/vol:id&files=true&sortSubpaths=true": "/volume",
+ "/namespace/du?path=/vol:id/&files=true&sortSubPaths=true": "/volume",
+ "/namespace/du?path=/dummyVolume&files=true&sortSubPaths=true": "/volume",
+ "/namespace/du?path=/vol:id/bucket:id&files=true&sortSubPaths=true":
"/bucket",
+ "/namespace/du?path=/dummyVolume/bucket:id&files=true&sortSubPaths=true":
"/bucket",
+ "/namespace/du?path=/dummyVolume/bucket:id/&files=true&sortSubPaths=true":
"/bucket",
+ "/namespace/du?path=/dummyVolume/dummyBucket&files=true&sortSubPaths=true":
"/bucket",
+ "/namespace/du?path=/vol:id/bucket:id/dir:id*&files=true&sortSubPaths=true":
"/dir",
+
"/namespace/du?path=/dummyVolume/dummyBucket/dir:id&files=true&sortSubPaths=true":
"/dir",
+
"/namespace/du?path=/dummyVolume/dummyBucket/dir:id/&files=true&sortSubPaths=true":
"/dir",
+
"/namespace/du?path=/dummyVolume/dummyBucket/dummyDir&files=true&sortSubPaths=true":
"/dir",
"/namespace/du?path=/vol:id/bucket:id/dir:id/key:id*": "/key",
"/namespace/du?path=/dummyVolume/dummyBucket/dummyDir/key:id*": "/key",
"/namespace/du?path=/vol:id/bucket:id/key:id*": "/key",
"/namespace/du?path=/dummyVolume/dummyBucket/key:id*": "/key",
- "/namespace/du?path=/vol1/bucket1/empty&files=true": "/empty",
- "/namespace/du?path=/clunky&files=true": "/clunky",
+ "/namespace/du?path=/vol1/bucket1/empty&files=true&sortSubPaths=true":
"/empty",
+ "/namespace/du?path=/clunky&files=true&sortSubpaths=true": "/clunky",
"/namespace/summary?path=*": "/metadata",
"/namespace/quota?path=*": "/quota",
"/task/status": "/taskStatus",
diff --git
a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/components/rightDrawer/rightDrawer.tsx
b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/components/rightDrawer/rightDrawer.tsx
index e1d3c36869..8ff69717ba 100644
---
a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/components/rightDrawer/rightDrawer.tsx
+++
b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/components/rightDrawer/rightDrawer.tsx
@@ -54,8 +54,8 @@ export class DetailPanel extends
React.Component<IRightDrawerProps> {
<Drawer
title={`Metadata Summary for ${path}`}
placement='right'
- width='40%'
- closable={false}
+ width='25%'
+ closable={true}
visible={visible}
getContainer={false}
style={{position: 'absolute'}}
diff --git
a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/diskUsage/diskUsage.tsx
b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/diskUsage/diskUsage.tsx
index 29866aacf0..e82d8f71fc 100644
---
a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/diskUsage/diskUsage.tsx
+++
b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/diskUsage/diskUsage.tsx
@@ -28,6 +28,7 @@ import { AxiosGetHelper, cancelRequests } from
'utils/axiosRequestHelper';
const DEFAULT_DISPLAY_LIMIT = 10;
const OTHER_PATH_NAME = 'Other Objects';
+const MAX_DISPLAY_LIMIT = 30;
interface IDUSubpath {
path: string;
@@ -129,7 +130,7 @@ export class DiskUsage extends
React.Component<Record<string, object>, IDUState>
this.setState({
isLoading: true
});
- const duEndpoint = `/api/v1/namespace/du?path=${path}&files=true`;
+ const duEndpoint =
`/api/v1/namespace/du?path=${path}&files=true&sortSubPaths=true`;
const { request, controller } = AxiosGetHelper(duEndpoint, cancelPieSignal)
cancelPieSignal = controller;
request.then(response => {
@@ -144,20 +145,24 @@ export class DiskUsage extends
React.Component<Record<string, object>, IDUState>
const dataSize = duResponse.size;
let subpaths: IDUSubpath[] = duResponse.subPaths;
- subpaths.sort((a, b) => (a.size < b.size) ? 1 : -1);
-
- // Only show top n blocks with the most DU,
- // other blocks are merged as a single block
- if (subpaths.length > limit) {
+ // We need to calculate the size of "Other objects" in two cases:
+ // 1) If we have more subpaths listed, than the limit.
+ // 2) If the limit is set to the maximum limit (30) and we have any
number of subpaths. In this case we won't
+ // necessarily have "Other objects", but later we check if the other
objects's size is more than zero (we will have
+ // other objects if there are more than 30 subpaths, but we can't check
on that, as the response will always have
+ // 30 subpaths, but from the total size and the subpaths size we can
calculate it).
+
+ if (subpaths.length > limit || (subpaths.length > 0 && limit ===
MAX_DISPLAY_LIMIT)) {
subpaths = subpaths.slice(0, limit);
let topSize = 0;
- for (let i = 0; i < limit; ++i) {
+ for (let i = 0; i < subpaths.length; ++i) {
topSize += subpaths[i].size;
}
-
const otherSize = dataSize - topSize;
- const other: IDUSubpath = {path: OTHER_PATH_NAME, size: otherSize};
- subpaths.push(other);
+ if (otherSize > 0) {
+ const other: IDUSubpath = {path: OTHER_PATH_NAME, size: otherSize};
+ subpaths.push(other);
+ }
}
let pathLabels, values, percentage, sizeStr, pieces, subpathName;
@@ -264,13 +269,7 @@ export class DiskUsage extends
React.Component<Record<string, object>, IDUState>
}
updateDisplayLimit(e): void {
- let res = -1;
- if (e.key === 'all') {
- res = Number.MAX_VALUE;
- } else {
- res = Number.parseInt(e.key, 10);
- }
-
+ let res = Number.parseInt(e.key, 10);
this.updatePieChart(this.state.inputPath, res);
}
@@ -517,8 +516,8 @@ export class DiskUsage extends
React.Component<Record<string, object>, IDUState>
<Menu.Item key='20'>
20
</Menu.Item>
- <Menu.Item key='all'>
- All
+ <Menu.Item key='30'>
+ 30
</Menu.Item>
</Menu>
);
@@ -549,7 +548,7 @@ export class DiskUsage extends
React.Component<Record<string, object>, IDUState>
</div>
<div className='dropdown-button'>
<Dropdown overlay={menu} placement='bottomCenter'>
- <Button>Display Limit: {(displayLimit ===
Number.MAX_VALUE) ? 'All' : displayLimit}</Button>
+ <Button>Display Limit: {displayLimit}</Button>
</Dropdown>
</div>
<div className='metadata-button'>
@@ -569,7 +568,7 @@ export class DiskUsage extends
React.Component<Record<string, object>, IDUState>
layout={
{
width: 1200,
- height: 750,
+ height: 900,
font: {
family: 'Roboto, sans-serif',
size: 15
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
