This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 588dc5f12a [feature](cold_on_s3) Show remote data usage via SHOW
BACKENDS and SHOW TABLETS statements (#11450)
588dc5f12a is described below
commit 588dc5f12a3001796932c42d4ea8c02551bdc3fe
Author: plat1ko <[email protected]>
AuthorDate: Thu Aug 25 15:36:15 2022 +0800
[feature](cold_on_s3) Show remote data usage via SHOW BACKENDS and SHOW
TABLETS statements (#11450)
---
be/src/io/fs/s3_file_system.cpp | 2 +-
.../src/main/java/org/apache/doris/catalog/DiskInfo.java | 10 ++++++++++
.../org/apache/doris/common/proc/BackendsProcDir.java | 9 ++++++++-
.../org/apache/doris/common/proc/ReplicasProcNode.java | 11 +++++------
.../java/org/apache/doris/common/proc/TabletsProcDir.java | 4 +++-
.../src/main/java/org/apache/doris/system/Backend.java | 15 +++++++++++++++
.../org/apache/doris/utframe/DemoMultiBackendsTest.java | 2 +-
.../suites/compaction/test_compaction_agg_keys.groovy | 4 ++--
.../suites/compaction/test_compaction_dup_keys.groovy | 4 ++--
.../suites/compaction/test_compaction_uniq_keys.groovy | 4 ++--
10 files changed, 49 insertions(+), 16 deletions(-)
diff --git a/be/src/io/fs/s3_file_system.cpp b/be/src/io/fs/s3_file_system.cpp
index 575a51302b..b3b7c74a02 100644
--- a/be/src/io/fs/s3_file_system.cpp
+++ b/be/src/io/fs/s3_file_system.cpp
@@ -141,7 +141,7 @@ Status S3FileSystem::open_file(const Path& path,
FileReaderSPtr* reader) {
RETURN_IF_ERROR(file_size(path, &fsize));
auto key = get_key(path);
auto fs_path = Path(_s3_conf.endpoint) / _s3_conf.bucket / key;
- *reader = std::make_unique<S3FileReader>(std::move(fs_path), fsize,
std::move(key),
+ *reader = std::make_shared<S3FileReader>(std::move(fs_path), fsize,
std::move(key),
_s3_conf.bucket, this);
return Status::OK();
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/DiskInfo.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/DiskInfo.java
index a96b83869e..a40c1cb44c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/DiskInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/DiskInfo.java
@@ -47,6 +47,8 @@ public class DiskInfo implements Writable {
private long totalCapacityB;
@SerializedName("dataUsedCapacityB")
private long dataUsedCapacityB;
+ @SerializedName("remoteUsedCapacity")
+ private long remoteUsedCapacity = 0;
@SerializedName("diskAvailableCapacityB")
private long diskAvailableCapacityB;
@SerializedName("state")
@@ -89,6 +91,14 @@ public class DiskInfo implements Writable {
this.dataUsedCapacityB = dataUsedCapacityB;
}
+ public long getRemoteUsedCapacity() {
+ return remoteUsedCapacity;
+ }
+
+ public void setRemoteUsedCapacity(long remoteUsedCapacity) {
+ this.remoteUsedCapacity = remoteUsedCapacity;
+ }
+
public long getDiskUsedCapacityB() {
return totalCapacityB - diskAvailableCapacityB;
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/common/proc/BackendsProcDir.java
b/fe/fe-core/src/main/java/org/apache/doris/common/proc/BackendsProcDir.java
index d40f09df50..aa3c9821b0 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/proc/BackendsProcDir.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/BackendsProcDir.java
@@ -51,7 +51,7 @@ public class BackendsProcDir implements ProcDirInterface {
.add("BePort").add("HttpPort").add("BrpcPort").add("LastStartTime").add("LastHeartbeat").add("Alive")
.add("SystemDecommissioned").add("ClusterDecommissioned").add("TabletNum")
.add("DataUsedCapacity").add("AvailCapacity").add("TotalCapacity").add("UsedPct")
-
.add("MaxDiskUsedPct").add("Tag").add("ErrMsg").add("Version").add("Status")
+
.add("MaxDiskUsedPct").add("RemoteUsedCapacity").add("Tag").add("ErrMsg").add("Version").add("Status")
.build();
public static final int HOSTNAME_INDEX = 3;
@@ -163,6 +163,13 @@ public class BackendsProcDir implements ProcDirInterface {
}
backendInfo.add(String.format("%.2f", used) + " %");
backendInfo.add(String.format("%.2f", backend.getMaxDiskUsedPct()
* 100) + " %");
+
+ // remote used capacity
+ long remoteUsedB = backend.getRemoteUsedCapacityB();
+ Pair<Double, String> totalRemoteUsedCapacity =
DebugUtil.getByteUint(remoteUsedB);
+
backendInfo.add(DebugUtil.DECIMAL_FORMAT_SCALE_3.format(totalRemoteUsedCapacity.first)
+ " "
+ + totalRemoteUsedCapacity.second);
+
// tags
backendInfo.add(backend.getTagMapString());
// err msg
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/common/proc/ReplicasProcNode.java
b/fe/fe-core/src/main/java/org/apache/doris/common/proc/ReplicasProcNode.java
index d9572f4221..d7bb526ea1 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/common/proc/ReplicasProcNode.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/common/proc/ReplicasProcNode.java
@@ -33,12 +33,10 @@ import java.util.List;
* show replicas' detail info within a tablet
*/
public class ReplicasProcNode implements ProcNodeInterface {
- public static final ImmutableList<String> TITLE_NAMES = new
ImmutableList.Builder<String>()
- .add("ReplicaId").add("BackendId").add("Version")
- .add("LstSuccessVersion").add("LstFailedVersion")
-
.add("LstFailedTime").add("SchemaHash").add("DataSize").add("RowCount").add("State")
-
.add("IsBad").add("VersionCount").add("PathHash").add("MetaUrl").add("CompactionStatus")
- .build();
+ public static final ImmutableList<String> TITLE_NAMES = new
ImmutableList.Builder<String>().add("ReplicaId")
+
.add("BackendId").add("Version").add("LstSuccessVersion").add("LstFailedVersion").add("LstFailedTime")
+
.add("SchemaHash").add("LocalDataSize").add("RemoteDataSize").add("RowCount").add("State").add("IsBad")
+
.add("VersionCount").add("PathHash").add("MetaUrl").add("CompactionStatus").build();
private long tabletId;
private List<Replica> replicas;
@@ -77,6 +75,7 @@ public class ReplicasProcNode implements ProcNodeInterface {
TimeUtils.longToTimeString(replica.getLastFailedTimestamp()),
String.valueOf(replica.getSchemaHash()),
String.valueOf(replica.getDataSize()),
+
String.valueOf(replica.getRemoteDataSize()),
String.valueOf(replica.getRowCount()),
String.valueOf(replica.getState()),
String.valueOf(replica.isBad()),
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/common/proc/TabletsProcDir.java
b/fe/fe-core/src/main/java/org/apache/doris/common/proc/TabletsProcDir.java
index 79fc9fd4b1..4ebcf7a310 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/proc/TabletsProcDir.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/TabletsProcDir.java
@@ -45,7 +45,7 @@ public class TabletsProcDir implements ProcDirInterface {
public static final ImmutableList<String> TITLE_NAMES = new
ImmutableList.Builder<String>()
.add("TabletId").add("ReplicaId").add("BackendId").add("SchemaHash").add("Version")
.add("LstSuccessVersion").add("LstFailedVersion").add("LstFailedTime")
- .add("DataSize").add("RowCount").add("State")
+
.add("LocalDataSize").add("RemoteDataSize").add("RowCount").add("State")
.add("LstConsistencyCheckTime").add("CheckVersion")
.add("VersionCount").add("PathHash").add("MetaUrl").add("CompactionStatus")
.build();
@@ -84,6 +84,7 @@ public class TabletsProcDir implements ProcDirInterface {
tabletInfo.add(-1); // lst failed version hash
tabletInfo.add(-1); // lst failed time
tabletInfo.add(-1); // data size
+ tabletInfo.add(-1); // remote data size
tabletInfo.add(-1); // row count
tabletInfo.add(FeConstants.null_string); // state
tabletInfo.add(-1); // lst consistency check time
@@ -113,6 +114,7 @@ public class TabletsProcDir implements ProcDirInterface {
tabletInfo.add(replica.getLastFailedVersion());
tabletInfo.add(TimeUtils.longToTimeString(replica.getLastFailedTimestamp()));
tabletInfo.add(replica.getDataSize());
+ tabletInfo.add(replica.getRemoteDataSize());
tabletInfo.add(replica.getRowCount());
tabletInfo.add(replica.getState());
diff --git a/fe/fe-core/src/main/java/org/apache/doris/system/Backend.java
b/fe/fe-core/src/main/java/org/apache/doris/system/Backend.java
index 76be9c2518..fc5758374c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/system/Backend.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/system/Backend.java
@@ -407,6 +407,17 @@ public class Backend implements Writable {
return dataUsedCapacityB;
}
+ public long getRemoteUsedCapacityB() {
+ ImmutableMap<String, DiskInfo> disks = disksRef;
+ long totalRemoteUsedCapacityB = 0L;
+ for (DiskInfo diskInfo : disks.values()) {
+ if (diskInfo.getState() == DiskState.ONLINE) {
+ totalRemoteUsedCapacityB += diskInfo.getRemoteUsedCapacity();
+ }
+ }
+ return totalRemoteUsedCapacityB;
+ }
+
public double getMaxDiskUsedPct() {
ImmutableMap<String, DiskInfo> disks = disksRef;
double maxPct = 0.0;
@@ -506,6 +517,10 @@ public class Backend implements Writable {
diskInfo.setTotalCapacityB(totalCapacityB);
diskInfo.setDataUsedCapacityB(dataUsedCapacityB);
diskInfo.setAvailableCapacityB(diskAvailableCapacityB);
+ if (tDisk.isSetRemoteUsedCapacity()) {
+ diskInfo.setRemoteUsedCapacity(tDisk.getRemoteUsedCapacity());
+ }
+
if (tDisk.isSetPathHash()) {
diskInfo.setPathHash(tDisk.getPathHash());
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/utframe/DemoMultiBackendsTest.java
b/fe/fe-core/src/test/java/org/apache/doris/utframe/DemoMultiBackendsTest.java
index 983c8e317a..7f619f8267 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/utframe/DemoMultiBackendsTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/utframe/DemoMultiBackendsTest.java
@@ -198,7 +198,7 @@ public class DemoMultiBackendsTest {
BackendsProcDir dir = new BackendsProcDir(Env.getCurrentSystemInfo());
ProcResult result = dir.fetchResult();
Assert.assertEquals(BackendsProcDir.TITLE_NAMES.size(),
result.getColumnNames().size());
- Assert.assertEquals("{\"location\" : \"default\"}",
result.getRows().get(0).get(19));
+ Assert.assertEquals("{\"location\" : \"default\"}",
result.getRows().get(0).get(20));
Assert.assertEquals("{\"lastSuccessReportTabletsTime\":\"N/A\",\"lastStreamLoadTime\":-1,\"isQueryDisabled\":false,\"isLoadDisabled\":false}",
result.getRows().get(0).get(BackendsProcDir.TITLE_NAMES.size()
- 1));
}
diff --git a/regression-test/suites/compaction/test_compaction_agg_keys.groovy
b/regression-test/suites/compaction/test_compaction_agg_keys.groovy
index 3a558d2f3c..9cf5feb43c 100644
--- a/regression-test/suites/compaction/test_compaction_agg_keys.groovy
+++ b/regression-test/suites/compaction/test_compaction_agg_keys.groovy
@@ -116,7 +116,7 @@ suite("test_compaction_agg_keys") {
qt_select_default """ SELECT * FROM ${tableName} t ORDER BY user_id;
"""
-
//TabletId,ReplicaId,BackendId,SchemaHash,Version,LstSuccessVersion,LstFailedVersion,LstFailedTime,DataSize,RowCount,State,LstConsistencyCheckTime,CheckVersion,VersionCount,PathHash,MetaUrl,CompactionStatus
+
//TabletId,ReplicaId,BackendId,SchemaHash,Version,LstSuccessVersion,LstFailedVersion,LstFailedTime,LocalDataSize,RemoteDataSize,RowCount,State,LstConsistencyCheckTime,CheckVersion,VersionCount,PathHash,MetaUrl,CompactionStatus
String[][] tablets = sql """ show tablets from ${tableName}; """
// trigger compactions for all tablets in ${tableName}
@@ -182,7 +182,7 @@ suite("test_compaction_agg_keys") {
for (String[] tablet in tablets) {
String tablet_id = tablet[0]
StringBuilder sb = new StringBuilder();
- def compactionStatusUrlIndex = 16
+ def compactionStatusUrlIndex = 17
sb.append("curl -X GET ")
sb.append(tablet[compactionStatusUrlIndex])
String command = sb.toString()
diff --git a/regression-test/suites/compaction/test_compaction_dup_keys.groovy
b/regression-test/suites/compaction/test_compaction_dup_keys.groovy
index 6abbf06baa..24249df221 100644
--- a/regression-test/suites/compaction/test_compaction_dup_keys.groovy
+++ b/regression-test/suites/compaction/test_compaction_dup_keys.groovy
@@ -114,7 +114,7 @@ suite("test_compaction_dup_keys") {
qt_select_default """ SELECT * FROM ${tableName} t ORDER BY
user_id,date,city,age,sex,last_visit_date,last_update_date,last_visit_date_not_null,cost,max_dwell_time,min_dwell_time;
"""
-
//TabletId,ReplicaId,BackendId,SchemaHash,Version,LstSuccessVersion,LstFailedVersion,LstFailedTime,DataSize,RowCount,State,LstConsistencyCheckTime,CheckVersion,VersionCount,PathHash,MetaUrl,CompactionStatus
+
//TabletId,ReplicaId,BackendId,SchemaHash,Version,LstSuccessVersion,LstFailedVersion,LstFailedTime,LocalDataSize,RemoteDataSize,RowCount,State,LstConsistencyCheckTime,CheckVersion,VersionCount,PathHash,MetaUrl,CompactionStatus
String[][] tablets = sql """ show tablets from ${tableName}; """
// trigger compactions for all tablets in ${tableName}
@@ -180,7 +180,7 @@ suite("test_compaction_dup_keys") {
for (String[] tablet in tablets) {
String tablet_id = tablet[0]
StringBuilder sb = new StringBuilder();
- def compactionStatusUrlIndex = 16
+ def compactionStatusUrlIndex = 17
sb.append("curl -X GET ")
sb.append(tablet[compactionStatusUrlIndex])
String command = sb.toString()
diff --git a/regression-test/suites/compaction/test_compaction_uniq_keys.groovy
b/regression-test/suites/compaction/test_compaction_uniq_keys.groovy
index 948ff0b921..6f15595067 100644
--- a/regression-test/suites/compaction/test_compaction_uniq_keys.groovy
+++ b/regression-test/suites/compaction/test_compaction_uniq_keys.groovy
@@ -114,7 +114,7 @@ suite("test_compaction_uniq_keys") {
qt_select_default """ SELECT * FROM ${tableName} t ORDER BY user_id;
"""
-
//TabletId,ReplicaId,BackendId,SchemaHash,Version,LstSuccessVersion,LstFailedVersion,LstFailedTime,DataSize,RowCount,State,LstConsistencyCheckTime,CheckVersion,VersionCount,PathHash,MetaUrl,CompactionStatus
+
//TabletId,ReplicaId,BackendId,SchemaHash,Version,LstSuccessVersion,LstFailedVersion,LstFailedTime,LocalDataSize,RemoteDataSize,RowCount,State,LstConsistencyCheckTime,CheckVersion,VersionCount,PathHash,MetaUrl,CompactionStatus
String[][] tablets = sql """ show tablets from ${tableName}; """
// trigger compactions for all tablets in ${tableName}
@@ -180,7 +180,7 @@ suite("test_compaction_uniq_keys") {
for (String[] tablet in tablets) {
String tablet_id = tablet[0]
StringBuilder sb = new StringBuilder();
- def compactionStatusUrlIndex = 16
+ def compactionStatusUrlIndex = 17
sb.append("curl -X GET ")
sb.append(tablet[compactionStatusUrlIndex])
String command = sb.toString()
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
