This is an automated email from the ASF dual-hosted git repository.
cwylie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new 2754405f76d add storage_size to sys.servers (#18979)
2754405f76d is described below
commit 2754405f76d7e615f9e569046583c722eb986ea4
Author: Clint Wylie <[email protected]>
AuthorDate: Tue Feb 3 10:06:50 2026 -0800
add storage_size to sys.servers (#18979)
---
.../embedded/query/QueryVirtualStorageTest.java | 25 +++++++++++++++----
.../druid/sql/calcite/schema/SystemSchema.java | 4 +++
.../druid/sql/calcite/schema/SystemSchemaTest.java | 29 +++++++++++++++++++---
3 files changed, 50 insertions(+), 8 deletions(-)
diff --git
a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/query/QueryVirtualStorageTest.java
b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/query/QueryVirtualStorageTest.java
index 02d0fae66f8..ea16aebb4ff 100644
---
a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/query/QueryVirtualStorageTest.java
+++
b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/query/QueryVirtualStorageTest.java
@@ -66,6 +66,11 @@ import java.util.concurrent.ThreadLocalRandom;
*/
class QueryVirtualStorageTest extends EmbeddedClusterTestBase
{
+ // size of wiki segments, adjust this if segment size changes for some reason
+ private static final long SIZE_BYTES = 3776682L;
+ private static final long CACHE_SIZE = HumanReadableBytes.parse("1MiB");
+ private static final long MAX_SIZE = HumanReadableBytes.parse("100MiB");
+
private final EmbeddedBroker broker = new EmbeddedBroker();
private final EmbeddedIndexer indexer = new EmbeddedIndexer();
private final EmbeddedOverlord overlord = new EmbeddedOverlord();
@@ -88,11 +93,11 @@ class QueryVirtualStorageTest extends
EmbeddedClusterTestBase
StringUtils.format(
"[{\"path\":\"%s\",\"maxSize\":\"%s\"}]",
cluster.getTestFolder().newFolder().getAbsolutePath(),
- HumanReadableBytes.parse("1MiB")
+ CACHE_SIZE
)
)
)
- .addProperty("druid.server.maxSize",
String.valueOf(HumanReadableBytes.parse("100MiB")));
+ .addProperty("druid.server.maxSize", String.valueOf(MAX_SIZE));
broker.setServerMemory(200_000_000)
.addProperty("druid.msq.dart.controller.maxRetainedReportCount",
"10")
@@ -236,12 +241,12 @@ class QueryVirtualStorageTest extends
EmbeddedClusterTestBase
coordinatorEmitter.waitForEvent(event ->
event.hasMetricName(Stats.Tier.STORAGE_CAPACITY.getMetricName()));
Assertions.assertEquals(
- HumanReadableBytes.parse("1MiB"),
+ CACHE_SIZE,
coordinatorEmitter.getLatestMetricEventValue(Stats.Tier.STORAGE_CAPACITY.getMetricName())
);
coordinatorEmitter.waitForEvent(event ->
event.hasMetricName(Stats.Tier.TOTAL_CAPACITY.getMetricName()));
Assertions.assertEquals(
- HumanReadableBytes.parse("100MiB"),
+ MAX_SIZE,
coordinatorEmitter.getLatestMetricEventValue(Stats.Tier.TOTAL_CAPACITY.getMetricName())
);
}
@@ -293,11 +298,21 @@ class QueryVirtualStorageTest extends
EmbeddedClusterTestBase
Assertions.assertTrue(segmentChannelCounters.getLoadFiles()[0] > 0 &&
segmentChannelCounters.getLoadFiles()[0] <=
segmentChannelCounters.getFiles()[0]);
// size of all segments at time of writing, possibly we have to load all
of them, but possibly less depending on
// test order
- Assertions.assertTrue(segmentChannelCounters.getLoadBytes()[0] > 0 &&
segmentChannelCounters.getLoadBytes()[0] <= 3776682L);
+ Assertions.assertTrue(segmentChannelCounters.getLoadBytes()[0] > 0 &&
segmentChannelCounters.getLoadBytes()[0] <= SIZE_BYTES);
Assertions.assertTrue(segmentChannelCounters.getLoadTime()[0] > 0);
Assertions.assertTrue(segmentChannelCounters.getLoadWait()[0] > 0);
}
+ @Test
+ void testQuerySysTables()
+ {
+ String query = "SELECT curr_size, max_size, storage_size FROM sys.servers
WHERE tier IS NOT NULL AND server_type = 'historical'";
+ Assertions.assertEquals(
+ StringUtils.format("%s,%s,%s", SIZE_BYTES, MAX_SIZE, CACHE_SIZE),
+ cluster.callApi().runSql(query)
+ );
+ }
+
private void assertQueryMetrics(int expectedEventCount, @Nullable Long
expectedLoadCount)
{
diff --git
a/sql/src/main/java/org/apache/druid/sql/calcite/schema/SystemSchema.java
b/sql/src/main/java/org/apache/druid/sql/calcite/schema/SystemSchema.java
index 35c92e403a1..f461acfa687 100644
--- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/SystemSchema.java
+++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/SystemSchema.java
@@ -188,6 +188,7 @@ public class SystemSchema extends AbstractSchema
.add("tier", ColumnType.STRING)
.add("curr_size", ColumnType.LONG)
.add("max_size", ColumnType.LONG)
+ .add("storage_size", ColumnType.LONG)
.add("is_leader", ColumnType.LONG)
.add("start_time", ColumnType.STRING)
.add("version", ColumnType.STRING)
@@ -692,6 +693,7 @@ public class SystemSchema extends AbstractSchema
null,
UNKNOWN_SIZE,
UNKNOWN_SIZE,
+ UNKNOWN_SIZE,
null,
toStringOrNull(discoveryDruidNode.getStartTime()),
node.getVersion(),
@@ -719,6 +721,7 @@ public class SystemSchema extends AbstractSchema
null,
UNKNOWN_SIZE,
UNKNOWN_SIZE,
+ UNKNOWN_SIZE,
isLeader ? 1L : 0L,
toStringOrNull(discoveryDruidNode.getStartTime()),
node.getVersion(),
@@ -758,6 +761,7 @@ public class SystemSchema extends AbstractSchema
druidServerToUse.getTier(),
currentSize,
druidServerToUse.getMaxSize(),
+ druidServerToUse.getStorageSize(),
null,
toStringOrNull(discoveryDruidNode.getStartTime()),
node.getVersion(),
diff --git
a/sql/src/test/java/org/apache/druid/sql/calcite/schema/SystemSchemaTest.java
b/sql/src/test/java/org/apache/druid/sql/calcite/schema/SystemSchemaTest.java
index 3914448c733..ae15e124198 100644
---
a/sql/src/test/java/org/apache/druid/sql/calcite/schema/SystemSchemaTest.java
+++
b/sql/src/test/java/org/apache/druid/sql/calcite/schema/SystemSchemaTest.java
@@ -583,7 +583,7 @@ public class SystemSchemaTest extends CalciteTestBase
final SystemSchema.ServersTable serversTable = (SystemSchema.ServersTable)
schema.getTableMap().get("servers");
final RelDataType serverRowType = serversTable.getRowType(new
JavaTypeFactoryImpl());
final List<RelDataTypeField> serverFields = serverRowType.getFieldList();
- Assert.assertEquals(14, serverFields.size());
+ Assert.assertEquals(15, serverFields.size());
Assert.assertEquals("server", serverFields.get(0).getName());
Assert.assertEquals(SqlTypeName.VARCHAR,
serverFields.get(0).getType().getSqlTypeName());
@@ -847,7 +847,7 @@ public class SystemSchemaTest extends CalciteTestBase
final List<DruidServer> servers = new ArrayList<>();
servers.add(mockDataServer(historical1.getDruidNode().getHostAndPortToUse(),
200L, 1000L, "tier"));
-
servers.add(mockDataServer(historical2.getDruidNode().getHostAndPortToUse(),
400L, 1000L, "tier"));
+
servers.add(mockDataServer(historical2.getDruidNode().getHostAndPortToUse(),
400L, 1000L, 100L, "tier2"));
servers.add(mockDataServer(peon1.getDruidNode().getHostAndPortToUse(), 0L,
1000L, "tier"));
servers.add(mockDataServer(peon2.getDruidNode().getHostAndPortToUse(), 0L,
1000L, "tier"));
servers.add(mockDataServer(broker1.getDruidNode().getHostAndPortToUse(),
0L, 1000L, "tier"));
@@ -888,6 +888,7 @@ public class SystemSchemaTest extends CalciteTestBase
null,
0L,
0L,
+ 0L,
nonLeader,
startTimeStr,
version,
@@ -906,6 +907,7 @@ public class SystemSchemaTest extends CalciteTestBase
"tier",
0L,
1000L,
+ 1000L,
nonLeader,
startTimeStr,
version,
@@ -921,9 +923,10 @@ public class SystemSchemaTest extends CalciteTestBase
8083,
-1,
NodeRole.HISTORICAL,
- "tier",
+ "tier2",
400L,
1000L,
+ 100L,
nonLeader,
startTimeStr,
version,
@@ -942,6 +945,7 @@ public class SystemSchemaTest extends CalciteTestBase
"tier",
0L,
1000L,
+ 1000L,
nonLeader,
startTimeStr,
version,
@@ -960,6 +964,7 @@ public class SystemSchemaTest extends CalciteTestBase
"tier",
0L,
1000L,
+ 1000L,
nonLeader,
startTimeStr,
version,
@@ -977,6 +982,7 @@ public class SystemSchemaTest extends CalciteTestBase
"tier",
0L,
1000L,
+ 1000L,
nonLeader,
startTimeStr,
version,
@@ -994,6 +1000,7 @@ public class SystemSchemaTest extends CalciteTestBase
null,
0L,
0L,
+ 0L,
1L,
startTimeStr,
version,
@@ -1012,6 +1019,7 @@ public class SystemSchemaTest extends CalciteTestBase
null,
0L,
0L,
+ 0L,
nonLeader,
startTimeStr,
version,
@@ -1030,6 +1038,7 @@ public class SystemSchemaTest extends CalciteTestBase
"tier",
200L,
1000L,
+ 1000L,
nonLeader,
startTimeStr,
version,
@@ -1048,6 +1057,7 @@ public class SystemSchemaTest extends CalciteTestBase
null,
0L,
0L,
+ 0L,
1L,
startTimeStr,
version,
@@ -1067,6 +1077,7 @@ public class SystemSchemaTest extends CalciteTestBase
0L,
0L,
0L,
+ 0L,
startTimeStr,
version,
null,
@@ -1085,6 +1096,7 @@ public class SystemSchemaTest extends CalciteTestBase
0L,
0L,
0L,
+ 0L,
startTimeStr,
version,
null,
@@ -1102,6 +1114,7 @@ public class SystemSchemaTest extends CalciteTestBase
null,
0L,
0L,
+ 0L,
nonLeader,
startTimeStr,
version,
@@ -1120,6 +1133,7 @@ public class SystemSchemaTest extends CalciteTestBase
null,
0L,
0L,
+ 0L,
nonLeader,
startTimeStr,
version,
@@ -1137,6 +1151,7 @@ public class SystemSchemaTest extends CalciteTestBase
"tier",
0L,
1000L,
+ 1000L,
nonLeader,
startTimeStr,
version,
@@ -1154,6 +1169,11 @@ public class SystemSchemaTest extends CalciteTestBase
}
private DruidServer mockDataServer(String name, long currentSize, long
maxSize, String tier)
+ {
+ return mockDataServer(name, currentSize, maxSize, maxSize, tier);
+ }
+
+ private DruidServer mockDataServer(String name, long currentSize, long
maxSize, long storageSize, String tier)
{
final DruidServer server = EasyMock.createMock(DruidServer.class);
EasyMock.expect(serverInventoryView.getInventoryValue(name))
@@ -1161,6 +1181,7 @@ public class SystemSchemaTest extends CalciteTestBase
.once();
EasyMock.expect(server.getCurrSize()).andReturn(currentSize).once();
EasyMock.expect(server.getMaxSize()).andReturn(maxSize).once();
+ EasyMock.expect(server.getStorageSize()).andReturn(storageSize).once();
EasyMock.expect(server.getTier()).andReturn(tier).once();
return server;
}
@@ -1174,6 +1195,7 @@ public class SystemSchemaTest extends CalciteTestBase
@Nullable String tier,
@Nullable Long currSize,
@Nullable Long maxSize,
+ @Nullable Long storageSize,
@Nullable Long isLeader,
String startTime,
String version,
@@ -1191,6 +1213,7 @@ public class SystemSchemaTest extends CalciteTestBase
tier,
currSize,
maxSize,
+ storageSize,
isLeader,
startTime,
version,
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
