This is an automated email from the ASF dual-hosted git repository.
cwylie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-druid.git
The following commit(s) were added to refs/heads/master by this push:
new 631cda6 Include replicated segment size property for datasources
endpoint (#8039)
631cda6 is described below
commit 631cda649b388324c6e4d383e56cae2786144c27
Author: Atul Mohan <[email protected]>
AuthorDate: Thu Jul 11 03:10:38 2019 -0500
Include replicated segment size property for datasources endpoint (#8039)
* Add replication size
* Summon comma
---
docs/content/operations/api-reference.md | 4 +-
.../druid/server/http/DataSourcesResource.java | 10 ++++
.../druid/server/http/DataSourcesResourceTest.java | 56 ++++++++++++++++++++++
3 files changed, 68 insertions(+), 2 deletions(-)
diff --git a/docs/content/operations/api-reference.md
b/docs/content/operations/api-reference.md
index 473282f..3b49b12 100644
--- a/docs/content/operations/api-reference.md
+++ b/docs/content/operations/api-reference.md
@@ -162,7 +162,7 @@ Returns a list of datasource names found in the cluster.
* `/druid/coordinator/v1/datasources?simple`
-Returns a list of JSON objects containing the name and properties of
datasources found in the cluster. Properties include segment count, total
segment byte size, minTime, and maxTime.
+Returns a list of JSON objects containing the name and properties of
datasources found in the cluster. Properties include segment count, total
segment byte size, replicated total segment byte size, minTime, and maxTime.
* `/druid/coordinator/v1/datasources?full`
@@ -170,7 +170,7 @@ Returns a list of datasource names found in the cluster
with all metadata about
* `/druid/coordinator/v1/datasources/{dataSourceName}`
-Returns a JSON object containing the name and properties of a datasource.
Properties include segment count, total segment byte size, minTime, and maxTime.
+Returns a JSON object containing the name and properties of a datasource.
Properties include segment count, total segment byte size, replicated total
segment byte size, minTime, and maxTime.
* `/druid/coordinator/v1/datasources/{dataSourceName}?full`
diff --git
a/server/src/main/java/org/apache/druid/server/http/DataSourcesResource.java
b/server/src/main/java/org/apache/druid/server/http/DataSourcesResource.java
index 9255e7c..a2f04ff 100644
--- a/server/src/main/java/org/apache/druid/server/http/DataSourcesResource.java
+++ b/server/src/main/java/org/apache/druid/server/http/DataSourcesResource.java
@@ -538,6 +538,8 @@ public class DataSourcesResource
Map<String, HashSet<Object>> tierDistinctSegments = new HashMap<>();
long totalSegmentSize = 0;
+ long totalReplicatedSize = 0;
+
DateTime minTime = DateTimes.MAX;
DateTime maxTime = DateTimes.MIN;
String tier;
@@ -552,6 +554,8 @@ public class DataSourcesResource
tierDistinctSegments.computeIfAbsent(tier, k -> new HashSet<>());
long dataSourceSegmentSize = 0;
+ long replicatedSegmentSize = 0;
+
for (DataSegment dataSegment : druidDataSource.getSegments()) {
// tier segments stats
if (!tierDistinctSegments.get(tier).contains(dataSegment.getId())) {
@@ -565,6 +569,8 @@ public class DataSourcesResource
minTime = DateTimes.min(minTime,
dataSegment.getInterval().getStart());
maxTime = DateTimes.max(maxTime, dataSegment.getInterval().getEnd());
}
+ totalReplicatedSize += dataSegment.getSize();
+ replicatedSegmentSize += dataSegment.getSize();
}
// tier stats
@@ -577,10 +583,14 @@ public class DataSourcesResource
long segmentSize = MapUtils.getLong(tierStats, "size", 0L);
tierStats.put("size", segmentSize + dataSourceSegmentSize);
+
+ long replicatedSize = MapUtils.getLong(tierStats, "replicatedSize", 0L);
+ tierStats.put("replicatedSize", replicatedSize + replicatedSegmentSize);
}
segments.put("count", totalDistinctSegments.size());
segments.put("size", totalSegmentSize);
+ segments.put("replicatedSize", totalReplicatedSize);
segments.put("minTime", minTime);
segments.put("maxTime", maxTime);
return retVal;
diff --git
a/server/src/test/java/org/apache/druid/server/http/DataSourcesResourceTest.java
b/server/src/test/java/org/apache/druid/server/http/DataSourcesResourceTest.java
index 02885ff..1ae294b 100644
---
a/server/src/test/java/org/apache/druid/server/http/DataSourcesResourceTest.java
+++
b/server/src/test/java/org/apache/druid/server/http/DataSourcesResourceTest.java
@@ -373,11 +373,13 @@ public class DataSourcesResourceTest
Map<String, Map<String, Object>> result = (Map<String, Map<String,
Object>>) response.getEntity();
Assert.assertEquals(1, ((Map)
(result.get("tiers").get(null))).get("segmentCount"));
Assert.assertEquals(10L, ((Map)
(result.get("tiers").get(null))).get("size"));
+ Assert.assertEquals(10L, ((Map)
(result.get("tiers").get(null))).get("replicatedSize"));
Assert.assertNotNull(result.get("segments"));
Assert.assertEquals("2010-01-01T00:00:00.000Z",
result.get("segments").get("minTime").toString());
Assert.assertEquals("2010-01-02T00:00:00.000Z",
result.get("segments").get("maxTime").toString());
Assert.assertEquals(1, result.get("segments").get("count"));
Assert.assertEquals(10L, result.get("segments").get("size"));
+ Assert.assertEquals(10L, result.get("segments").get("replicatedSize"));
EasyMock.verify(inventoryView, server);
}
@@ -413,6 +415,7 @@ public class DataSourcesResourceTest
Map<String, Map<String, Object>> result = (Map<String, Map<String,
Object>>) response.getEntity();
Assert.assertEquals(2, ((Map)
(result.get("tiers").get("cold"))).get("segmentCount"));
Assert.assertEquals(30L, ((Map)
(result.get("tiers").get("cold"))).get("size"));
+ Assert.assertEquals(30L, ((Map)
(result.get("tiers").get("cold"))).get("replicatedSize"));
Assert.assertEquals(1, ((Map)
(result.get("tiers").get("hot"))).get("segmentCount"));
Assert.assertEquals(20L, ((Map)
(result.get("tiers").get("hot"))).get("size"));
Assert.assertNotNull(result.get("segments"));
@@ -420,10 +423,63 @@ public class DataSourcesResourceTest
Assert.assertEquals("2010-01-23T00:00:00.000Z",
result.get("segments").get("maxTime").toString());
Assert.assertEquals(2, result.get("segments").get("count"));
Assert.assertEquals(30L, result.get("segments").get("size"));
+ Assert.assertEquals(50L, result.get("segments").get("replicatedSize"));
EasyMock.verify(inventoryView, server, server2, server3);
}
@Test
+ public void testSimpleGetTheDataSourceWithReplicatedSegments()
+ {
+ server = new DruidServer("server1", "host1", null, 1234,
ServerType.HISTORICAL, "tier1", 0);
+ DruidServer server2 = new DruidServer("server2", "host2", null, 1234,
ServerType.HISTORICAL, "tier2", 0);
+ DruidServer server3 = new DruidServer("server3", "host3", null, 1234,
ServerType.HISTORICAL, "tier1", 0);
+
+ server.addDataSegment(dataSegmentList.get(0));
+ server.addDataSegment(dataSegmentList.get(1));
+ server.addDataSegment(dataSegmentList.get(2));
+ server2.addDataSegment(dataSegmentList.get(0));
+ server2.addDataSegment(dataSegmentList.get(1));
+ server3.addDataSegment(dataSegmentList.get(2));
+
+ EasyMock.expect(inventoryView.getInventory()).andReturn(
+ ImmutableList.of(server, server2, server3)
+ ).atLeastOnce();
+
+ EasyMock.replay(inventoryView);
+
+ DataSourcesResource dataSourcesResource =
+ new DataSourcesResource(inventoryView, null, null, null, new
AuthConfig(), null);
+ Response response = dataSourcesResource.getTheDataSource("datasource1",
null);
+ Assert.assertEquals(200, response.getStatus());
+ Map<String, Map<String, Object>> result1 = (Map<String, Map<String,
Object>>) response.getEntity();
+ Assert.assertEquals(2, ((Map)
(result1.get("tiers").get("tier1"))).get("segmentCount"));
+ Assert.assertEquals(30L, ((Map)
(result1.get("tiers").get("tier1"))).get("size"));
+ Assert.assertEquals(30L, ((Map)
(result1.get("tiers").get("tier1"))).get("replicatedSize"));
+ Assert.assertEquals(2, ((Map)
(result1.get("tiers").get("tier2"))).get("segmentCount"));
+ Assert.assertEquals(30L, ((Map)
(result1.get("tiers").get("tier2"))).get("size"));
+ Assert.assertNotNull(result1.get("segments"));
+ Assert.assertEquals("2010-01-01T00:00:00.000Z",
result1.get("segments").get("minTime").toString());
+ Assert.assertEquals("2010-01-23T00:00:00.000Z",
result1.get("segments").get("maxTime").toString());
+ Assert.assertEquals(2, result1.get("segments").get("count"));
+ Assert.assertEquals(30L, result1.get("segments").get("size"));
+ Assert.assertEquals(60L, result1.get("segments").get("replicatedSize"));
+
+ response = dataSourcesResource.getTheDataSource("datasource2", null);
+ Assert.assertEquals(200, response.getStatus());
+ Map<String, Map<String, Object>> result2 = (Map<String, Map<String,
Object>>) response.getEntity();
+ Assert.assertEquals(1, ((Map)
(result2.get("tiers").get("tier1"))).get("segmentCount"));
+ Assert.assertEquals(30L, ((Map)
(result2.get("tiers").get("tier1"))).get("size"));
+ Assert.assertEquals(60L, ((Map)
(result2.get("tiers").get("tier1"))).get("replicatedSize"));
+ Assert.assertNotNull(result2.get("segments"));
+ Assert.assertEquals("2010-01-01T00:00:00.000Z",
result2.get("segments").get("minTime").toString());
+ Assert.assertEquals("2010-01-02T00:00:00.000Z",
result2.get("segments").get("maxTime").toString());
+ Assert.assertEquals(1, result2.get("segments").get("count"));
+ Assert.assertEquals(30L, result2.get("segments").get("size"));
+ Assert.assertEquals(60L, result2.get("segments").get("replicatedSize"));
+ EasyMock.verify(inventoryView);
+ }
+
+ @Test
public void testGetSegmentDataSourceIntervals()
{
server = new DruidServer("who", "host", null, 1234, ServerType.HISTORICAL,
"tier1", 0);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
