This is an automated email from the ASF dual-hosted git repository.
cwylie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new fc55598 Remove payload field from table sys.segment (#9883)
fc55598 is described below
commit fc555980e8cadc61fcb8cee9d6dadec9cf0e0496
Author: Yuanli Han <[email protected]>
AuthorDate: Tue Jun 30 13:20:23 2020 +0800
Remove payload field from table sys.segment (#9883)
* remove payload field from table sys.segments
* update doc
* fix test
* fix CI failure
* add necessary fields
* fix doc
* fix comment
---
docs/querying/sql.md | 4 +-
.../results/auth_test_sys_schema_segments.json | 4 +-
.../druid/sql/calcite/schema/SystemSchema.java | 112 ++++++++++-----------
.../druid/sql/calcite/schema/SystemSchemaTest.java | 7 +-
.../src/views/segments-view/segments-view.tsx | 14 +--
5 files changed, 68 insertions(+), 73 deletions(-)
diff --git a/docs/querying/sql.md b/docs/querying/sql.md
index 12e73a9..895972f 100644
--- a/docs/querying/sql.md
+++ b/docs/querying/sql.md
@@ -994,7 +994,9 @@ Segments table provides details on all Druid segments,
whether they are publishe
|is_available|LONG|Boolean is represented as long type where 1 = true, 0 =
false. 1 if this segment is currently being served by any process(Historical or
realtime). See the [Architecture
page](../design/architecture.md#segment-lifecycle) for more details.|
|is_realtime|LONG|Boolean is represented as long type where 1 = true, 0 =
false. 1 if this segment is _only_ served by realtime tasks, and 0 if any
historical process is serving this segment.|
|is_overshadowed|LONG|Boolean is represented as long type where 1 = true, 0 =
false. 1 if this segment is published and is _fully_ overshadowed by some other
published segments. Currently, is_overshadowed is always false for unpublished
segments, although this may change in the future. You can filter for segments
that "should be published" by filtering for `is_published = 1 AND
is_overshadowed = 0`. Segments can briefly be both published and overshadowed
if they were recently replaced, b [...]
-|payload|STRING|JSON-serialized data segment payload|
+|shardSpec|STRING|The toString of specific `ShardSpec`|
+|dimensions|STRING|The dimensions of the segment|
+|metrics|STRING|The metrics of the segment|
For example to retrieve all segments for datasource "wikipedia", use the query:
diff --git
a/integration-tests/src/test/resources/results/auth_test_sys_schema_segments.json
b/integration-tests/src/test/resources/results/auth_test_sys_schema_segments.json
index a169cfe..d59b14f 100644
---
a/integration-tests/src/test/resources/results/auth_test_sys_schema_segments.json
+++
b/integration-tests/src/test/resources/results/auth_test_sys_schema_segments.json
@@ -13,6 +13,8 @@
"is_available": 1,
"is_realtime": 0,
"is_overshadowed": 0,
- "payload":
"{\"overshadowed\":false,\"dataSource\":\"auth_test\",\"interval\":\"2012-12-29T00:00:00.000Z/2013-01-10T08:00:00.000Z\",\"version\":\"2013-01-10T08:13:47.830Z_v9\",\"loadSpec\":{\"load
spec is pruned, because it's not needed on Brokers, but eats a lot of heap
space\":\"\"},\"dimensions\":\"anonymous,area_code,city,continent_code,country_name,dma_code,geo,language,namespace,network,newpage,page,postal_code,region_lookup,robot,unpatrolled,user\",\"metrics\":\"added,count,de
[...]
+ "shardSpec": "NoneShardSpec",
+ "dimensions": "[anonymous, area_code, city, continent_code, country_name,
dma_code, geo, language, namespace, network, newpage, page, postal_code,
region_lookup, robot, unpatrolled, user]",
+ "metrics": "[added, count, deleted, delta, delta_hist, unique_users,
variation]"
}
]
diff --git
a/sql/src/main/java/org/apache/druid/sql/calcite/schema/SystemSchema.java
b/sql/src/main/java/org/apache/druid/sql/calcite/schema/SystemSchema.java
index c6bff43..72633a1 100644
--- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/SystemSchema.java
+++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/SystemSchema.java
@@ -19,7 +19,6 @@
package org.apache.druid.sql.calcite.schema;
-import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.JavaType;
import com.fasterxml.jackson.databind.ObjectMapper;
@@ -56,7 +55,6 @@ import org.apache.druid.discovery.DruidNodeDiscoveryProvider;
import org.apache.druid.discovery.NodeRole;
import org.apache.druid.indexer.TaskStatusPlus;
import org.apache.druid.indexing.overlord.supervisor.SupervisorStatus;
-import org.apache.druid.java.util.common.RE;
import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.java.util.common.parsers.CloseableIterator;
import org.apache.druid.java.util.http.client.Request;
@@ -143,7 +141,9 @@ public class SystemSchema extends AbstractSchema
.add("is_available", ValueType.LONG)
.add("is_realtime", ValueType.LONG)
.add("is_overshadowed", ValueType.LONG)
- .add("payload", ValueType.STRING)
+ .add("shardSpec", ValueType.STRING)
+ .add("dimensions", ValueType.STRING)
+ .add("metrics", ValueType.STRING)
.build();
static final RowSignature SERVERS_SIGNATURE = RowSignature
@@ -294,37 +294,34 @@ public class SystemSchema extends AbstractSchema
final FluentIterable<Object[]> publishedSegments = FluentIterable
.from(() -> getAuthorizedPublishedSegments(metadataStoreSegments,
root))
.transform(val -> {
- try {
- final DataSegment segment = val.getDataSegment();
- segmentsAlreadySeen.add(segment.getId());
- final PartialSegmentData partialSegmentData =
partialSegmentDataMap.get(segment.getId());
- long numReplicas = 0L, numRows = 0L, isRealtime = 0L,
isAvailable = 0L;
- if (partialSegmentData != null) {
- numReplicas = partialSegmentData.getNumReplicas();
- numRows = partialSegmentData.getNumRows();
- isAvailable = partialSegmentData.isAvailable();
- isRealtime = partialSegmentData.isRealtime();
- }
- return new Object[]{
- segment.getId(),
- segment.getDataSource(),
- segment.getInterval().getStart().toString(),
- segment.getInterval().getEnd().toString(),
- segment.getSize(),
- segment.getVersion(),
- Long.valueOf(segment.getShardSpec().getPartitionNum()),
- numReplicas,
- numRows,
- IS_PUBLISHED_TRUE, //is_published is true for published
segments
- isAvailable,
- isRealtime,
- val.isOvershadowed() ? IS_OVERSHADOWED_TRUE :
IS_OVERSHADOWED_FALSE,
- jsonMapper.writeValueAsString(val)
- };
- }
- catch (JsonProcessingException e) {
- throw new RE(e, "Error getting segment payload for segment %s",
val.getDataSegment().getId());
+ final DataSegment segment = val.getDataSegment();
+ segmentsAlreadySeen.add(segment.getId());
+ final PartialSegmentData partialSegmentData =
partialSegmentDataMap.get(segment.getId());
+ long numReplicas = 0L, numRows = 0L, isRealtime = 0L, isAvailable
= 0L;
+ if (partialSegmentData != null) {
+ numReplicas = partialSegmentData.getNumReplicas();
+ numRows = partialSegmentData.getNumRows();
+ isAvailable = partialSegmentData.isAvailable();
+ isRealtime = partialSegmentData.isRealtime();
}
+ return new Object[]{
+ segment.getId(),
+ segment.getDataSource(),
+ segment.getInterval().getStart().toString(),
+ segment.getInterval().getEnd().toString(),
+ segment.getSize(),
+ segment.getVersion(),
+ Long.valueOf(segment.getShardSpec().getPartitionNum()),
+ numReplicas,
+ numRows,
+ IS_PUBLISHED_TRUE, //is_published is true for published
segments
+ isAvailable,
+ isRealtime,
+ val.isOvershadowed() ? IS_OVERSHADOWED_TRUE :
IS_OVERSHADOWED_FALSE,
+ segment.getShardSpec(),
+ segment.getDimensions(),
+ segment.getMetrics()
+ };
});
final FluentIterable<Object[]> availableSegments = FluentIterable
@@ -333,33 +330,30 @@ public class SystemSchema extends AbstractSchema
root
))
.transform(val -> {
- try {
- if (segmentsAlreadySeen.contains(val.getKey())) {
- return null;
- }
- final PartialSegmentData partialSegmentData =
partialSegmentDataMap.get(val.getKey());
- final long numReplicas = partialSegmentData == null ? 0L :
partialSegmentData.getNumReplicas();
- return new Object[]{
- val.getKey(),
- val.getKey().getDataSource(),
- val.getKey().getInterval().getStart().toString(),
- val.getKey().getInterval().getEnd().toString(),
- val.getValue().getSegment().getSize(),
- val.getKey().getVersion(),
- (long)
val.getValue().getSegment().getShardSpec().getPartitionNum(),
- numReplicas,
- val.getValue().getNumRows(),
- IS_PUBLISHED_FALSE, // is_published is false for unpublished
segments
- // is_available is assumed to be always true for segments
announced by historicals or realtime tasks
- IS_AVAILABLE_TRUE,
- val.getValue().isRealtime(),
- IS_OVERSHADOWED_FALSE, // there is an assumption here that
unpublished segments are never overshadowed
- jsonMapper.writeValueAsString(val.getKey())
- };
- }
- catch (JsonProcessingException e) {
- throw new RE(e, "Error getting segment payload for segment %s",
val.getKey());
+ if (segmentsAlreadySeen.contains(val.getKey())) {
+ return null;
}
+ final PartialSegmentData partialSegmentData =
partialSegmentDataMap.get(val.getKey());
+ final long numReplicas = partialSegmentData == null ? 0L :
partialSegmentData.getNumReplicas();
+ return new Object[]{
+ val.getKey(),
+ val.getKey().getDataSource(),
+ val.getKey().getInterval().getStart().toString(),
+ val.getKey().getInterval().getEnd().toString(),
+ val.getValue().getSegment().getSize(),
+ val.getKey().getVersion(),
+ (long)
val.getValue().getSegment().getShardSpec().getPartitionNum(),
+ numReplicas,
+ val.getValue().getNumRows(),
+ IS_PUBLISHED_FALSE, // is_published is false for unpublished
segments
+ // is_available is assumed to be always true for segments
announced by historicals or realtime tasks
+ IS_AVAILABLE_TRUE,
+ val.getValue().isRealtime(),
+ IS_OVERSHADOWED_FALSE, // there is an assumption here that
unpublished segments are never overshadowed
+ val.getValue().getSegment().getShardSpec(),
+ val.getValue().getSegment().getDimensions(),
+ val.getValue().getSegment().getMetrics()
+ };
});
final Iterable<Object[]> allSegments = Iterables.unmodifiableIterable(
diff --git
a/sql/src/test/java/org/apache/druid/sql/calcite/schema/SystemSchemaTest.java
b/sql/src/test/java/org/apache/druid/sql/calcite/schema/SystemSchemaTest.java
index ace7aca..a0f3083 100644
---
a/sql/src/test/java/org/apache/druid/sql/calcite/schema/SystemSchemaTest.java
+++
b/sql/src/test/java/org/apache/druid/sql/calcite/schema/SystemSchemaTest.java
@@ -91,6 +91,7 @@ import org.apache.druid.timeline.DataSegment;
import org.apache.druid.timeline.SegmentId;
import org.apache.druid.timeline.SegmentWithOvershadowedStatus;
import org.apache.druid.timeline.partition.NumberedShardSpec;
+import org.apache.druid.timeline.partition.ShardSpec;
import org.easymock.EasyMock;
import org.jboss.netty.handler.codec.http.HttpMethod;
import org.jboss.netty.handler.codec.http.HttpResponse;
@@ -472,7 +473,7 @@ public class SystemSchemaTest extends CalciteTestBase
final RelDataType rowType = segmentsTable.getRowType(new
JavaTypeFactoryImpl());
final List<RelDataTypeField> fields = rowType.getFieldList();
- Assert.assertEquals(14, fields.size());
+ Assert.assertEquals(16, fields.size());
final SystemSchema.TasksTable tasksTable = (SystemSchema.TasksTable)
schema.getTableMap().get("tasks");
final RelDataType sysRowType = tasksTable.getRowType(new
JavaTypeFactoryImpl());
@@ -1248,6 +1249,10 @@ public class SystemSchemaTest extends CalciteTestBase
case STRING:
if (signature.getColumnName(i).equals("segment_id")) {
expectedClass = SegmentId.class;
+ } else if (signature.getColumnName(i).equals("shardSpec")) {
+ expectedClass = ShardSpec.class;
+ } else if (signature.getColumnName(i).equals("dimensions") ||
signature.getColumnName(i).equals("metrics")) {
+ expectedClass = List.class;
} else {
expectedClass = String.class;
}
diff --git a/web-console/src/views/segments-view/segments-view.tsx
b/web-console/src/views/segments-view/segments-view.tsx
index a7d4e39..474e672 100644
--- a/web-console/src/views/segments-view/segments-view.tsx
+++ b/web-console/src/views/segments-view/segments-view.tsx
@@ -146,7 +146,6 @@ interface SegmentQueryResultRow {
version: string;
size: 0;
partition_num: number;
- payload: any;
num_rows: number;
num_replicas: number;
is_available: number;
@@ -211,6 +210,7 @@ export class SegmentsView extends
React.PureComponent<SegmentsViewProps, Segment
`FROM sys.segments`,
whereClause ? `WHERE ${whereClause}` : '',
`GROUP BY 1`,
+ `ORDER BY 1 DESC`,
`LIMIT ${totalQuerySize}`,
]).join('\n');
@@ -221,7 +221,7 @@ export class SegmentsView extends
React.PureComponent<SegmentsViewProps, Segment
queryParts = compact([
`SELECT`,
` ("start" || '/' || "end") AS "interval",`,
- ` "segment_id", "datasource", "start", "end", "size", "version",
"partition_num", "num_replicas", "num_rows", "is_published", "is_available",
"is_realtime", "is_overshadowed", "payload"`,
+ ` "segment_id", "datasource", "start", "end", "size", "version",
"partition_num", "num_replicas", "num_rows", "is_published", "is_available",
"is_realtime", "is_overshadowed"`,
`FROM sys.segments`,
`WHERE`,
intervals ? ` ("start" || '/' || "end") IN (${intervals})` :
'FALSE',
@@ -240,7 +240,7 @@ export class SegmentsView extends
React.PureComponent<SegmentsViewProps, Segment
queryParts.push(`LIMIT ${totalQuerySize * 1000}`);
} else {
queryParts = [
- `SELECT "segment_id", "datasource", "start", "end", "size",
"version", "partition_num", "num_replicas", "num_rows", "is_published",
"is_available", "is_realtime", "is_overshadowed", "payload"`,
+ `SELECT "segment_id", "datasource", "start", "end", "size",
"version", "partition_num", "num_replicas", "num_rows", "is_published",
"is_available", "is_realtime", "is_overshadowed"`,
`FROM sys.segments`,
];
@@ -264,13 +264,6 @@ export class SegmentsView extends
React.PureComponent<SegmentsViewProps, Segment
const results: any[] = (await queryDruidSql({ query: sqlQuery
})).slice(
query.page * query.pageSize,
);
- results.forEach(result => {
- try {
- result.payload = JSON.parse(result.payload);
- } catch {
- result.payload = {};
- }
- });
return results;
},
onStateChange: ({ result, loading, error }) => {
@@ -299,7 +292,6 @@ export class SegmentsView extends
React.PureComponent<SegmentsViewProps, Segment
version: segment.version,
partition_num: segment.shardSpec.partitionNum ? 0 :
segment.shardSpec.partitionNum,
size: segment.size,
- payload: segment,
num_rows: -1,
num_replicas: -1,
is_available: -1,
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
