This is an automated email from the ASF dual-hosted git repository.
mayanks pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 2831223 Support get metadata and indexes API for REALTIME tables.
(#7169)
2831223 is described below
commit 28312234d1a10224f92a055c89cf3b606e3350f5
Author: Ramakrishna Baratam <[email protected]>
AuthorDate: Mon Aug 2 16:42:12 2021 -0700
Support get metadata and indexes API for REALTIME tables. (#7169)
* Support get metadata and indexes API for REALTIME tables.
- Used segment.getDataSource(columnName) API which is implemented for both
Mutable and Immutable segments.
- Removed the exposed private data API for ImmutableSegment.
- Returning metadata and indexes for non-virtual columns
- Added additional checks in test and checking for same number of columns
returned in metadata and indexes
* Including changes for UI for the Reload Status button (from Sanket)
---
.../api/resources/PinotSegmentRestletResource.java | 4 -
.../src/main/resources/app/requests/index.ts | 2 +-
.../immutable/ImmutableSegmentImpl.java | 4 -
.../spi/index/metadata/SegmentMetadataImpl.java | 2 +-
.../api/resources/SegmentMetadataFetcher.java | 121 +++++++++++----------
.../pinot/server/api/TablesResourceTest.java | 14 ++-
6 files changed, 74 insertions(+), 73 deletions(-)
diff --git
a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotSegmentRestletResource.java
b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotSegmentRestletResource.java
index 3ffd601..2a0cc86 100644
---
a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotSegmentRestletResource.java
+++
b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotSegmentRestletResource.java
@@ -598,10 +598,6 @@ public class PinotSegmentRestletResource {
@ApiParam(value = "Columns name", allowMultiple = true)
@QueryParam("columns") @DefaultValue("") List<String> columns) {
LOGGER.info("Received a request to fetch metadata for all segments for
table {}", tableName);
TableType tableType = Constants.validateTableType(tableTypeStr);
- if (tableType == TableType.REALTIME) {
- throw new ControllerApplicationException(LOGGER, "Table type : " +
tableTypeStr + " not yet supported.",
- Status.NOT_IMPLEMENTED);
- }
String tableNameWithType =
ResourceUtils.getExistingTableNamesWithType(_pinotHelixResourceManager,
tableName, tableType, LOGGER).get(0);
diff --git a/pinot-controller/src/main/resources/app/requests/index.ts
b/pinot-controller/src/main/resources/app/requests/index.ts
index ae729ed..2502896 100644
--- a/pinot-controller/src/main/resources/app/requests/index.ts
+++ b/pinot-controller/src/main/resources/app/requests/index.ts
@@ -133,7 +133,7 @@ export const reloadAllSegments = (tableName: string,
tableType: string): Promise
baseApi.post(`/segments/${tableName}/reload?type=${tableType}`, null,
{headers});
export const reloadStatus = (tableName: string, tableType: string):
Promise<AxiosResponse<OperationResponse>> =>
- baseApi.get(`/segments/${tableName}/metadata?type=${tableType}`);
+ baseApi.get(`/segments/${tableName}/metadata?type=${tableType}&columns=*`);
export const deleteSegment = (tableName: string, instanceName: string):
Promise<AxiosResponse<OperationResponse>> =>
baseApi.delete(`/segments/${tableName}/${instanceName}`, {headers});
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/indexsegment/immutable/ImmutableSegmentImpl.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/indexsegment/immutable/ImmutableSegmentImpl.java
index d642179..143c464 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/indexsegment/immutable/ImmutableSegmentImpl.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/indexsegment/immutable/ImmutableSegmentImpl.java
@@ -193,8 +193,4 @@ public class ImmutableSegmentImpl implements
ImmutableSegment {
throw new RuntimeException("Failed to use PinotSegmentRecordReader to
read immutable segment");
}
}
-
- public Map<String, ColumnIndexContainer> getIndexContainerMap() {
- return _indexContainerMap;
- }
}
diff --git
a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/metadata/SegmentMetadataImpl.java
b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/metadata/SegmentMetadataImpl.java
index 9cb43f2..bd66a92 100644
---
a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/metadata/SegmentMetadataImpl.java
+++
b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/metadata/SegmentMetadataImpl.java
@@ -400,7 +400,7 @@ public class SegmentMetadataImpl implements SegmentMetadata
{
segmentMetadata.put("endTimeReadable",
_timeInterval.getEnd().toString());
}
- segmentMetadata.put("segmentVersion", _segmentVersion.toString());
+ segmentMetadata.put("segmentVersion", ((_segmentVersion != null) ?
_segmentVersion.toString() : null));
segmentMetadata.put("creatorName", _creatorName);
ObjectNode customConfigs = JsonUtils.newObjectNode();
diff --git
a/pinot-server/src/main/java/org/apache/pinot/server/api/resources/SegmentMetadataFetcher.java
b/pinot-server/src/main/java/org/apache/pinot/server/api/resources/SegmentMetadataFetcher.java
index 2441de1..a2f5481 100644
---
a/pinot-server/src/main/java/org/apache/pinot/server/api/resources/SegmentMetadataFetcher.java
+++
b/pinot-server/src/main/java/org/apache/pinot/server/api/resources/SegmentMetadataFetcher.java
@@ -29,15 +29,15 @@ import java.util.Objects;
import java.util.Set;
import javax.annotation.Nullable;
import org.apache.pinot.segment.local.data.manager.SegmentDataManager;
-import
org.apache.pinot.segment.local.indexsegment.immutable.ImmutableSegmentImpl;
import org.apache.pinot.segment.spi.IndexSegment;
import org.apache.pinot.segment.spi.SegmentMetadata;
-import org.apache.pinot.segment.spi.index.column.ColumnIndexContainer;
+import org.apache.pinot.segment.spi.datasource.DataSource;
import
org.apache.pinot.segment.spi.index.startree.AggregationFunctionColumnPair;
import org.apache.pinot.segment.spi.index.startree.StarTreeV2;
import org.apache.pinot.segment.spi.index.startree.StarTreeV2Metadata;
import org.apache.pinot.spi.utils.JsonUtils;
+
/**
* This is a wrapper class for fetching segment metadata related information.
*/
@@ -61,93 +61,95 @@ public class SegmentMetadataFetcher {
private static final String STAR_TREE_MAX_LEAF_RECORDS = "max-leaf-records";
private static final String STAR_TREE_DIMENSION_COLUMNS_SKIPPED =
"dimension-columns-skipped";
-
/**
* This is a helper method that fetches the segment metadata for a given
segment.
* @param columns Columns to include for metadata
*/
public static String getSegmentMetadata(SegmentDataManager
segmentDataManager, List<String> columns)
throws JsonProcessingException {
- SegmentMetadata segmentMetadata =
segmentDataManager.getSegment().getSegmentMetadata();
+ IndexSegment segment = segmentDataManager.getSegment();
+ SegmentMetadata segmentMetadata = segment.getSegmentMetadata();
Set<String> columnSet;
if (columns.size() == 1 && columns.get(0).equals("*")) {
- columnSet = null;
+ // Making code consistent and returning metadata and indexes only for
non-virtual columns.
+ columnSet = segment.getPhysicalColumnNames();
} else {
columnSet = new HashSet<>(columns);
}
ObjectNode segmentMetadataJson = (ObjectNode)
segmentMetadata.toJson(columnSet);
- segmentMetadataJson.set(COLUMN_INDEX_KEY,
JsonUtils.objectToJsonNode(getIndexesForSegmentColumns(segmentDataManager)));
- segmentMetadataJson.set(STAR_TREE_INDEX_KEY,
JsonUtils.objectToJsonNode((getStarTreeIndexesForSegment(segmentDataManager))));
+ segmentMetadataJson
+ .set(COLUMN_INDEX_KEY,
JsonUtils.objectToJsonNode(getIndexesForSegmentColumns(segmentDataManager,
columnSet)));
+ segmentMetadataJson
+ .set(STAR_TREE_INDEX_KEY,
JsonUtils.objectToJsonNode((getStarTreeIndexesForSegment(segmentDataManager))));
return JsonUtils.objectToString(segmentMetadataJson);
}
/**
* Get the JSON object with the segment column's indexing metadata.
+ * Lists all the columns if the parameter columnSet is null.
*/
- @Nullable
- private static Map<String, Map<String, String>>
getIndexesForSegmentColumns(SegmentDataManager segmentDataManager) {
+ private static Map<String, Map<String, String>>
getIndexesForSegmentColumns(SegmentDataManager segmentDataManager,
+ @Nullable Set<String> columnSet) {
IndexSegment segment = segmentDataManager.getSegment();
- if (segment instanceof ImmutableSegmentImpl) {
- return getImmutableSegmentColumnIndexes(((ImmutableSegmentImpl)
segment).getIndexContainerMap());
- } else {
- return null;
+ Map<String, Map<String, String>> columnIndexMap = new LinkedHashMap<>();
+ for (String physicalColumnName : segment.getPhysicalColumnNames()) {
+ if (columnSet == null || columnSet.contains(physicalColumnName)) {
+ DataSource dataSource = segment.getDataSource(physicalColumnName);
+ columnIndexMap.put(physicalColumnName, getColumnIndexes(dataSource));
+ }
}
+ return columnIndexMap;
}
/**
- * Helper to loop through column index container to create a index map as
follows for each column:
+ * Helper to loop through column datasource to create a index map as follows
for each column:
* {<"bloom-filter", "YES">, <"dictionary", "NO">}
*/
- private static Map<String, Map<String, String>>
getImmutableSegmentColumnIndexes(Map<String, ColumnIndexContainer>
columnIndexContainerMap) {
- Map<String, Map<String, String>> columnIndexMap = new LinkedHashMap<>();
- for (Map.Entry<String, ColumnIndexContainer> entry :
columnIndexContainerMap.entrySet()) {
- ColumnIndexContainer columnIndexContainer = entry.getValue();
- Map<String, String> indexStatus = new LinkedHashMap<>();
- if (Objects.isNull(columnIndexContainer.getBloomFilter())) {
- indexStatus.put(BLOOM_FILTER, INDEX_NOT_AVAILABLE);
- } else {
- indexStatus.put(BLOOM_FILTER, INDEX_AVAILABLE);
- }
-
- if (Objects.isNull(columnIndexContainer.getDictionary())) {
- indexStatus.put(DICTIONARY, INDEX_NOT_AVAILABLE);
- } else {
- indexStatus.put(DICTIONARY, INDEX_AVAILABLE);
- }
+ private static Map<String, String> getColumnIndexes(DataSource dataSource) {
+ Map<String, String> indexStatus = new LinkedHashMap<>();
+ if (Objects.isNull(dataSource.getBloomFilter())) {
+ indexStatus.put(BLOOM_FILTER, INDEX_NOT_AVAILABLE);
+ } else {
+ indexStatus.put(BLOOM_FILTER, INDEX_AVAILABLE);
+ }
- if (Objects.isNull(columnIndexContainer.getForwardIndex())) {
- indexStatus.put(FORWARD_INDEX, INDEX_NOT_AVAILABLE);
- } else {
- indexStatus.put(FORWARD_INDEX, INDEX_AVAILABLE);
- }
+ if (Objects.isNull(dataSource.getDictionary())) {
+ indexStatus.put(DICTIONARY, INDEX_NOT_AVAILABLE);
+ } else {
+ indexStatus.put(DICTIONARY, INDEX_AVAILABLE);
+ }
- if (Objects.isNull(columnIndexContainer.getInvertedIndex())) {
- indexStatus.put(INVERTED_INDEX, INDEX_NOT_AVAILABLE);
- } else {
- indexStatus.put(INVERTED_INDEX, INDEX_AVAILABLE);
- }
+ if (Objects.isNull(dataSource.getForwardIndex())) {
+ indexStatus.put(FORWARD_INDEX, INDEX_NOT_AVAILABLE);
+ } else {
+ indexStatus.put(FORWARD_INDEX, INDEX_AVAILABLE);
+ }
- if (Objects.isNull(columnIndexContainer.getNullValueVector())) {
- indexStatus.put(NULL_VALUE_VECTOR_READER, INDEX_NOT_AVAILABLE);
- } else {
- indexStatus.put(NULL_VALUE_VECTOR_READER, INDEX_AVAILABLE);
- }
+ if (Objects.isNull(dataSource.getInvertedIndex())) {
+ indexStatus.put(INVERTED_INDEX, INDEX_NOT_AVAILABLE);
+ } else {
+ indexStatus.put(INVERTED_INDEX, INDEX_AVAILABLE);
+ }
- if (Objects.isNull(columnIndexContainer.getRangeIndex())) {
- indexStatus.put(RANGE_INDEX, INDEX_NOT_AVAILABLE);
- } else {
- indexStatus.put(RANGE_INDEX, INDEX_AVAILABLE);
- }
+ if (Objects.isNull(dataSource.getNullValueVector())) {
+ indexStatus.put(NULL_VALUE_VECTOR_READER, INDEX_NOT_AVAILABLE);
+ } else {
+ indexStatus.put(NULL_VALUE_VECTOR_READER, INDEX_AVAILABLE);
+ }
- if (Objects.isNull(columnIndexContainer.getJsonIndex())){
- indexStatus.put(JSON_INDEX, INDEX_NOT_AVAILABLE);
- } else {
- indexStatus.put(JSON_INDEX, INDEX_AVAILABLE);
- }
+ if (Objects.isNull(dataSource.getRangeIndex())) {
+ indexStatus.put(RANGE_INDEX, INDEX_NOT_AVAILABLE);
+ } else {
+ indexStatus.put(RANGE_INDEX, INDEX_AVAILABLE);
+ }
- columnIndexMap.put(entry.getKey(), indexStatus);
+ if (Objects.isNull(dataSource.getJsonIndex())) {
+ indexStatus.put(JSON_INDEX, INDEX_NOT_AVAILABLE);
+ } else {
+ indexStatus.put(JSON_INDEX, INDEX_AVAILABLE);
}
- return columnIndexMap;
+
+ return indexStatus;
}
/**
@@ -162,7 +164,7 @@ public class SegmentMetadataFetcher {
/**
* Helper to loop over star trees of a segment to create a map containing
star tree details.
*/
- private static List<Map<String, Object>> getStarTreeIndexes(List<StarTreeV2>
starTrees){
+ private static List<Map<String, Object>> getStarTreeIndexes(List<StarTreeV2>
starTrees) {
List<Map<String, Object>> startreeDetails = new ArrayList<>();
for (StarTreeV2 starTree : starTrees) {
StarTreeV2Metadata starTreeMetadata = starTree.getMetadata();
@@ -180,7 +182,8 @@ public class SegmentMetadataFetcher {
starTreeIndexMap.put(STAR_TREE_METRIC_AGGREGATIONS,
starTreeMetricAggregations);
starTreeIndexMap.put(STAR_TREE_MAX_LEAF_RECORDS,
starTreeMetadata.getMaxLeafRecords());
- starTreeIndexMap.put(STAR_TREE_DIMENSION_COLUMNS_SKIPPED,
starTreeMetadata.getSkipStarNodeCreationForDimensions());
+ starTreeIndexMap
+ .put(STAR_TREE_DIMENSION_COLUMNS_SKIPPED,
starTreeMetadata.getSkipStarNodeCreationForDimensions());
startreeDetails.add(starTreeIndexMap);
}
return startreeDetails;
diff --git
a/pinot-server/src/test/java/org/apache/pinot/server/api/TablesResourceTest.java
b/pinot-server/src/test/java/org/apache/pinot/server/api/TablesResourceTest.java
index 997dbfb..b368745 100644
---
a/pinot-server/src/test/java/org/apache/pinot/server/api/TablesResourceTest.java
+++
b/pinot-server/src/test/java/org/apache/pinot/server/api/TablesResourceTest.java
@@ -117,15 +117,19 @@ public class TablesResourceTest extends BaseResourceTest {
Assert.assertTrue(jsonResponse.has("endTimeReadable"));
Assert.assertTrue(jsonResponse.has("creationTimeReadable"));
Assert.assertEquals(jsonResponse.get("columns").size(), 0);
+ Assert.assertEquals(jsonResponse.get("indexes").size(), 0);
jsonResponse = JsonUtils.stringToJsonNode(
_webTarget.path(segmentMetadataPath).queryParam("columns",
"column1").queryParam("columns", "column2").request()
.get(String.class));
Assert.assertEquals(jsonResponse.get("columns").size(), 2);
+ Assert.assertEquals(jsonResponse.get("indexes").size(), 2);
jsonResponse = JsonUtils.stringToJsonNode(
(_webTarget.path(segmentMetadataPath).queryParam("columns",
"*").request().get(String.class)));
- Assert.assertEquals(jsonResponse.get("columns").size(),
segmentMetadata.getAllColumns().size());
+ int physicalColumnCount = defaultSegment.getPhysicalColumnNames().size();
+ Assert.assertEquals(jsonResponse.get("columns").size(),
physicalColumnCount);
+ Assert.assertEquals(jsonResponse.get("indexes").size(),
physicalColumnCount);
Response response = _webTarget.path("/tables/UNKNOWN_TABLE/segments/" +
defaultSegment.getSegmentName()).request()
.get(Response.class);
@@ -264,18 +268,20 @@ public class TablesResourceTest extends BaseResourceTest {
Assert.assertTrue(jsonResponse.has("endTimeReadable"));
Assert.assertTrue(jsonResponse.has("creationTimeReadable"));
Assert.assertEquals(jsonResponse.get("columns").size(), 0);
- Assert.assertEquals(jsonResponse.get("indexes").size(), 17);
+ Assert.assertEquals(jsonResponse.get("indexes").size(), 0);
jsonResponse = JsonUtils.stringToJsonNode(
_webTarget.path(segmentMetadataPath).queryParam("columns",
"column1").queryParam("columns", "column2").request()
.get(String.class));
Assert.assertEquals(jsonResponse.get("columns").size(), 2);
- Assert.assertEquals(jsonResponse.get("indexes").size(), 17);
+ Assert.assertEquals(jsonResponse.get("indexes").size(), 2);
Assert.assertEquals(jsonResponse.get("star-tree-index").size(), 0);
jsonResponse = JsonUtils.stringToJsonNode(
(_webTarget.path(segmentMetadataPath).queryParam("columns",
"*").request().get(String.class)));
- Assert.assertEquals(jsonResponse.get("columns").size(),
segmentMetadata.getAllColumns().size());
+ int physicalColumnCount = defaultSegment.getPhysicalColumnNames().size();
+ Assert.assertEquals(jsonResponse.get("columns").size(),
physicalColumnCount);
+ Assert.assertEquals(jsonResponse.get("indexes").size(),
physicalColumnCount);
Response response = _webTarget.path("/tables/UNKNOWN_TABLE/segments/" +
defaultSegment.getSegmentName()).request()
.get(Response.class);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
